From 18d1e8557fb77163b8775c647f130b0f6f9aa1aa Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 30 Dec 2015 15:44:54 +0100 Subject: [PATCH 001/521] wip --- .gitignore | 1 + README.md | 18 +++ pom.xml | 113 ++++++++++++++++++ .../messageprocessor/parser/RuleLang.g4 | 0 .../messageprocessor/EvaluationContext.java | 4 + .../plugins/messageprocessor/FieldSet.java | 9 ++ .../messageprocessor/ProcessorPlugin.java | 23 ++++ .../ProcessorPluginMetaData.java | 51 ++++++++ .../ProcessorPluginModule.java | 44 +++++++ .../plugins/messageprocessor/ast/Rule.java | 4 + .../ast/expressions/AndExpression.java | 3 + .../ast/expressions/BinaryExpression.java | 3 + .../ast/expressions/BooleanExpression.java | 3 + .../ast/expressions/ComparisonExpression.java | 7 ++ .../ast/expressions/ConstantExpression.java | 4 + .../ast/expressions/DoubleExpression.java | 3 + .../ast/expressions/EqualityExpression.java | 32 +++++ .../ast/expressions/Expression.java | 8 ++ .../ast/expressions/FunctionExpression.java | 7 ++ .../ast/expressions/LogicalExpression.java | 4 + .../ast/expressions/LongExpression.java | 3 + .../ast/expressions/MessageRefExpression.java | 4 + .../ast/expressions/NotExpression.java | 3 + .../ast/expressions/NumericExpression.java | 4 + .../ast/expressions/OrExpression.java | 3 + .../ast/expressions/StringExpression.java | 3 + .../ast/expressions/UnaryExpression.java | 3 + .../ast/expressions/VarRefExpression.java | 4 + .../ast/functions/Function.java | 4 + .../ast/functions/FunctionDescriptor.java | 4 + .../ast/functions/ParameterDescriptor.java | 4 + .../ast/statements/FunctionStatement.java | 4 + .../ast/statements/Statement.java | 4 + .../ast/statements/VarAssignStatement.java | 16 +++ .../db/RuleSourceService.java | 57 +++++++++ .../parser/InvalidReference.java | 12 ++ .../messageprocessor/parser/ParseError.java | 4 + .../parser/ParseException.java | 15 +++ .../messageprocessor/parser/RuleParser.java | 9 ++ .../processors/NaiveRuleProcessor.java | 52 ++++++++ .../rest/MessageProcessorRuleResource.java | 4 + .../messageprocessor/rest/RuleSource.java | 4 + .../services/org.graylog2.plugin.Plugin | 1 + .../parser/RuleParserTest.java | 13 ++ src/test/resources/log4j2-test.xml | 14 +++ .../messageprocessor/parser/basicRule.txt | 0 .../parser/undeclaredIdentifier.txt | 5 + 47 files changed, 594 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pom.xml create mode 100644 src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java create mode 100644 src/main/resources/META-INF/services/org.graylog2.plugin.Plugin create mode 100644 src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java create mode 100644 src/test/resources/log4j2-test.xml create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..bfa6a22a5248 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +# Created by .ignore support plugin (hsz.mobi) diff --git a/README.md b/README.md new file mode 100644 index 000000000000..28a1c86ebfd6 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Graylog ProcessorPlugin Plugin + +Welcome to your new Graylog plugin! + +Please refer to http://docs.graylog.org/en/latest/pages/plugins.html for documentation on how to write +plugins for Graylog. + + +Getting started +--------------- + +This project is using Maven 3 and requires Java 7 or higher. The plugin will require Graylog 1.0.0 or higher. + +* Clone this repository. +* Run `mvn package` to build a JAR file. +* Optional: Run `mvn jdeb:jdeb` and `mvn rpm:rpm` to create a DEB and RPM package respectively. +* Copy generated JAR file in target directory to your Graylog plugin directory. +* Restart the Graylog. diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000000..7a0882f9f75c --- /dev/null +++ b/pom.xml @@ -0,0 +1,113 @@ + + + 4.0.0 + + org.graylog.plugins + message-processor + 1.0.0-SNAPSHOT + jar + + ${project.artifactId} + Graylog ${project.artifactId} plugin. + https://www.graylog.org + + + UTF-8 + 1.7 + 1.7 + 1.0.0 + /usr/share/graylog-server/plugin + + + + + org.graylog2 + graylog2-plugin + ${graylog2.version} + provided + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + false + + + + package + + shade + + + + + + + + + + + + jdeb + org.vafer + 1.3 + + ${project.build.directory}/${project.artifactId}-${project.version}.deb + + + ${project.build.directory}/${project.build.finalName}.jar + file + + perm + ${graylog2.plugin-dir} + 644 + root + root + + + + + + + + org.codehaus.mojo + rpm-maven-plugin + 2.1.2 + + Application/Internet + + /usr + + + _unpackaged_files_terminate_build 0 + _binaries_in_noarch_packages_terminate_build 0 + + 644 + 755 + root + root + + + ${graylog2.plugin-dir} + + + ${project.build.directory}/ + + ${project.build.finalName}.jar + + + + + + + + + + diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java new file mode 100644 index 000000000000..b695a31c6456 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor; + +public class EvaluationContext { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java b/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java new file mode 100644 index 000000000000..0f10f814b9f2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java @@ -0,0 +1,9 @@ +package org.graylog.plugins.messageprocessor; + +public class FieldSet { + private static final FieldSet INSTANCE = new FieldSet(); + + public static FieldSet empty() { + return INSTANCE; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java new file mode 100644 index 000000000000..301f3adb7bef --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java @@ -0,0 +1,23 @@ +package org.graylog.plugins.messageprocessor; + +import org.graylog2.plugin.Plugin; +import org.graylog2.plugin.PluginMetaData; +import org.graylog2.plugin.PluginModule; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Implement the Plugin interface here. + */ +public class ProcessorPlugin implements Plugin { + @Override + public PluginMetaData metadata() { + return new ProcessorPluginMetaData(); + } + + @Override + public Collection modules () { + return Arrays.asList(new ProcessorPluginModule()); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java new file mode 100644 index 000000000000..3417de36244c --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java @@ -0,0 +1,51 @@ +package org.graylog.plugins.messageprocessor; + +import org.graylog2.plugin.PluginMetaData; +import org.graylog2.plugin.ServerStatus; +import org.graylog2.plugin.Version; + +import java.net.URI; +import java.util.Collections; +import java.util.Set; + +public class ProcessorPluginMetaData implements PluginMetaData { + @Override + public String getUniqueId() { + return "org.graylog.plugins.messageprocessor.ProcessorPluginPlugin"; + } + + @Override + public String getName() { + return "Message Processor Plugin"; + } + + @Override + public String getAuthor() { + return "Graylog, Inc"; + } + + @Override + public URI getURL() { + return URI.create("https://www.graylog.org/"); + } + + @Override + public Version getVersion() { + return new Version(1, 0, 0); + } + + @Override + public String getDescription() { + return "Pluggable message processing framework"; + } + + @Override + public Version getRequiredVersion() { + return new Version(2, 0, 0); + } + + @Override + public Set getRequiredCapabilities() { + return Collections.emptySet(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java new file mode 100644 index 000000000000..647498143444 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -0,0 +1,44 @@ +package org.graylog.plugins.messageprocessor; + +import org.graylog2.plugin.PluginConfigBean; +import org.graylog2.plugin.PluginModule; + +import java.util.Collections; +import java.util.Set; + +/** + * Extend the PluginModule abstract class here to add you plugin to the system. + */ +public class ProcessorPluginModule extends PluginModule { + /** + * Returns all configuration beans required by this plugin. + * + * Implementing this method is optional. The default method returns an empty {@link Set}. + */ + @Override + public Set getConfigBeans() { + return Collections.emptySet(); + } + + @Override + protected void configure() { + /* + * Register your plugin types here. + * + * Examples: + * + * addMessageInput(Class); + * addMessageFilter(Class); + * addMessageOutput(Class); + * addPeriodical(Class); + * addAlarmCallback(Class); + * addInitializer(Class); + * addRestResource(Class); + * + * + * Add all configuration beans returned by getConfigBeans(): + * + * addConfigBeans(); + */ + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java new file mode 100644 index 000000000000..2445edc71038 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast; + +public class Rule { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java new file mode 100644 index 000000000000..d20178555dca --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java @@ -0,0 +1,7 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public class ComparisonExpression extends BinaryExpression { + public ComparisonExpression(Expression left, Expression right) { + super(left, right); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java new file mode 100644 index 000000000000..375d9fc4c694 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public class ConstantExpression implements Expression { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java new file mode 100644 index 000000000000..e1525ff6962d --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java @@ -0,0 +1,32 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public class EqualityExpression extends BinaryExpression implements LogicalExpression { + private final boolean equal; + + public EqualityExpression(Expression left, Expression right, boolean equal) { + super(left, right); + this.equal = equal; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return evaluateBool(context, message); + } + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + final boolean equals = left.evaluate(context, message).equals(right.evaluate(context, message)); + if (equal) { + return equals; + } + return !equals; + } + + @Override + public String toString() { + return left.toString() + (equal ? " == " : " != ") + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java new file mode 100644 index 000000000000..46c72989400f --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java @@ -0,0 +1,8 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public interface Expression { + T evaluate(EvaluationContext context, Message message); +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java new file mode 100644 index 000000000000..d48fbcef6ef0 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -0,0 +1,7 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public class FunctionExpression extends UnaryExpression { + public FunctionExpression(Expression right) { + super(right); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java new file mode 100644 index 000000000000..e38d605c72a1 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public interface LogicalExpression { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java new file mode 100644 index 000000000000..5ac6fd0cff0c --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public class MessageRefExpression { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java new file mode 100644 index 000000000000..aa166fd7d74b --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public interface NumericExpression { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java new file mode 100644 index 000000000000..5eabb0732a36 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java @@ -0,0 +1,3 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + + diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java new file mode 100644 index 000000000000..41880f8ea5fb --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +public class VarRefExpression { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java new file mode 100644 index 000000000000..c884e44e8206 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.functions; + +public interface Function { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java new file mode 100644 index 000000000000..748a2b7532bc --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.functions; + +public class FunctionDescriptor { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java new file mode 100644 index 000000000000..8cbd99e31768 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.functions; + +public class ParameterDescriptor { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java new file mode 100644 index 000000000000..ecedfb793bd9 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.statements; + +public class FunctionStatement { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java new file mode 100644 index 000000000000..e9f22326082a --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.ast.statements; + +public class Statement { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java new file mode 100644 index 000000000000..a1334247dfd6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java @@ -0,0 +1,16 @@ +package org.graylog.plugins.messageprocessor.ast.statements; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.statements.Statement; +import org.graylog2.plugin.Message; + +public class VarAssignStatement implements Statement { + public VarAssignStatement(String name, Expression expr) { + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return null; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java new file mode 100644 index 000000000000..af5a8d4aed38 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java @@ -0,0 +1,57 @@ +package org.graylog.plugins.messageprocessor.db; + +import com.google.common.collect.Sets; +import com.mongodb.MongoException; +import org.graylog.plugins.messageprocessor.rest.ProcessingRule; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoConnection; +import org.graylog2.database.NotFoundException; +import org.mongojack.DBCursor; +import org.mongojack.JacksonDBCollection; +import org.mongojack.WriteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.Collections; + +public class RuleSourceService { + private static final Logger log = LoggerFactory.getLogger(RuleSourceService.class); + + public static final String COLLECTION = "message_processor_rules"; + + private final JacksonDBCollection dbCollection; + + @Inject + public RuleSourceService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + dbCollection = JacksonDBCollection.wrap( + mongoConnection.getDatabase().getCollection(COLLECTION), + ProcessingRule.class, + String.class, + mapper.get()); + } + + public ProcessingRule save(ProcessingRule rule) { + final WriteResult save = dbCollection.save(rule); + return rule.withId(save.getSavedId()); + } + + public ProcessingRule load(String id) throws NotFoundException { + final ProcessingRule rule = dbCollection.findOneById(id); + if (rule == null) { + throw new NotFoundException("No rule with id " + id); + } + return rule; + } + + public Collection loadAll() { + try { + final DBCursor processingRules = dbCollection.find(); + return Sets.newHashSet(processingRules.iterator()); + } catch (MongoException e) { + log.error("Unable to load processing rules", e); + return Collections.emptySet(); + } + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java new file mode 100644 index 000000000000..be8174d0d95a --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java @@ -0,0 +1,12 @@ +package org.graylog.plugins.messageprocessor.parser; + +import org.graylog.plugins.messageprocessor.parser.ParseError; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; + +public class InvalidReference implements ParseError { + private final RuleLangParser.IdentifierContext ctx; + + public InvalidReference(RuleLangParser.IdentifierContext ctx) { + this.ctx = ctx; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java new file mode 100644 index 000000000000..c791bceeac52 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.parser; + +public interface ParseError { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java new file mode 100644 index 000000000000..ee0d68822151 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java @@ -0,0 +1,15 @@ +package org.graylog.plugins.messageprocessor.parser; + +import java.util.List; + +public class ParseException extends Throwable { + private final List errors; + + public ParseException(List errors) { + this.errors = errors; + } + + public List getErrors() { + return errors; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java new file mode 100644 index 000000000000..7b8ed6d0889a --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -0,0 +1,9 @@ +package org.graylog.plugins.messageprocessor.parser; + +public class RuleParser { + + public void parseRule(String rule) { + + } + +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java new file mode 100644 index 000000000000..567adbfb8483 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -0,0 +1,52 @@ +package org.graylog.plugins.messageprocessor.processors; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.Rule; +import org.graylog.plugins.messageprocessor.ast.statements.Statement; +import org.graylog.plugins.messageprocessor.db.RuleSourceService; +import org.graylog.plugins.messageprocessor.parser.RuleParser; +import org.graylog.plugins.messageprocessor.rest.RuleSource; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Messages; +import org.graylog2.plugin.messageprocessors.MessageProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +public class NaiveRuleProcessor implements MessageProcessor { + private static final Logger log = LoggerFactory.getLogger(NaiveRuleProcessor.class); + + private final RuleSourceService ruleSourceService; + private final RuleParser ruleParser; + + @Inject + public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser ruleParser) { + this.ruleSourceService = ruleSourceService; + this.ruleParser = ruleParser; + } + + @Override + public Messages process(Messages messages) { + for (RuleSource ruleSource : ruleSourceService.loadAll()) { + Rule rule = ruleParser.parseRule(ruleSource.source()); + log.info("Evaluation rule {}", rule.name()); + + for (Message message : messages) { + final EvaluationContext context = new EvaluationContext(); + if (rule.when().evaluateBool(context, message)) { + log.info("[✓] Message {} matches condition", message.getId()); + + for (Statement statement : rule.then()) { + statement.evaluate(context, message); + } + + + } else { + log.info("[✕] Message {} does not match condition", message.getId()); + } + } + } + return messages; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java new file mode 100644 index 000000000000..242170b5cb16 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.rest; + +public class MessageProcessorRuleResource { +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java new file mode 100644 index 000000000000..838e45488bfe --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java @@ -0,0 +1,4 @@ +package org.graylog.plugins.messageprocessor.rest; + +public class RuleSource { +} diff --git a/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin b/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin new file mode 100644 index 000000000000..0220a5c6a703 --- /dev/null +++ b/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin @@ -0,0 +1 @@ +org.graylog.plugins.messageprocessor.ProcessorPlugin \ No newline at end of file diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java new file mode 100644 index 000000000000..e03bf236609f --- /dev/null +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -0,0 +1,13 @@ +package org.graylog.plugins.messageprocessor.parser; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class RuleParserTest { + + @Test + public void testParseRule() throws Exception { + + } +} \ No newline at end of file diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000000..39bb58711ad1 --- /dev/null +++ b/src/test/resources/log4j2-test.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt new file mode 100644 index 000000000000..1ba46189b223 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt @@ -0,0 +1,5 @@ +rule "identifier" +when true +then + some_func(x); +end \ No newline at end of file From c25040c48d039a2ba7778dd0e28230ade04c0f20 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 30 Dec 2015 15:48:04 +0100 Subject: [PATCH 002/521] wip --- .gitignore | 98 +++++ pom.xml | 146 ++++++- .../messageprocessor/parser/RuleLang.g4 | 389 ++++++++++++++++++ .../messageprocessor/EvaluationContext.java | 6 + .../plugins/messageprocessor/FieldSet.java | 63 ++- .../messageprocessor/ProcessorPlugin.java | 7 +- .../ProcessorPluginMetaData.java | 6 +- .../ProcessorPluginModule.java | 32 +- .../plugins/messageprocessor/ast/Rule.java | 33 +- .../ast/expressions/AndExpression.java | 23 ++ .../ast/expressions/BinaryExpression.java | 14 + .../ast/expressions/BooleanExpression.java | 26 ++ .../ast/expressions/ComparisonExpression.java | 69 +++- .../ast/expressions/ConstantExpression.java | 19 +- .../ast/expressions/DoubleExpression.java | 40 ++ .../ast/expressions/EqualityExpression.java | 10 +- .../ast/expressions/Expression.java | 9 +- .../ast/expressions/FunctionExpression.java | 41 +- .../ast/expressions/LogicalExpression.java | 7 +- .../ast/expressions/LongExpression.java | 40 ++ .../ast/expressions/MessageRefExpression.java | 24 +- .../ast/expressions/NotExpression.java | 22 + .../ast/expressions/NumericExpression.java | 10 +- .../ast/expressions/OrExpression.java | 23 ++ .../ast/expressions/StringExpression.java | 21 + .../ast/expressions/UnaryExpression.java | 17 + .../ast/expressions/VarRefExpression.java | 30 +- .../ast/functions/Function.java | 9 + .../ast/functions/FunctionDescriptor.java | 28 +- .../ast/functions/ParameterDescriptor.java | 20 +- .../ast/statements/FunctionStatement.java | 21 +- .../ast/statements/Statement.java | 7 +- .../ast/statements/VarAssignStatement.java | 13 +- .../db/RuleSourceService.java | 20 +- .../parser/InvalidReference.java | 22 +- .../parser/ParseException.java | 11 +- .../messageprocessor/parser/RuleParser.java | 290 ++++++++++++- .../processors/NaiveRuleProcessor.java | 9 +- .../rest/MessageProcessorRuleResource.java | 94 ++++- .../messageprocessor/rest/RuleSource.java | 56 ++- .../services/org.graylog2.plugin.Plugin | 2 +- .../parser/RuleParserTest.java | 59 ++- src/test/resources/log4j2-test.xml | 1 + .../messageprocessor/parser/basicRule.txt | 6 + .../parser/undeclaredIdentifier.txt | 2 +- 45 files changed, 1813 insertions(+), 82 deletions(-) diff --git a/.gitignore b/.gitignore index bfa6a22a5248..aef32ad0cd39 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,99 @@ # Created by .ignore support plugin (hsz.mobi) +### OSX template +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk +### Java template +*.class + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio + +*.iml + +## Directory-based project format: +.idea/ +# if you remove the above rule, at least ignore the following: + +# User-specific stuff: +# .idea/workspace.xml +# .idea/tasks.xml +# .idea/dictionaries + +# Sensitive or high-churn files: +# .idea/dataSources.ids +# .idea/dataSources.xml +# .idea/sqlDataSources.xml +# .idea/dynamic.xml +# .idea/uiDesigner.xml + +# Gradle: +# .idea/gradle.xml +# .idea/libraries + +# Mongo Explorer plugin: +# .idea/mongoSettings.xml + +## File-based project format: +*.ipr +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties + +## Maven + +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + diff --git a/pom.xml b/pom.xml index 7a0882f9f75c..1a54a078ce74 100644 --- a/pom.xml +++ b/pom.xml @@ -15,12 +15,73 @@ UTF-8 - 1.7 - 1.7 - 1.0.0 + 1.8 + 1.8 + 2.0.0-SNAPSHOT /usr/share/graylog-server/plugin + 4.5.1 + 1.7.13 + 2.5 + + + + org.antlr + antlr4-runtime + ${antlr.version} + + + com.google.auto.value + auto-value + 1.1 + + + junit + junit + 4.12 + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + org.slf4j + jul-to-slf4j + ${slf4j.version} + + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + + org.graylog2 @@ -28,9 +89,75 @@ ${graylog2.version} provided + + org.graylog2 + graylog2-shared + ${graylog2.version} + provided + + + org.graylog2 + graylog2-server + ${graylog2.version} + provided + + + com.google.auto.value + auto-value + + + + org.antlr + antlr4-runtime + + + junit + junit + test + + + + org.apache.logging.log4j + log4j-api + test + + + org.apache.logging.log4j + log4j-core + test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + org.slf4j + jul-to-slf4j + test + + + org.slf4j + jcl-over-slf4j + test + + + org.slf4j + log4j-over-slf4j + test + + + + + org.antlr + antlr4-maven-plugin + 4.5 + + + org.apache.maven.plugins @@ -108,6 +235,19 @@ + + + org.antlr + antlr4-maven-plugin + + + antlr + + antlr4 + + + + diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index e69de29bb2d1..44356fecb1e2 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -0,0 +1,389 @@ +/* + Parts of the grammar are derived from the Java.g4 grammar at https://github.com/antlr/grammars-v4/blob/master/java/Java.g4 + Those parts are under the following license: + + [The "BSD licence"] + Copyright (c) 2013 Terence Parr, Sam Harwell + All rights reserved. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +grammar RuleLang; + +ruleDeclaration + : Rule name=String + (During Stage stage=Integer)? + When condition=expression + Then actions=statement* + End + EOF + ; + +expression + : primary # PrimaryExpression + | fieldSet=expression '.' field=expression # Nested + | fieldSet=expression '[' expression ']' # Array + | functionCall # Func + | left=expression comparison=('<=' | '>=' | '>' | '<') right=expression # Comparison + | left=expression equality=('==' | '!=') right=expression # Equality + | left=expression and=And right=expression # And + | left=expression or=Or right=expression # Or + | not=Not expression # Not + ; + +primary + : '(' expression ')' # ParenExpr + | literal # LiteralPrimary + | Identifier # Identifier + | MessageRef # MessageRef + ; + +statement + : functionCall ';' # FuncStmt + | Let varName=Identifier '=' expression ';' # VarAssignStmt + | ';' # EmptyStmt + ; + +functionCall + : funcName=Identifier '(' arguments? ')' + ; + +arguments + : expression (',' expression)* + ; + +literal + : Integer # Integer + | Float # Float + | Char # Char + | String # String + | Boolean # Boolean + | 'null' # Null + ; + +// Lexer + +And : A N D | '&&'; +Or: O R | '||'; +Not: N O T | '!'; +Rule: R U L E; +During: D U R I N G; +Stage: S T A G E; +When: W H E N; +Then: T H E N; +End: E N D; +Let: L E T; +MessageRef: '$message'; + +Boolean + : 'true'|'false' + ; + +// Integer literals + +Integer + : DecimalIntegerLiteral + | HexIntegerLiteral + | OctalIntegerLiteral + | BinaryIntegerLiteral + ; + +fragment +DecimalIntegerLiteral + : DecimalNumeral IntegerTypeSuffix? + ; + +fragment +HexIntegerLiteral + : HexNumeral IntegerTypeSuffix? + ; + +fragment +OctalIntegerLiteral + : OctalNumeral IntegerTypeSuffix? + ; + +fragment +BinaryIntegerLiteral + : BinaryNumeral IntegerTypeSuffix? + ; + +fragment +IntegerTypeSuffix + : [lL] + ; + +fragment +DecimalNumeral + : '0' + | NonZeroDigit (Digits? | Underscores Digits) + ; + +fragment +Digits + : Digit (DigitOrUnderscore* Digit)? + ; + +fragment +Digit + : '0' + | NonZeroDigit + ; + +fragment +NonZeroDigit + : [1-9] + ; + +fragment +DigitOrUnderscore + : Digit + | '_' + ; + +fragment +Underscores + : '_'+ + ; + +fragment +HexNumeral + : '0' [xX] HexDigits + ; + +fragment +HexDigits + : HexDigit (HexDigitOrUnderscore* HexDigit)? + ; + +fragment +HexDigit + : [0-9a-fA-F] + ; + +fragment +HexDigitOrUnderscore + : HexDigit + | '_' + ; + +fragment +OctalNumeral + : '0' Underscores? OctalDigits + ; + +fragment +OctalDigits + : OctalDigit (OctalDigitOrUnderscore* OctalDigit)? + ; + +fragment +OctalDigit + : [0-7] + ; + +fragment +OctalDigitOrUnderscore + : OctalDigit + | '_' + ; + +fragment +BinaryNumeral + : '0' [bB] BinaryDigits + ; + +fragment +BinaryDigits + : BinaryDigit (BinaryDigitOrUnderscore* BinaryDigit)? + ; + +fragment +BinaryDigit + : [01] + ; + +fragment +BinaryDigitOrUnderscore + : BinaryDigit + | '_' + ; + +// Floats +Float + : DecimalFloatingPointLiteral + | HexadecimalFloatingPointLiteral + ; + +fragment +DecimalFloatingPointLiteral + : Digits '.' Digits? ExponentPart? FloatTypeSuffix? + | '.' Digits ExponentPart? FloatTypeSuffix? + | Digits ExponentPart FloatTypeSuffix? + | Digits FloatTypeSuffix + ; + +fragment +ExponentPart + : ExponentIndicator SignedInteger + ; + +fragment +ExponentIndicator + : [eE] + ; + +fragment +SignedInteger + : Sign? Digits + ; + +fragment +Sign + : [+-] + ; + +fragment +FloatTypeSuffix + : [fFdD] + ; + +fragment +HexadecimalFloatingPointLiteral + : HexSignificand BinaryExponent FloatTypeSuffix? + ; + +fragment +HexSignificand + : HexNumeral '.'? + | '0' [xX] HexDigits? '.' HexDigits + ; + +fragment +BinaryExponent + : BinaryExponentIndicator SignedInteger + ; + +fragment +BinaryExponentIndicator + : [pP] + ; + +// Char + +Char + : '\'' SingleCharacter '\'' + | '\'' EscapeSequence '\'' + ; + +fragment +SingleCharacter + : ~['\\] + ; + +// String literals +String + : '"' StringCharacters? '"' + ; +fragment +StringCharacters + : StringCharacter+ + ; +fragment +StringCharacter + : ~["\\] + | EscapeSequence + ; +// §3.10.6 Escape Sequences for Character and String Literals +fragment +EscapeSequence + : '\\' [btnfr"'\\] + | OctalEscape + | UnicodeEscape + ; + +fragment +OctalEscape + : '\\' OctalDigit + | '\\' OctalDigit OctalDigit + | '\\' ZeroToThree OctalDigit OctalDigit + ; + +fragment +UnicodeEscape + : '\\' 'u' HexDigit HexDigit HexDigit HexDigit + ; + +fragment +ZeroToThree + : [0-3] + ; + +Identifier + : [a-zA-Z_] [a-zA-Z_0-9]* + ; + + +// +// Whitespace and comments +// + +WS : [ \t\r\n\u000C]+ -> skip + ; + +COMMENT + : '/*' .*? '*/' -> skip + ; + +LINE_COMMENT + : '//' ~[\r\n]* -> skip + ; + + +// to support case insensitive keywords + +fragment A : [aA]; +fragment B : [bB]; +fragment C : [cC]; +fragment D : [dD]; +fragment E : [eE]; +fragment F : [fF]; +fragment G : [gG]; +fragment H : [hH]; +fragment I : [iI]; +fragment J : [jJ]; +fragment K : [kK]; +fragment L : [lL]; +fragment M : [mM]; +fragment N : [nN]; +fragment O : [oO]; +fragment P : [pP]; +fragment Q : [qQ]; +fragment R : [rR]; +fragment S : [sS]; +fragment T : [tT]; +fragment U : [uU]; +fragment V : [vV]; +fragment W : [wW]; +fragment X : [xX]; +fragment Y : [yY]; +fragment Z : [zZ]; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java index b695a31c6456..756ed51d3f16 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java @@ -1,4 +1,10 @@ package org.graylog.plugins.messageprocessor; public class EvaluationContext { + public void define(String name, Object result) { + } + + public Object get(String identifier) { + return null; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java b/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java index 0f10f814b9f2..539cfc4fc56d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java @@ -1,9 +1,70 @@ package org.graylog.plugins.messageprocessor; +import com.google.common.collect.Maps; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + public class FieldSet { - private static final FieldSet INSTANCE = new FieldSet(); + private static final FieldSet INSTANCE = new FieldSet(Collections.emptyMap()); + + private FieldSet(Map map) { + fields = map; + } public static FieldSet empty() { return INSTANCE; } + + private final Map fields; + + public FieldSet() { + this.fields = Maps.newHashMap(); + } + + public Object get(String key) { + return fields.get(key); + } + + public Object remove(String key) { + return fields.remove(key); + } + + public FieldSet put(String key, Object value) { + fields.put(key, value); + return this; + } + + public int size() { + return fields.size(); + } + + public boolean isEmpty() { + return fields.isEmpty(); + } + + @Override + public int hashCode() { + return fields.hashCode(); + } + + @Override + public boolean equals(Object o) { + return fields.equals(o); + } + + public Set> entrySet() { + return fields.entrySet(); + } + + public Set keySet() { + return fields.keySet(); + } + + public Collection values() { + return fields.values(); + } + } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java index 301f3adb7bef..6a7b840237b9 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java @@ -4,12 +4,9 @@ import org.graylog2.plugin.PluginMetaData; import org.graylog2.plugin.PluginModule; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; -/** - * Implement the Plugin interface here. - */ public class ProcessorPlugin implements Plugin { @Override public PluginMetaData metadata() { @@ -18,6 +15,6 @@ public PluginMetaData metadata() { @Override public Collection modules () { - return Arrays.asList(new ProcessorPluginModule()); + return Collections.singletonList(new ProcessorPluginModule()); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java index 3417de36244c..35934a5451d2 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java @@ -1,17 +1,17 @@ package org.graylog.plugins.messageprocessor; +import com.google.common.collect.Sets; import org.graylog2.plugin.PluginMetaData; import org.graylog2.plugin.ServerStatus; import org.graylog2.plugin.Version; import java.net.URI; -import java.util.Collections; import java.util.Set; public class ProcessorPluginMetaData implements PluginMetaData { @Override public String getUniqueId() { - return "org.graylog.plugins.messageprocessor.ProcessorPluginPlugin"; + return "org.graylog.plugins.messageprocessor.ProcessorPlugin"; } @Override @@ -46,6 +46,6 @@ public Version getRequiredVersion() { @Override public Set getRequiredCapabilities() { - return Collections.emptySet(); + return Sets.newHashSet(ServerStatus.Capability.SERVER); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index 647498143444..1c614350e427 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -1,20 +1,15 @@ package org.graylog.plugins.messageprocessor; +import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; +import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; import java.util.Collections; import java.util.Set; -/** - * Extend the PluginModule abstract class here to add you plugin to the system. - */ public class ProcessorPluginModule extends PluginModule { - /** - * Returns all configuration beans required by this plugin. - * - * Implementing this method is optional. The default method returns an empty {@link Set}. - */ + @Override public Set getConfigBeans() { return Collections.emptySet(); @@ -22,23 +17,8 @@ public Set getConfigBeans() { @Override protected void configure() { - /* - * Register your plugin types here. - * - * Examples: - * - * addMessageInput(Class); - * addMessageFilter(Class); - * addMessageOutput(Class); - * addPeriodical(Class); - * addAlarmCallback(Class); - * addInitializer(Class); - * addRestResource(Class); - * - * - * Add all configuration beans returned by getConfigBeans(): - * - * addConfigBeans(); - */ + addMessageProcessor(NaiveRuleProcessor.class); + + addRestResource(MessageProcessorRuleResource.class); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java index 2445edc71038..08c1eb44d579 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java @@ -1,4 +1,35 @@ package org.graylog.plugins.messageprocessor.ast; -public class Rule { +import com.google.auto.value.AutoValue; +import org.graylog.plugins.messageprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.messageprocessor.ast.statements.Statement; + +import java.util.Collection; + +@AutoValue +public abstract class Rule { + + public abstract String name(); + + public abstract int stage(); + + public abstract LogicalExpression when(); + + public abstract Collection then(); + + public static Builder builder() { + return new AutoValue_Rule.Builder().stage(0); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder name(String name); + public abstract Builder stage(int stage); + public abstract Builder when(LogicalExpression condition); + public abstract Builder then(Collection actions); + + public abstract Rule build(); + } + } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java index 5eabb0732a36..1a8eca892cc4 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java @@ -1,3 +1,26 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class AndExpression extends BinaryExpression implements LogicalExpression { + public AndExpression(LogicalExpression left, + LogicalExpression right) { + super(left, right); + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return evaluateBool(context, message); + } + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + return ((LogicalExpression)left).evaluateBool(context, message) && ((LogicalExpression)right).evaluateBool(context, message); + } + + @Override + public String toString() { + return left.toString() + " AND " + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java index 5eabb0732a36..fee036ef7e44 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java @@ -1,3 +1,17 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +public abstract class BinaryExpression extends UnaryExpression { + protected final Expression left; + + public BinaryExpression(Expression left, Expression right) { + super(right); + this.left = left; + } + + @Override + public boolean isConstant() { + return left.isConstant() && right.isConstant(); + } + +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java index 5eabb0732a36..363a17bfe9bb 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java @@ -1,3 +1,29 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class BooleanExpression extends ConstantExpression implements LogicalExpression { + private final boolean value; + + public BooleanExpression(boolean value) { + super(Boolean.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return value; + } + + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + return value; + } + + @Override + public String toString() { + return Boolean.toString(value); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java index d20178555dca..e87f7bd4590b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java @@ -1,7 +1,72 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public class ComparisonExpression extends BinaryExpression { - public ComparisonExpression(Expression left, Expression right) { +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public class ComparisonExpression extends BinaryExpression implements LogicalExpression { + private final String operator; + private final boolean numericArgs; + + public ComparisonExpression(Expression left, Expression right, String operator) { super(left, right); + this.numericArgs = left instanceof NumericExpression && right instanceof NumericExpression; + + this.operator = operator; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return evaluateBool(context, message); + } + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + if (!numericArgs) { + return false; + } + final NumericExpression numericLeft = (NumericExpression) this.left; + final NumericExpression numericRight = (NumericExpression) this.right; + if (numericLeft.isFloatingPoint() || numericRight.isFloatingPoint()) { + return compareDouble(operator, numericLeft.doubleValue(), numericRight.doubleValue()); + } else { + return compareLong(operator, numericLeft.longValue(), numericRight.longValue()); + } + } + + @SuppressWarnings("Duplicates") + private boolean compareLong(String operator, long left, long right) { + switch (operator) { + case ">": + return left > right; + case ">=": + return left >= right; + case "<": + return left < right; + case "<=": + return left <= right; + default: + return false; + } + } + + @SuppressWarnings("Duplicates") + private boolean compareDouble(String operator, double left, double right) { + switch (operator) { + case ">": + return left > right; + case ">=": + return left >= right; + case "<": + return left < right; + case "<=": + return left <= right; + default: + return false; + } + } + + @Override + public String toString() { + return left.toString() + " " + operator + " " + right.toString(); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java index 375d9fc4c694..8f39fb19e5bc 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java @@ -1,4 +1,21 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public class ConstantExpression implements Expression { +public abstract class ConstantExpression implements Expression { + + private final Class type; + + protected ConstantExpression(Class type) { + this.type = type; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public Class getType() { + return type; + } + } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java index 5eabb0732a36..c75ea007f142 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java @@ -1,3 +1,43 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class DoubleExpression extends ConstantExpression implements NumericExpression { + private final double value; + + public DoubleExpression(double value) { + super(Double.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return value; + } + + @Override + public String toString() { + return Double.toString(value); + } + + @Override + public boolean isIntegral() { + return false; + } + + @Override + public boolean isFloatingPoint() { + return true; + } + + @Override + public long longValue() { + return (long) value; + } + + @Override + public double doubleValue() { + return value; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java index e1525ff6962d..8e05997f02ef 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java @@ -4,11 +4,11 @@ import org.graylog2.plugin.Message; public class EqualityExpression extends BinaryExpression implements LogicalExpression { - private final boolean equal; + private final boolean checkEquality; - public EqualityExpression(Expression left, Expression right, boolean equal) { + public EqualityExpression(Expression left, Expression right, boolean checkEquality) { super(left, right); - this.equal = equal; + this.checkEquality = checkEquality; } @Override @@ -19,7 +19,7 @@ public Object evaluate(EvaluationContext context, Message message) { @Override public boolean evaluateBool(EvaluationContext context, Message message) { final boolean equals = left.evaluate(context, message).equals(right.evaluate(context, message)); - if (equal) { + if (checkEquality) { return equals; } return !equals; @@ -27,6 +27,6 @@ public boolean evaluateBool(EvaluationContext context, Message message) { @Override public String toString() { - return left.toString() + (equal ? " == " : " != ") + right.toString(); + return left.toString() + (checkEquality ? " == " : " != ") + right.toString(); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java index 46c72989400f..ba6b8428c7ad 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java @@ -3,6 +3,11 @@ import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog2.plugin.Message; -public interface Expression { - T evaluate(EvaluationContext context, Message message); +public interface Expression { + + boolean isConstant(); + + Object evaluate(EvaluationContext context, Message message); + + Class getType(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index d48fbcef6ef0..b6e8508f631d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -1,7 +1,42 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public class FunctionExpression extends UnaryExpression { - public FunctionExpression(Expression right) { - super(right); +import com.google.common.base.Joiner; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.FieldSet; +import org.graylog2.plugin.Message; + +import java.util.List; + +public class FunctionExpression implements Expression { + private final String name; + private final List args; + + public FunctionExpression(String name, List args) { + this.name = name; + this.args = args; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return FieldSet.empty(); + } + + @Override + public Class getType() { + return FieldSet.class; + } + + @Override + public String toString() { + String join = ""; + if (args != null) { + join = Joiner.on(", ").join(args); + } + return name + "(" + join + ")"; } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java index e38d605c72a1..75becef2c789 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java @@ -1,4 +1,9 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public interface LogicalExpression { +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public interface LogicalExpression extends Expression { + + boolean evaluateBool(EvaluationContext context, Message message); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java index 5eabb0732a36..a44d58bd4d6a 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java @@ -1,3 +1,43 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class LongExpression extends ConstantExpression implements NumericExpression { + private final long value; + + public LongExpression(long value) { + super(Long.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return value; + } + + @Override + public String toString() { + return Long.toString(value); + } + + @Override + public boolean isIntegral() { + return true; + } + + @Override + public boolean isFloatingPoint() { + return false; + } + + @Override + public long longValue() { + return value; + } + + @Override + public double doubleValue() { + return value; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java index 5ac6fd0cff0c..55c8e0ae6540 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java @@ -1,4 +1,26 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public class MessageRefExpression { +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public class MessageRefExpression implements Expression { + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return message; + } + + @Override + public Class getType() { + return Message.class; + } + + @Override + public String toString() { + return "$message"; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java index 5eabb0732a36..4fdfb579b864 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java @@ -1,3 +1,25 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class NotExpression extends UnaryExpression implements LogicalExpression { + public NotExpression(LogicalExpression right) { + super(right); + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return !evaluateBool(context, message); + } + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + return !((LogicalExpression)right).evaluateBool(context, message); + } + + @Override + public String toString() { + return "NOT " + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java index aa166fd7d74b..4fa650ba723f 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java @@ -1,4 +1,12 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public interface NumericExpression { +public interface NumericExpression extends Expression { + + boolean isIntegral(); + + boolean isFloatingPoint(); + + long longValue(); + + double doubleValue(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java index 5eabb0732a36..f221e7714ff2 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java @@ -1,3 +1,26 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class OrExpression extends BinaryExpression implements LogicalExpression { + public OrExpression(LogicalExpression left, + LogicalExpression right) { + super(left, right); + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return evaluateBool(context, message); + } + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + return ((LogicalExpression)left).evaluateBool(context, message) || ((LogicalExpression)right).evaluateBool(context, message); + } + + @Override + public String toString() { + return left.toString() + " OR " + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java index 5eabb0732a36..6022b2a8219a 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java @@ -1,3 +1,24 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +public class StringExpression extends ConstantExpression { + + private final String value; + + public StringExpression(String value) { + super(String.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return value; + } + + @Override + public String toString() { + return '"' + value + '"'; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java index 5eabb0732a36..62bc231c7ddd 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java @@ -1,3 +1,20 @@ package org.graylog.plugins.messageprocessor.ast.expressions; +public abstract class UnaryExpression implements Expression { + protected final Expression right; + + public UnaryExpression(Expression right) { + this.right = right; + } + + @Override + public boolean isConstant() { + return right.isConstant(); + } + + @Override + public Class getType() { + return right.getType(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java index 41880f8ea5fb..105434e03b38 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java @@ -1,4 +1,32 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -public class VarRefExpression { +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public class VarRefExpression implements Expression { + private final String identifier; + + public VarRefExpression(String identifier) { + this.identifier = identifier; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return context.get(identifier); + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return identifier; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java index c884e44e8206..7c8439a9cb09 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java @@ -1,4 +1,13 @@ package org.graylog.plugins.messageprocessor.ast.functions; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; + +import java.util.List; + public interface Function { + + Object evaluate(List args); + + FunctionDescriptor descriptor(); + } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java index 748a2b7532bc..2538c188b561 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java @@ -1,4 +1,30 @@ package org.graylog.plugins.messageprocessor.ast.functions; -public class FunctionDescriptor { +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; + +@AutoValue +public abstract class FunctionDescriptor { + + public abstract String name(); + + public abstract boolean pure(); + + public abstract Class returnType(); + + public abstract ImmutableList params(); + + public static Builder builder() { + return new AutoValue_FunctionDescriptor.Builder().pure(false); + } + + @AutoValue.Builder + public static abstract class Builder { + public abstract FunctionDescriptor build(); + + public abstract Builder name(String name); + public abstract Builder pure(boolean pure); + public abstract Builder returnType(Class type); + public abstract Builder params(ImmutableList params); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java index 8cbd99e31768..640aa8e4b6af 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java @@ -1,4 +1,22 @@ package org.graylog.plugins.messageprocessor.ast.functions; -public class ParameterDescriptor { +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class ParameterDescriptor { + + public abstract Class type(); + + public abstract String name(); + + public static Builder builder() { + return new AutoValue_ParameterDescriptor.Builder(); + } + + @AutoValue.Builder + public static abstract class Builder { + public abstract Builder type(Class type); + public abstract Builder name(String name); + public abstract ParameterDescriptor build(); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java index ecedfb793bd9..9f74a6f7551e 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java @@ -1,4 +1,23 @@ package org.graylog.plugins.messageprocessor.ast.statements; -public class FunctionStatement { +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog2.plugin.Message; + +public class FunctionStatement implements Statement { + private final Expression functionExpression; + + public FunctionStatement(Expression functionExpression) { + this.functionExpression = functionExpression; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return functionExpression.evaluate(context, message); + } + + @Override + public String toString() { + return functionExpression.toString(); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java index e9f22326082a..5e4429b4bc71 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java @@ -1,4 +1,9 @@ package org.graylog.plugins.messageprocessor.ast.statements; -public class Statement { +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public interface Statement { + + Object evaluate(EvaluationContext context, Message message); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java index a1334247dfd6..94d740c36dd5 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java @@ -2,15 +2,26 @@ import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog2.plugin.Message; public class VarAssignStatement implements Statement { + private final String name; + private final Expression expr; + public VarAssignStatement(String name, Expression expr) { + this.name = name; + this.expr = expr; } @Override public Object evaluate(EvaluationContext context, Message message) { + final Object result = expr.evaluate(context, message); + context.define(name, result); return null; } + + @Override + public String toString() { + return "let " + name + " = " + expr.toString(); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java index af5a8d4aed38..c5575890db57 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java @@ -2,7 +2,7 @@ import com.google.common.collect.Sets; import com.mongodb.MongoException; -import org.graylog.plugins.messageprocessor.rest.ProcessingRule; +import org.graylog.plugins.messageprocessor.rest.RuleSource; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; @@ -21,34 +21,34 @@ public class RuleSourceService { public static final String COLLECTION = "message_processor_rules"; - private final JacksonDBCollection dbCollection; + private final JacksonDBCollection dbCollection; @Inject public RuleSourceService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { dbCollection = JacksonDBCollection.wrap( mongoConnection.getDatabase().getCollection(COLLECTION), - ProcessingRule.class, + RuleSource.class, String.class, mapper.get()); } - public ProcessingRule save(ProcessingRule rule) { - final WriteResult save = dbCollection.save(rule); + public RuleSource save(RuleSource rule) { + final WriteResult save = dbCollection.save(rule); return rule.withId(save.getSavedId()); } - public ProcessingRule load(String id) throws NotFoundException { - final ProcessingRule rule = dbCollection.findOneById(id); + public RuleSource load(String id) throws NotFoundException { + final RuleSource rule = dbCollection.findOneById(id); if (rule == null) { throw new NotFoundException("No rule with id " + id); } return rule; } - public Collection loadAll() { + public Collection loadAll() { try { - final DBCursor processingRules = dbCollection.find(); - return Sets.newHashSet(processingRules.iterator()); + final DBCursor ruleSources = dbCollection.find(); + return Sets.newHashSet(ruleSources.iterator()); } catch (MongoException e) { log.error("Unable to load processing rules", e); return Collections.emptySet(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java index be8174d0d95a..57cf2bddf852 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java @@ -1,12 +1,30 @@ package org.graylog.plugins.messageprocessor.parser; -import org.graylog.plugins.messageprocessor.parser.ParseError; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; public class InvalidReference implements ParseError { + @JsonIgnore private final RuleLangParser.IdentifierContext ctx; public InvalidReference(RuleLangParser.IdentifierContext ctx) { this.ctx = ctx; } + + @Override + public String toString() { + return "Undeclared variable " + ctx.Identifier().getText() + " in" + + " line " + ctx.getStart().getLine() + + " pos " + ctx.getStart().getCharPositionInLine(); + } + + @JsonProperty + public int line() { + return ctx.getStart().getLine(); + } + + @JsonProperty + public int positionInLine() { + return ctx.getStart().getCharPositionInLine(); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java index ee0d68822151..8cbdd9f98236 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java @@ -2,7 +2,7 @@ import java.util.List; -public class ParseException extends Throwable { +public class ParseException extends RuntimeException { private final List errors; public ParseException(List errors) { @@ -12,4 +12,13 @@ public ParseException(List errors) { public List getErrors() { return errors; } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder("Errors:\n"); + for (ParseError parseError : getErrors()) { + sb.append(" ").append(parseError).append("\n"); + } + return sb.toString(); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 7b8ed6d0889a..302b8df5eca8 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -1,9 +1,297 @@ package org.graylog.plugins.messageprocessor.parser; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.tree.ParseTreeProperty; +import org.antlr.v4.runtime.tree.ParseTreeWalker; +import org.graylog.plugins.messageprocessor.ast.Rule; +import org.graylog.plugins.messageprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.messageprocessor.ast.statements.FunctionStatement; +import org.graylog.plugins.messageprocessor.ast.statements.Statement; +import org.graylog.plugins.messageprocessor.ast.statements.VarAssignStatement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + public class RuleParser { - public void parseRule(String rule) { + private static final Logger log = LoggerFactory.getLogger(RuleParser.class); + public static final ParseTreeWalker WALKER = ParseTreeWalker.DEFAULT; + + public Rule parseRule(String rule) throws ParseException { + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(rule)); + final RuleLangParser parser = new RuleLangParser(new CommonTokenStream(lexer)); + + final RuleLangParser.RuleDeclarationContext ruleDeclaration = parser.ruleDeclaration(); + + final ParseContext parseContext = new ParseContext(); + + // parsing stages: + // 1. build AST nodes, checks for invalid var refs + // 2. checker: static type check w/ coercion nodes + // 3. optimizer: TODO + + WALKER.walk(new AstBuilder(parseContext), ruleDeclaration); + WALKER.walk(new SanityCheck(parseContext), ruleDeclaration); + + if (parseContext.getErrors().isEmpty()) { + return parseContext.getRule(); + } + throw new ParseException(parseContext.getErrors()); + } + + private static class AstBuilder extends RuleLangBaseListener { + + private final ParseContext parseContext; + private final ParseTreeProperty> args; + private final ParseTreeProperty exprs; + + private final Set definedVars = Sets.newHashSet(); + + public AstBuilder(ParseContext parseContext) { + this.parseContext = parseContext; + args = parseContext.arguments(); + exprs = parseContext.expressions(); + } + + @Override + public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { + final Expression expr = exprs.get(ctx.functionCall()); + final FunctionStatement functionStatement = new FunctionStatement(expr); + parseContext.statements.add(functionStatement); + } + + @Override + public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { + final String name = ctx.varName.getText(); + final Expression expr = exprs.get(ctx.expression()); + definedVars.add(name); + parseContext.statements.add(new VarAssignStatement(name, expr)); + } + + @Override + public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { + final String name = ctx.funcName.getText(); + final List args = this.args.get(ctx.arguments()); + final FunctionExpression expr = new FunctionExpression(name, args); + log.info("FUNC: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitArguments(RuleLangParser.ArgumentsContext ctx) { + // collect all expressions into the args list to pass to function AST node + List argExprs = ctx.expression().stream().map(exprs::get).collect(Collectors.toList()); + args.put(ctx, argExprs); + } + + @Override + public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { + final Rule.Builder ruleBuilder = Rule.builder(); + ruleBuilder.name(ctx.name.getText()); + if (ctx.stage != null) { + ruleBuilder.stage(Integer.parseInt(ctx.stage.getText())); + } + ruleBuilder.when((LogicalExpression) exprs.get(ctx.condition)); + ruleBuilder.then(parseContext.statements); + final Rule rule = ruleBuilder.build(); + parseContext.setRule(rule); + log.info("Declaring rule {}", rule); + } + + @Override + public void exitNot(RuleLangParser.NotContext ctx) { + final LogicalExpression expression = (LogicalExpression) exprs.get(ctx.expression()); + final NotExpression expr = new NotExpression(expression); + log.info("NOT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitAnd(RuleLangParser.AndContext ctx) { + final LogicalExpression left = (LogicalExpression) exprs.get(ctx.left); + final LogicalExpression right = (LogicalExpression) exprs.get(ctx.right); + final AndExpression expr = new AndExpression(left, right); + log.info("AND: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitOr(RuleLangParser.OrContext ctx) { + final LogicalExpression left = (LogicalExpression) exprs.get(ctx.left); + final LogicalExpression right = (LogicalExpression) exprs.get(ctx.right); + final OrExpression expr = new OrExpression(left, right); + log.info("OR: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitEquality(RuleLangParser.EqualityContext ctx) { + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); + final boolean equals = ctx.equality.getText().equals("=="); + final EqualityExpression expr = new EqualityExpression(left, right, equals); + log.info("EQUAL: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitComparison(RuleLangParser.ComparisonContext ctx) { + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); + final String operator = ctx.comparison.getText(); + final ComparisonExpression expr = new ComparisonExpression(left, right, operator); + log.info("COMPARE: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitInteger(RuleLangParser.IntegerContext ctx) { + final LongExpression expr = new LongExpression(Long.parseLong(ctx.getText())); + log.info("INT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitFloat(RuleLangParser.FloatContext ctx) { + final DoubleExpression expr = new DoubleExpression(Double.parseDouble(ctx.getText())); + log.info("FLOAT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitChar(RuleLangParser.CharContext ctx) { + // TODO + super.exitChar(ctx); + } + + @Override + public void exitString(RuleLangParser.StringContext ctx) { + final StringExpression expr = new StringExpression(ctx.getText()); + log.info("STRING: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitBoolean(RuleLangParser.BooleanContext ctx) { + final BooleanExpression expr = new BooleanExpression(Boolean.valueOf(ctx.getText())); + log.info("BOOL: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { + // nothing to do, just propagate the ConstantExpression + exprs.put(ctx, exprs.get(ctx.literal())); + } + + @Override + public void exitParenExpr(RuleLangParser.ParenExprContext ctx) { + // nothing to do, just propagate + exprs.put(ctx, exprs.get(ctx.expression())); + } + + @Override + public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { + final MessageRefExpression expr = new MessageRefExpression(); + log.info("$MSG: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { + final String variableName = ctx.Identifier().getText(); + if (!definedVars.contains(variableName)) { + parseContext.addError(new InvalidReference(ctx)); + } + final VarRefExpression expr = new VarRefExpression(variableName); + log.info("VAR: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitPrimaryExpression(RuleLangParser.PrimaryExpressionContext ctx) { + // nothing to do, just propagate + exprs.put(ctx, exprs.get(ctx.primary())); + } + + @Override + public void exitFunc(RuleLangParser.FuncContext ctx) { + // nothing to do, just propagate + exprs.put(ctx, exprs.get(ctx.functionCall())); + } + + @Override + public void exitNull(RuleLangParser.NullContext ctx) { + // TODO + super.exitNull(ctx); + } + } + + private static class SanityCheck extends RuleLangBaseListener { + private final ParseContext parseContext; + + public SanityCheck(ParseContext parseContext) { + this.parseContext = parseContext; + } + + + } + + /** + * Contains meta data about the parse tree, such as AST nodes, link to the function registry etc. + * + * Being used by tree walkers or visitors to perform AST construction, type checking and so on. + */ + private static class ParseContext { + private final ParseTreeProperty exprs = new ParseTreeProperty<>(); + private final ParseTreeProperty> args = new ParseTreeProperty<>(); + private List errors = Lists.newArrayList(); + + public List statements = Lists.newArrayList(); + public Rule rule; + + public ParseTreeProperty expressions() { + return exprs; + } + + public ParseTreeProperty> arguments() { + return args; + } + + public Rule getRule() { + return rule; + } + + public void setRule(Rule rule) { + this.rule = rule; + } + + public List getErrors() { + return errors; + } + public void addError(ParseError error) { + errors.add(error); + } } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index 567adbfb8483..c76ad760b612 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -4,6 +4,7 @@ import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.db.RuleSourceService; +import org.graylog.plugins.messageprocessor.parser.ParseException; import org.graylog.plugins.messageprocessor.parser.RuleParser; import org.graylog.plugins.messageprocessor.rest.RuleSource; import org.graylog2.plugin.Message; @@ -29,7 +30,13 @@ public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser rulePa @Override public Messages process(Messages messages) { for (RuleSource ruleSource : ruleSourceService.loadAll()) { - Rule rule = ruleParser.parseRule(ruleSource.source()); + final Rule rule; + try { + rule = ruleParser.parseRule(ruleSource.source()); + } catch (ParseException parseException) { + log.error("Unable to parse rule: " + parseException.getMessage()); + continue; + } log.info("Evaluation rule {}", rule.name()); for (Message message : messages) { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java index 242170b5cb16..b3a6bb702417 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java @@ -1,4 +1,96 @@ package org.graylog.plugins.messageprocessor.rest; -public class MessageProcessorRuleResource { +import com.wordnik.swagger.annotations.Api; +import com.wordnik.swagger.annotations.ApiOperation; +import com.wordnik.swagger.annotations.ApiParam; +import org.graylog.plugins.messageprocessor.db.RuleSourceService; +import org.graylog.plugins.messageprocessor.parser.ParseException; +import org.graylog.plugins.messageprocessor.parser.RuleParser; +import org.graylog2.database.NotFoundException; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.shared.rest.resources.RestResource; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Api(value = "MessageProcessing", description = "Message processing pipeline") +@Path("/messageprocessors") +public class MessageProcessorRuleResource extends RestResource implements PluginRestResource { + + private static final Logger log = LoggerFactory.getLogger(MessageProcessorRuleResource.class); + + private final RuleSourceService ruleSourceService; + private final RuleParser ruleParser; + + @Inject + public MessageProcessorRuleResource(RuleSourceService ruleSourceService, RuleParser ruleParser) { + this.ruleSourceService = ruleSourceService; + this.ruleParser = ruleParser; + } + + + @ApiOperation(value = "Create a processing rule from source", notes = "") + @Consumes(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) + @Path("/rule") + @POST + public RuleSource createFromParser(@ApiParam(name = "processingRule", required = true) @NotNull String ruleSource) throws ParseException { + try { + ruleParser.parseRule(ruleSource); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final RuleSource newRuleSource = RuleSource.builder() + .source(ruleSource) + .createdAt(DateTime.now()) + .modifiedAt(DateTime.now()) + .build(); + final RuleSource save = ruleSourceService.save(newRuleSource); + log.info("Created new rule {}", save); + return save; + } + + @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/rule/{id}") + @GET + public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + return ruleSourceService.load(id); + } + + @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/rule/{id}") + @PUT + public Response update(@ApiParam(name = "id") @PathParam("id") String id) { + // TODO + return Response.ok().build(); + } + + @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/rule/{id}") + @DELETE + public void delete(@ApiParam(name = "id") @PathParam("id") String id) { + // TODO + } + + } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java index 838e45488bfe..df09a15f6d9e 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java @@ -1,4 +1,58 @@ package org.graylog.plugins.messageprocessor.rest; -public class RuleSource { +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.joda.time.DateTime; + +import javax.annotation.Nullable; + +@AutoValue +public abstract class RuleSource { + + @JsonProperty + @Nullable + public abstract String id(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + public abstract DateTime createdAt(); + + @JsonProperty + public abstract DateTime modifiedAt(); + + public static Builder builder() { + return new AutoValue_RuleSource.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static RuleSource create(@JsonProperty("id") @Nullable String id, @JsonProperty("source") String source, @JsonProperty("created_at") DateTime createdAt, @JsonProperty("modified_at") DateTime modifiedAt) { + return builder() + .id(id) + .source(source) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + public RuleSource withId(String savedId) { + return toBuilder().id(savedId).build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract RuleSource build(); + + public abstract Builder id(String id); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + } } diff --git a/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin b/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin index 0220a5c6a703..3eee36d919bc 100644 --- a/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin +++ b/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin @@ -1 +1 @@ -org.graylog.plugins.messageprocessor.ProcessorPlugin \ No newline at end of file +org.graylog.plugins.messageprocessor.ProcessorPlugin diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index e03bf236609f..94c2d7931b12 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -1,13 +1,68 @@ package org.graylog.plugins.messageprocessor.parser; +import com.google.common.base.Charsets; +import com.google.common.collect.Iterables; +import org.graylog.plugins.messageprocessor.ast.Rule; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.rules.TestName; -import static org.junit.Assert.*; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class RuleParserTest { + @org.junit.Rule + public TestName name = new TestName(); + + private RuleParser parser; + + @Before + public void setup() { + parser = new RuleParser(); + } + + @After + public void tearDown() { + parser = null; + } + + @Test + public void basicRule() throws Exception { + final Rule rule = parser.parseRule(ruleForTest()); + Assert.assertNotNull("rule should be successfully parsed", rule); + } + @Test - public void testParseRule() throws Exception { + public void undeclaredIdentifier() throws Exception { + try { + parser.parseRule(ruleForTest()); + fail("should throw error: undeclared variable x"); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertTrue("Should find error InvalidReference", Iterables.getOnlyElement(e.getErrors()) instanceof InvalidReference); + } + } + private String ruleForTest() { + try { + final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); + final Path path = Paths.get(resource.toURI()); + final byte[] bytes = Files.readAllBytes(path); + return new String(bytes, Charsets.UTF_8); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } } + } \ No newline at end of file diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index 39bb58711ad1..4e08ed87f895 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -7,6 +7,7 @@ + diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt index e69de29bb2d1..f17a8f8e6bc6 100644 --- a/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt @@ -0,0 +1,6 @@ +rule "something" +during stage 0 +when double_valued_func() > 1.0d AND false == true +then +some_func(); +end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt index 1ba46189b223..ad032bc2e23f 100644 --- a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt @@ -1,4 +1,4 @@ -rule "identifier" +rule "undeclared variable" when true then some_func(x); From 1bb202ca1523f1af987db780071e3ab142224f3e Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 30 Dec 2015 16:23:52 +0100 Subject: [PATCH 003/521] add function registry - check for function being present during parsing - add test for present/missing function --- .../ProcessorPluginModule.java | 21 +++++++- .../parser/FunctionRegistry.java | 20 ++++++++ .../parser/InvalidReference.java | 30 ----------- .../messageprocessor/parser/ParseError.java | 29 ++++++++++- .../messageprocessor/parser/RuleParser.java | 18 +++++-- .../parser/UndeclaredFunction.java | 15 ++++++ .../parser/UndeclaredVariable.java | 19 +++++++ .../parser/RuleParserTest.java | 50 ++++++++++++++++++- .../parser/declaredFunction.txt | 4 ++ .../parser/undeclaredFunction.txt | 4 ++ 10 files changed, 172 insertions(+), 38 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/declaredFunction.txt create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredFunction.txt diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index 1c614350e427..ae938b757d26 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -1,5 +1,8 @@ package org.graylog.plugins.messageprocessor; +import com.google.inject.Binder; +import com.google.inject.multibindings.MapBinder; +import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; import org.graylog2.plugin.PluginConfigBean; @@ -18,7 +21,23 @@ public Set getConfigBeans() { @Override protected void configure() { addMessageProcessor(NaiveRuleProcessor.class); - addRestResource(MessageProcessorRuleResource.class); + + // built-in functions + addMessageProcessorFunction("", Function.class); + + } + + protected void addMessageProcessorFunction(String name, Class functionClass) { + addMessageProcessorFunction(binder(), name, functionClass); + } + + public static MapBinder processorFunctionBinder(Binder binder) { + return MapBinder.newMapBinder(binder, String.class, Function.class); + } + + public static void addMessageProcessorFunction(Binder binder, String name, Class functionClass) { + processorFunctionBinder(binder).addBinding(name).to(functionClass); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java new file mode 100644 index 000000000000..53e8b30b06b4 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java @@ -0,0 +1,20 @@ +package org.graylog.plugins.messageprocessor.parser; + +import org.graylog.plugins.messageprocessor.ast.functions.Function; + +import javax.inject.Inject; +import java.util.Map; + +public class FunctionRegistry { + private final Map functions; + + @Inject + public FunctionRegistry(Map functions) { + this.functions = functions; + } + + + public Function resolve(String name) { + return functions.get(name); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java deleted file mode 100644 index 57cf2bddf852..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/InvalidReference.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class InvalidReference implements ParseError { - @JsonIgnore - private final RuleLangParser.IdentifierContext ctx; - - public InvalidReference(RuleLangParser.IdentifierContext ctx) { - this.ctx = ctx; - } - - @Override - public String toString() { - return "Undeclared variable " + ctx.Identifier().getText() + " in" + - " line " + ctx.getStart().getLine() + - " pos " + ctx.getStart().getCharPositionInLine(); - } - - @JsonProperty - public int line() { - return ctx.getStart().getLine(); - } - - @JsonProperty - public int positionInLine() { - return ctx.getStart().getCharPositionInLine(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java index c791bceeac52..d1bc15a21bf6 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java @@ -1,4 +1,31 @@ package org.graylog.plugins.messageprocessor.parser; -public interface ParseError { +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.antlr.v4.runtime.ParserRuleContext; + +public abstract class ParseError { + + @JsonIgnore + private final ParserRuleContext ctx; + + protected ParseError(ParserRuleContext ctx) { + this.ctx = ctx; + } + + @JsonProperty + public int line() { + return ctx.getStart().getLine(); + } + + @JsonProperty + public int positionInLine() { + return ctx.getStart().getCharPositionInLine(); + } + + protected String positionString() { + return " in" + + " line " + line() + + " pos " + positionInLine(); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 302b8df5eca8..3384579dfece 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -33,6 +33,12 @@ public class RuleParser { + private final FunctionRegistry functionRegistry; + + public RuleParser(FunctionRegistry functionRegistry) { + this.functionRegistry = functionRegistry; + } + private static final Logger log = LoggerFactory.getLogger(RuleParser.class); public static final ParseTreeWalker WALKER = ParseTreeWalker.DEFAULT; @@ -45,7 +51,7 @@ public Rule parseRule(String rule) throws ParseException { final ParseContext parseContext = new ParseContext(); // parsing stages: - // 1. build AST nodes, checks for invalid var refs + // 1. build AST nodes, checks for invalid var, function refs // 2. checker: static type check w/ coercion nodes // 3. optimizer: TODO @@ -58,7 +64,7 @@ public Rule parseRule(String rule) throws ParseException { throw new ParseException(parseContext.getErrors()); } - private static class AstBuilder extends RuleLangBaseListener { + private class AstBuilder extends RuleLangBaseListener { private final ParseContext parseContext; private final ParseTreeProperty> args; @@ -92,6 +98,10 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); final List args = this.args.get(ctx.arguments()); final FunctionExpression expr = new FunctionExpression(name, args); + + if (functionRegistry.resolve(name) == null) { + parseContext.addError(new UndeclaredFunction(ctx)); + } log.info("FUNC: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -220,7 +230,7 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { final String variableName = ctx.Identifier().getText(); if (!definedVars.contains(variableName)) { - parseContext.addError(new InvalidReference(ctx)); + parseContext.addError(new UndeclaredVariable(ctx)); } final VarRefExpression expr = new VarRefExpression(variableName); log.info("VAR: ctx {} => {}", ctx, expr); @@ -246,7 +256,7 @@ public void exitNull(RuleLangParser.NullContext ctx) { } } - private static class SanityCheck extends RuleLangBaseListener { + private class SanityCheck extends RuleLangBaseListener { private final ParseContext parseContext; public SanityCheck(ParseContext parseContext) { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java new file mode 100644 index 000000000000..3d7ab28ec2d6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java @@ -0,0 +1,15 @@ +package org.graylog.plugins.messageprocessor.parser; + +public class UndeclaredFunction extends ParseError { + private final RuleLangParser.FunctionCallContext ctx; + + public UndeclaredFunction(RuleLangParser.FunctionCallContext ctx) { + super(ctx); + this.ctx = ctx; + } + + @Override + public String toString() { + return "Unknown function " + ctx.funcName.getText() + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java new file mode 100644 index 000000000000..2e0374ca29c0 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java @@ -0,0 +1,19 @@ +package org.graylog.plugins.messageprocessor.parser; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class UndeclaredVariable extends ParseError { + @JsonIgnore + private final RuleLangParser.IdentifierContext ctx; + + public UndeclaredVariable(RuleLangParser.IdentifierContext ctx) { + super(ctx); + this.ctx = ctx; + } + + @Override + public String toString() { + return "Undeclared variable " + ctx.Identifier().getText() + positionString(); + } + +} diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 94c2d7931b12..dcf29cafa0a2 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -2,10 +2,15 @@ import com.google.common.base.Charsets; import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; import org.graylog.plugins.messageprocessor.ast.Rule; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.rules.TestName; @@ -15,6 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -26,10 +33,30 @@ public class RuleParserTest { public TestName name = new TestName(); private RuleParser parser; + private static FunctionRegistry functionRegistry; + + @BeforeClass + public static void registerFunctions() { + final Map functions = Maps.newHashMap(); + functions.put("nein", new Function() { + @Override + public Object evaluate(List args) { + return false; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("nein") + .build(); + } + }); + functionRegistry = new FunctionRegistry(functions); + } @Before public void setup() { - parser = new RuleParser(); + parser = new RuleParser(functionRegistry); } @After @@ -50,10 +77,29 @@ public void undeclaredIdentifier() throws Exception { fail("should throw error: undeclared variable x"); } catch (ParseException e) { assertEquals(1, e.getErrors().size()); - assertTrue("Should find error InvalidReference", Iterables.getOnlyElement(e.getErrors()) instanceof InvalidReference); + assertTrue("Should find error UndeclaredVariable", Iterables.getOnlyElement(e.getErrors()) instanceof UndeclaredVariable); } } + @Test + public void declaredFunction() throws Exception { + try { + parser.parseRule(ruleForTest()); + } catch (ParseException e) { + fail("Should not fail to resolve function 'false'"); + } + } + + @Test + public void undeclaredFunction() throws Exception { + try { + parser.parseRule(ruleForTest()); + fail("should throw error: undeclared function 'unknown'"); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertTrue("Should find error UndeclaredFunction", Iterables.getOnlyElement(e.getErrors()) instanceof UndeclaredFunction); + } + } private String ruleForTest() { try { final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/declaredFunction.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/declaredFunction.txt new file mode 100644 index 000000000000..2804b06d62b3 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/declaredFunction.txt @@ -0,0 +1,4 @@ +rule "using declared function 'nein'" +when true == nein() +then +end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredFunction.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredFunction.txt new file mode 100644 index 000000000000..419077cee77e --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredFunction.txt @@ -0,0 +1,4 @@ +rule "undeclared function" +when false == unknown() +then +end \ No newline at end of file From 91118466cbc8bad6ed50e9a6fc6708af888654e8 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 30 Dec 2015 17:22:02 +0100 Subject: [PATCH 004/521] function arguments are now a map of named expressions - single argument syntactic sugar not implemented yet - function invocations work, but have no type checking yet - sample InputFunction implemented --- .../messageprocessor/parser/RuleLang.g4 | 3 +- .../messageprocessor/EvaluationContext.java | 20 +++++++++ .../ProcessorPluginModule.java | 4 +- .../ast/expressions/FunctionExpression.java | 10 ++--- .../ast/functions/Function.java | 6 ++- .../ast/functions/ParameterDescriptor.java | 8 ++++ .../ast/functions/builtin/InputFunction.java | 44 +++++++++++++++++++ .../db/RuleSourceService.java | 2 +- .../messageprocessor/parser/RuleParser.java | 40 ++++++++++++----- .../processors/NaiveRuleProcessor.java | 7 ++- .../messageprocessor/rest/RuleSource.java | 9 +++- .../parser/RuleParserTest.java | 5 ++- 12 files changed, 131 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index 44356fecb1e2..e937c3bdf646 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -68,7 +68,8 @@ functionCall ; arguments - : expression (',' expression)* + : Identifier ':' expression (',' Identifier ':' expression)* # NamedArgs + | expression # SingleDefaultArg ; literal diff --git a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java index 756ed51d3f16..0fc48149e541 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java @@ -1,10 +1,30 @@ package org.graylog.plugins.messageprocessor; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.parser.FunctionRegistry; +import org.graylog2.plugin.Message; + +import java.util.Map; + public class EvaluationContext { + + private final FunctionRegistry functionRegistry; + + public EvaluationContext(FunctionRegistry functionRegistry) { + this.functionRegistry = functionRegistry; + } + public void define(String name, Object result) { } public Object get(String identifier) { return null; } + + public Object invokeFunction(EvaluationContext context, Message message, String functionName, Map args) { + final Function function = functionRegistry.resolve(functionName); + // TODO missing function + return function.evaluate(args, context, message); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index ae938b757d26..286c06f41ba8 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -3,6 +3,7 @@ import com.google.inject.Binder; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.InputFunction; import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; import org.graylog2.plugin.PluginConfigBean; @@ -24,8 +25,7 @@ protected void configure() { addRestResource(MessageProcessorRuleResource.class); // built-in functions - addMessageProcessorFunction("", Function.class); - + addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); } protected void addMessageProcessorFunction(String name, Class functionClass) { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index b6e8508f631d..bb747a71b734 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -5,13 +5,13 @@ import org.graylog.plugins.messageprocessor.FieldSet; import org.graylog2.plugin.Message; -import java.util.List; +import java.util.Map; public class FunctionExpression implements Expression { private final String name; - private final List args; + private final Map args; - public FunctionExpression(String name, List args) { + public FunctionExpression(String name, Map args) { this.name = name; this.args = args; } @@ -23,7 +23,7 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context, Message message) { - return FieldSet.empty(); + return context.invokeFunction(context, message, name, args); } @Override @@ -35,7 +35,7 @@ public Class getType() { public String toString() { String join = ""; if (args != null) { - join = Joiner.on(", ").join(args); + join = Joiner.on(", ").withKeyValueSeparator(": ").join(args); // TODO order arg names } return name + "(" + join + ")"; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java index 7c8439a9cb09..7b267f240d08 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java @@ -1,12 +1,14 @@ package org.graylog.plugins.messageprocessor.ast.functions; +import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog2.plugin.Message; -import java.util.List; +import java.util.Map; public interface Function { - Object evaluate(List args); + Object evaluate(Map args, EvaluationContext context, Message message); FunctionDescriptor descriptor(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java index 640aa8e4b6af..5bccd06ffeeb 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java @@ -13,10 +13,18 @@ public static Builder builder() { return new AutoValue_ParameterDescriptor.Builder(); } + public static ParameterDescriptor string(String name) { + return builder().string(name).build(); + } + @AutoValue.Builder public static abstract class Builder { public abstract Builder type(Class type); public abstract Builder name(String name); public abstract ParameterDescriptor build(); + + public Builder string(String name) { + return type(String.class).name(name); + } } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java new file mode 100644 index 000000000000..a7de3d58b1c8 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java @@ -0,0 +1,44 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.IOState; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.inputs.MessageInput; +import org.graylog2.shared.inputs.InputRegistry; + +import javax.inject.Inject; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; + +public class InputFunction implements Function { + + public static final String NAME = "input"; + + private final InputRegistry inputRegistry; + + @Inject + public InputFunction(InputRegistry inputRegistry) { + this.inputRegistry = inputRegistry; + } + + @Override + public Object evaluate(Map args, EvaluationContext context, Message message) { + final Object id = args.get("id").evaluate(context, message); + final IOState inputState = inputRegistry.getInputState(id.toString()); + return inputState.getStoppable(); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(MessageInput.class) + .params(of(ParameterDescriptor.string("id"))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java index c5575890db57..183c984ee916 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java @@ -34,7 +34,7 @@ public RuleSourceService(MongoConnection mongoConnection, MongoJackObjectMapperP public RuleSource save(RuleSource rule) { final WriteResult save = dbCollection.save(rule); - return rule.withId(save.getSavedId()); + return save.getSavedObject(); } public RuleSource load(String id) throws NotFoundException { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 3384579dfece..f53bcda0e9dc 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -1,6 +1,7 @@ package org.graylog.plugins.messageprocessor.parser; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; @@ -27,14 +28,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; public class RuleParser { private final FunctionRegistry functionRegistry; + @Inject public RuleParser(FunctionRegistry functionRegistry) { this.functionRegistry = functionRegistry; } @@ -67,7 +71,7 @@ public Rule parseRule(String rule) throws ParseException { private class AstBuilder extends RuleLangBaseListener { private final ParseContext parseContext; - private final ParseTreeProperty> args; + private final ParseTreeProperty> args; private final ParseTreeProperty exprs; private final Set definedVars = Sets.newHashSet(); @@ -96,7 +100,7 @@ public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { @Override public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); - final List args = this.args.get(ctx.arguments()); + final Map args = this.args.get(ctx.arguments()); final FunctionExpression expr = new FunctionExpression(name, args); if (functionRegistry.resolve(name) == null) { @@ -107,10 +111,25 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { } @Override - public void exitArguments(RuleLangParser.ArgumentsContext ctx) { - // collect all expressions into the args list to pass to function AST node - List argExprs = ctx.expression().stream().map(exprs::get).collect(Collectors.toList()); - args.put(ctx, argExprs); + public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { + final Map argMap = Maps.newHashMap(); + final int argCount = ctx.Identifier().size(); + for (int i = 0; i < argCount; i++) { + final String argName = ctx.Identifier(i).getText(); + final Expression argValue = exprs.get(ctx.expression(i)); + argMap.put(argName, argValue); + } + args.put(ctx, argMap); + } + + @Override + public void exitSingleDefaultArg(RuleLangParser.SingleDefaultArgContext ctx) { + final Expression expr = exprs.get(ctx.expression()); + final HashMap singleArg = Maps.newHashMap(); + // null key means to use the single declared argument for this function, it's syntactic sugar + // this gets validated and expanded in a later parsing stage + singleArg.put(null, expr); + args.put(ctx, singleArg); } @Override @@ -195,7 +214,8 @@ public void exitChar(RuleLangParser.CharContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { - final StringExpression expr = new StringExpression(ctx.getText()); + final String text = ctx.getText(); + final StringExpression expr = new StringExpression(text.substring(1, text.length()-1)); log.info("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -273,7 +293,7 @@ public SanityCheck(ParseContext parseContext) { */ private static class ParseContext { private final ParseTreeProperty exprs = new ParseTreeProperty<>(); - private final ParseTreeProperty> args = new ParseTreeProperty<>(); + private final ParseTreeProperty> args = new ParseTreeProperty<>(); private List errors = Lists.newArrayList(); public List statements = Lists.newArrayList(); @@ -283,7 +303,7 @@ public ParseTreeProperty expressions() { return exprs; } - public ParseTreeProperty> arguments() { + public ParseTreeProperty> arguments() { return args; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index c76ad760b612..c4bb28743972 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -4,6 +4,7 @@ import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.db.RuleSourceService; +import org.graylog.plugins.messageprocessor.parser.FunctionRegistry; import org.graylog.plugins.messageprocessor.parser.ParseException; import org.graylog.plugins.messageprocessor.parser.RuleParser; import org.graylog.plugins.messageprocessor.rest.RuleSource; @@ -20,11 +21,13 @@ public class NaiveRuleProcessor implements MessageProcessor { private final RuleSourceService ruleSourceService; private final RuleParser ruleParser; + private final FunctionRegistry functionRegistry; @Inject - public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser ruleParser) { + public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser ruleParser, FunctionRegistry functionRegistry) { this.ruleSourceService = ruleSourceService; this.ruleParser = ruleParser; + this.functionRegistry = functionRegistry; } @Override @@ -40,7 +43,7 @@ public Messages process(Messages messages) { log.info("Evaluation rule {}", rule.name()); for (Message message : messages) { - final EvaluationContext context = new EvaluationContext(); + final EvaluationContext context = new EvaluationContext(functionRegistry); if (rule.when().evaluateBool(context, message)) { log.info("[✓] Message {} matches condition", message.getId()); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java index df09a15f6d9e..0d92e951cbe8 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java @@ -4,14 +4,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import org.joda.time.DateTime; +import org.mongojack.ObjectId; import javax.annotation.Nullable; @AutoValue public abstract class RuleSource { - @JsonProperty + @JsonProperty("_id") @Nullable + @ObjectId public abstract String id(); @JsonProperty @@ -30,7 +32,10 @@ public static Builder builder() { public abstract Builder toBuilder(); @JsonCreator - public static RuleSource create(@JsonProperty("id") @Nullable String id, @JsonProperty("source") String source, @JsonProperty("created_at") DateTime createdAt, @JsonProperty("modified_at") DateTime modifiedAt) { + public static RuleSource create(@JsonProperty("_id") @ObjectId @Nullable String id, + @JsonProperty("source") String source, + @JsonProperty("created_at") DateTime createdAt, + @JsonProperty("modified_at") DateTime modifiedAt) { return builder() .id(id) .source(source) diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index dcf29cafa0a2..241f83158bd5 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -3,10 +3,12 @@ import com.google.common.base.Charsets; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -20,7 +22,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -40,7 +41,7 @@ public static void registerFunctions() { final Map functions = Maps.newHashMap(); functions.put("nein", new Function() { @Override - public Object evaluate(List args) { + public Object evaluate(Map args, EvaluationContext context, Message message) { return false; } From f2d4e21e5490c35f8ea2082fac022752bc43c4e3 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 30 Dec 2015 18:17:31 +0100 Subject: [PATCH 005/521] add field ref expression and accessor ("id" -> "getId" bean style access), added drop_message builtin function --- .../ProcessorPluginModule.java | 2 + .../expressions/FieldAccessExpression.java | 50 +++++++++++++++++++ .../ast/expressions/FieldRefExpression.java | 32 ++++++++++++ .../builtin/DropMessageFunction.java | 31 ++++++++++++ .../messageprocessor/parser/RuleParser.java | 30 ++++++++++- .../processors/NaiveRuleProcessor.java | 21 ++++---- 6 files changed, 155 insertions(+), 11 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index 286c06f41ba8..440db0d0c77b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -3,6 +3,7 @@ import com.google.inject.Binder; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.DropMessageFunction; import org.graylog.plugins.messageprocessor.ast.functions.builtin.InputFunction; import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; @@ -26,6 +27,7 @@ protected void configure() { // built-in functions addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); + addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); } protected void addMessageProcessorFunction(String name, Class functionClass) { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java new file mode 100644 index 000000000000..6fe269665f3e --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java @@ -0,0 +1,50 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +public class FieldAccessExpression implements Expression { + private static final Logger log = LoggerFactory.getLogger(FieldAccessExpression.class); + + private final Expression object; + private final Expression field; + + public FieldAccessExpression(Expression object, Expression field) { + this.object = object; + this.field = field; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + final Object bean = this.object.evaluate(context, message); + final String fieldName = field.evaluate(context, message).toString(); + try { + final Method method = bean.getClass().getMethod("get"+ StringUtils.capitalize(fieldName)); + return method.invoke(bean); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + log.error("Oops"); + return null; + } + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return object.toString() + "." + field.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java new file mode 100644 index 000000000000..b60f00554c5f --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java @@ -0,0 +1,32 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public class FieldRefExpression implements Expression { + private final String variableName; + + public FieldRefExpression(String variableName) { + this.variableName = variableName; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return variableName; + } + + @Override + public Class getType() { + return String.class; + } + + @Override + public String toString() { + return variableName; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java new file mode 100644 index 000000000000..29c16025fc45 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java @@ -0,0 +1,31 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +public class DropMessageFunction implements Function { + + public static final String NAME = "drop_message"; + + @Override + public Void evaluate(Map args, EvaluationContext context, Message message) { + message.setFilterOut(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .pure(true) + .returnType(Void.class) + .params(ImmutableList.of()) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index f53bcda0e9dc..5bdc2372f597 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -14,6 +14,8 @@ import org.graylog.plugins.messageprocessor.ast.expressions.DoubleExpression; import org.graylog.plugins.messageprocessor.ast.expressions.EqualityExpression; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.FieldRefExpression; import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; import org.graylog.plugins.messageprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.messageprocessor.ast.expressions.LongExpression; @@ -76,6 +78,9 @@ private class AstBuilder extends RuleLangBaseListener { private final Set definedVars = Sets.newHashSet(); + // this is true for nested field accesses + private boolean idIsFieldAccess = false; + public AstBuilder(ParseContext parseContext) { this.parseContext = parseContext; args = parseContext.arguments(); @@ -146,6 +151,22 @@ public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { log.info("Declaring rule {}", rule); } + @Override + public void enterNested(RuleLangParser.NestedContext ctx) { + // nested field access is ok, these are not rule variables + idIsFieldAccess = true; + } + + @Override + public void exitNested(RuleLangParser.NestedContext ctx) { + idIsFieldAccess = false; // reset for error checks + final Expression object = exprs.get(ctx.fieldSet); + final Expression field = exprs.get(ctx.field); + final FieldAccessExpression expr = new FieldAccessExpression(object, field); + log.info("NOT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + @Override public void exitNot(RuleLangParser.NotContext ctx) { final LogicalExpression expression = (LogicalExpression) exprs.get(ctx.expression()); @@ -249,10 +270,15 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { final String variableName = ctx.Identifier().getText(); - if (!definedVars.contains(variableName)) { + if (!idIsFieldAccess && !definedVars.contains(variableName)) { parseContext.addError(new UndeclaredVariable(ctx)); } - final VarRefExpression expr = new VarRefExpression(variableName); + final Expression expr; + if (idIsFieldAccess) { + expr = new FieldRefExpression(variableName); + } else { + expr = new VarRefExpression(variableName); + } log.info("VAR: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index c4bb28743972..01140827e8ca 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -43,17 +43,20 @@ public Messages process(Messages messages) { log.info("Evaluation rule {}", rule.name()); for (Message message : messages) { - final EvaluationContext context = new EvaluationContext(functionRegistry); - if (rule.when().evaluateBool(context, message)) { - log.info("[✓] Message {} matches condition", message.getId()); - - for (Statement statement : rule.then()) { - statement.evaluate(context, message); - } + try { + final EvaluationContext context = new EvaluationContext(functionRegistry); + if (rule.when().evaluateBool(context, message)) { + log.info("[✓] Message {} matches condition", message.getId()); + for (Statement statement : rule.then()) { + statement.evaluate(context, message); + } - } else { - log.info("[✕] Message {} does not match condition", message.getId()); + } else { + log.info("[✕] Message {} does not match condition", message.getId()); + } + } catch (Exception e) { + log.error("Unable to process message", e); } } } From 4cb223df1ce8b79e304d150c220e128b0337cfb3 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 30 Dec 2015 19:03:06 +0100 Subject: [PATCH 006/521] clean up rest api - support delete, get all - properly render errors (not all parse errors are reflected yet) --- .../db/RuleSourceService.java | 4 +++ .../messageprocessor/parser/ParseError.java | 6 +++- .../parser/UndeclaredFunction.java | 5 +++- .../parser/UndeclaredVariable.java | 5 +++- .../processors/NaiveRuleProcessor.java | 22 +++++++++++++- .../rest/MessageProcessorRuleResource.java | 30 +++++++++++++++---- .../messageprocessor/rest/RuleSource.java | 2 ++ 7 files changed, 65 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java index 183c984ee916..b4529eb9dc51 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java @@ -54,4 +54,8 @@ public Collection loadAll() { return Collections.emptySet(); } } + + public void delete(String id) { + dbCollection.removeById(id); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java index d1bc15a21bf6..bdfbdbdbf443 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java @@ -6,10 +6,14 @@ public abstract class ParseError { + @JsonProperty + private final String type; + @JsonIgnore private final ParserRuleContext ctx; - protected ParseError(ParserRuleContext ctx) { + protected ParseError(String type, ParserRuleContext ctx) { + this.type = type; this.ctx = ctx; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java index 3d7ab28ec2d6..d1e8d40d5bd8 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java @@ -1,13 +1,16 @@ package org.graylog.plugins.messageprocessor.parser; +import com.fasterxml.jackson.annotation.JsonProperty; + public class UndeclaredFunction extends ParseError { private final RuleLangParser.FunctionCallContext ctx; public UndeclaredFunction(RuleLangParser.FunctionCallContext ctx) { - super(ctx); + super("undeclared_function", ctx); this.ctx = ctx; } + @JsonProperty("reason") @Override public String toString() { return "Unknown function " + ctx.funcName.getText() + positionString(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java index 2e0374ca29c0..3f26e693196a 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java @@ -1,16 +1,19 @@ package org.graylog.plugins.messageprocessor.parser; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; public class UndeclaredVariable extends ParseError { + @JsonIgnore private final RuleLangParser.IdentifierContext ctx; public UndeclaredVariable(RuleLangParser.IdentifierContext ctx) { - super(ctx); + super("undeclared_variable", ctx); this.ctx = ctx; } + @JsonProperty("reason") @Override public String toString() { return "Undeclared variable " + ctx.Identifier().getText() + positionString(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index 01140827e8ca..fe0239b87e04 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -1,5 +1,7 @@ package org.graylog.plugins.messageprocessor.processors; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.statements.Statement; @@ -11,23 +13,35 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; import org.graylog2.plugin.messageprocessors.MessageProcessor; +import org.graylog2.shared.buffers.processors.ProcessBufferProcessor; +import org.graylog2.shared.journal.Journal; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; +import static com.codahale.metrics.MetricRegistry.name; + public class NaiveRuleProcessor implements MessageProcessor { private static final Logger log = LoggerFactory.getLogger(NaiveRuleProcessor.class); private final RuleSourceService ruleSourceService; private final RuleParser ruleParser; private final FunctionRegistry functionRegistry; + private final Journal journal; + private final Meter filteredOutMessages; @Inject - public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser ruleParser, FunctionRegistry functionRegistry) { + public NaiveRuleProcessor(RuleSourceService ruleSourceService, + RuleParser ruleParser, + FunctionRegistry functionRegistry, + Journal journal, + MetricRegistry metricRegistry) { this.ruleSourceService = ruleSourceService; this.ruleParser = ruleParser; this.functionRegistry = functionRegistry; + this.journal = journal; + this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); } @Override @@ -58,6 +72,12 @@ public Messages process(Messages messages) { } catch (Exception e) { log.error("Unable to process message", e); } + if (message.getFilterOut()) { + log.info("[✝] Message {} was filtered out", message.getId()); + + filteredOutMessages.mark(); + journal.markJournalOffsetCommitted(message.getJournalOffset()); + } } } return messages; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java index b3a6bb702417..7cf150a01727 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java @@ -26,6 +26,7 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.Collection; @Api(value = "MessageProcessing", description = "Message processing pipeline") @Path("/messageprocessors") @@ -48,7 +49,7 @@ public MessageProcessorRuleResource(RuleSourceService ruleSourceService, RulePar @Produces(MediaType.APPLICATION_JSON) @Path("/rule") @POST - public RuleSource createFromParser(@ApiParam(name = "processingRule", required = true) @NotNull String ruleSource) throws ParseException { + public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull String ruleSource) throws ParseException { try { ruleParser.parseRule(ruleSource); } catch (ParseException e) { @@ -64,6 +65,15 @@ public RuleSource createFromParser(@ApiParam(name = "processingRule", required = return save; } + @ApiOperation(value = "Get all processing rules") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("/rule") + @GET + public Collection getAll() { + return ruleSourceService.loadAll(); + } + @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @@ -78,9 +88,19 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws @Produces(MediaType.APPLICATION_JSON) @Path("/rule/{id}") @PUT - public Response update(@ApiParam(name = "id") @PathParam("id") String id) { - // TODO - return Response.ok().build(); + public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, + @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { + final RuleSource ruleSource = ruleSourceService.load(id); + try { + ruleParser.parseRule(update.source()); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final RuleSource toSave = ruleSource.toBuilder() + .source(update.source()) + .modifiedAt(DateTime.now()) + .build(); + return ruleSourceService.save(toSave); } @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") @@ -89,7 +109,7 @@ public Response update(@ApiParam(name = "id") @PathParam("id") String id) { @Path("/rule/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) { - // TODO + ruleSourceService.delete(id); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java index 0d92e951cbe8..58a885860d12 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java @@ -1,5 +1,6 @@ package org.graylog.plugins.messageprocessor.rest; +import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; @@ -9,6 +10,7 @@ import javax.annotation.Nullable; @AutoValue +@JsonAutoDetect public abstract class RuleSource { @JsonProperty("_id") From 2ef8d6cd92355ba546a5ab8a7f8f71a727240c14 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 12:09:07 +0100 Subject: [PATCH 007/521] basic type checking - logical expressions now properly report their type (Boolean) - functions return their declared types - minimal expression walker checks type equality on operations and adds errors if necessary - fixed tests --- .../messageprocessor/parser/RuleLang.g4 | 4 + .../ast/expressions/AndExpression.java | 5 ++ .../ast/expressions/BinaryExpression.java | 3 + .../ast/expressions/ComparisonExpression.java | 5 ++ .../ast/expressions/EqualityExpression.java | 5 ++ .../expressions/FieldAccessExpression.java | 10 +-- .../ast/expressions/FunctionExpression.java | 13 ++- .../ast/expressions/NotExpression.java | 5 ++ .../ast/expressions/OrExpression.java | 5 ++ .../ast/expressions/UnaryExpression.java | 4 + .../ast/functions/Function.java | 23 ++++- .../ast/functions/FunctionDescriptor.java | 19 ++-- .../builtin/DropMessageFunction.java | 6 +- .../ast/functions/builtin/InputFunction.java | 8 +- .../parser/FunctionRegistry.java | 9 ++ .../parser/IncompatibleTypes.java | 28 ++++++ .../messageprocessor/parser/ParseError.java | 16 ++++ .../parser/ParseException.java | 8 +- .../messageprocessor/parser/RuleParser.java | 88 +++++++++++++++++-- .../parser/RuleParserTest.java | 47 ++++++++-- .../messageprocessor/parser/basicRule.txt | 2 +- .../parser/undeclaredIdentifier.txt | 2 +- 22 files changed, 267 insertions(+), 48 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index e937c3bdf646..b454ce0e97ae 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -29,6 +29,10 @@ grammar RuleLang; +@header { +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +} + ruleDeclaration : Rule name=String (During Stage stage=Integer)? diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java index 1a8eca892cc4..b2930634bb47 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java @@ -19,6 +19,11 @@ public boolean evaluateBool(EvaluationContext context, Message message) { return ((LogicalExpression)left).evaluateBool(context, message) && ((LogicalExpression)right).evaluateBool(context, message); } + @Override + public Class getType() { + return Boolean.class; + } + @Override public String toString() { return left.toString() + " AND " + right.toString(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java index fee036ef7e44..dcbb8dbcd635 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java @@ -14,4 +14,7 @@ public boolean isConstant() { return left.isConstant() && right.isConstant(); } + public Expression left() { + return left; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java index e87f7bd4590b..885b25bd85e3 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java @@ -19,6 +19,11 @@ public Object evaluate(EvaluationContext context, Message message) { return evaluateBool(context, message); } + @Override + public Class getType() { + return Boolean.class; + } + @Override public boolean evaluateBool(EvaluationContext context, Message message) { if (!numericArgs) { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java index 8e05997f02ef..aeb4c9dc0c04 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java @@ -16,6 +16,11 @@ public Object evaluate(EvaluationContext context, Message message) { return evaluateBool(context, message); } + @Override + public Class getType() { + return Boolean.class; + } + @Override public boolean evaluateBool(EvaluationContext context, Message message) { final boolean equals = left.evaluate(context, message).equals(right.evaluate(context, message)); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java index 6fe269665f3e..57450298a09d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java @@ -1,13 +1,12 @@ package org.graylog.plugins.messageprocessor.ast.expressions; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.beanutils.PropertyUtils; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog2.plugin.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; public class FieldAccessExpression implements Expression { private static final Logger log = LoggerFactory.getLogger(FieldAccessExpression.class); @@ -30,10 +29,11 @@ public Object evaluate(EvaluationContext context, Message message) { final Object bean = this.object.evaluate(context, message); final String fieldName = field.evaluate(context, message).toString(); try { - final Method method = bean.getClass().getMethod("get"+ StringUtils.capitalize(fieldName)); - return method.invoke(bean); + final Object property = PropertyUtils.getProperty(bean, fieldName); + log.debug("[field access] property {} of bean {}: {}", fieldName, bean.getClass().getTypeName(), property); + return property; } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { - log.error("Oops"); + log.error("Unable to read property {} from {}", fieldName, bean); return null; } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index bb747a71b734..4e192743ec75 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -2,7 +2,8 @@ import com.google.common.base.Joiner; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.FieldSet; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog2.plugin.Message; import java.util.Map; @@ -10,10 +11,14 @@ public class FunctionExpression implements Expression { private final String name; private final Map args; + private final Function function; + private final FunctionDescriptor descriptor; - public FunctionExpression(String name, Map args) { + public FunctionExpression(String name, Map args, Function function) { this.name = name; this.args = args; + this.function = function; + this.descriptor = function.descriptor(); } @Override @@ -23,12 +28,12 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context, Message message) { - return context.invokeFunction(context, message, name, args); + return descriptor.returnType().cast(function.evaluate(args, context, message)); } @Override public Class getType() { - return FieldSet.class; + return descriptor.returnType(); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java index 4fdfb579b864..294d0d7a6221 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java @@ -18,6 +18,11 @@ public boolean evaluateBool(EvaluationContext context, Message message) { return !((LogicalExpression)right).evaluateBool(context, message); } + @Override + public Class getType() { + return Boolean.class; + } + @Override public String toString() { return "NOT " + right.toString(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java index f221e7714ff2..42d1e0e546fe 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java @@ -19,6 +19,11 @@ public boolean evaluateBool(EvaluationContext context, Message message) { return ((LogicalExpression)left).evaluateBool(context, message) || ((LogicalExpression)right).evaluateBool(context, message); } + @Override + public Class getType() { + return Boolean.class; + } + @Override public String toString() { return left.toString() + " OR " + right.toString(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java index 62bc231c7ddd..65115b276924 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java @@ -17,4 +17,8 @@ public boolean isConstant() { public Class getType() { return right.getType(); } + + public Expression right() { + return right; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java index 7b267f240d08..92bc0552deae 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java @@ -1,15 +1,32 @@ package org.graylog.plugins.messageprocessor.ast.functions; +import com.google.common.collect.ImmutableList; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog2.plugin.Message; import java.util.Map; -public interface Function { +public interface Function { - Object evaluate(Map args, EvaluationContext context, Message message); + Function ERROR_FUNCTION = new Function() { + @Override + public Void evaluate(Map args, EvaluationContext context, Message message) { + return null; + } - FunctionDescriptor descriptor(); + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("__unresolved_function") + .returnType(Void.class) + .params(ImmutableList.of()) + .build(); + } + }; + + T evaluate(Map args, EvaluationContext context, Message message); + + FunctionDescriptor descriptor(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java index 2538c188b561..570bb8a87250 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java @@ -4,27 +4,28 @@ import com.google.common.collect.ImmutableList; @AutoValue -public abstract class FunctionDescriptor { +public abstract class FunctionDescriptor { public abstract String name(); public abstract boolean pure(); - public abstract Class returnType(); + public abstract Class returnType(); public abstract ImmutableList params(); - public static Builder builder() { + public static Builder builder() { + //noinspection unchecked return new AutoValue_FunctionDescriptor.Builder().pure(false); } @AutoValue.Builder - public static abstract class Builder { - public abstract FunctionDescriptor build(); + public static abstract class Builder { + public abstract FunctionDescriptor build(); - public abstract Builder name(String name); - public abstract Builder pure(boolean pure); - public abstract Builder returnType(Class type); - public abstract Builder params(ImmutableList params); + public abstract Builder name(String name); + public abstract Builder pure(boolean pure); + public abstract Builder returnType(Class type); + public abstract Builder params(ImmutableList params); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java index 29c16025fc45..f0d5975f8939 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java @@ -9,7 +9,7 @@ import java.util.Map; -public class DropMessageFunction implements Function { +public class DropMessageFunction implements Function { public static final String NAME = "drop_message"; @@ -20,8 +20,8 @@ public Void evaluate(Map args, EvaluationContext context, Me } @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() .name(NAME) .pure(true) .returnType(Void.class) diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java index a7de3d58b1c8..017885cc06a1 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java @@ -15,7 +15,7 @@ import static com.google.common.collect.ImmutableList.of; -public class InputFunction implements Function { +public class InputFunction implements Function { public static final String NAME = "input"; @@ -27,15 +27,15 @@ public InputFunction(InputRegistry inputRegistry) { } @Override - public Object evaluate(Map args, EvaluationContext context, Message message) { + public MessageInput evaluate(Map args, EvaluationContext context, Message message) { final Object id = args.get("id").evaluate(context, message); final IOState inputState = inputRegistry.getInputState(id.toString()); return inputState.getStoppable(); } @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() .name(NAME) .returnType(MessageInput.class) .params(of(ParameterDescriptor.string("id"))) diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java index 53e8b30b06b4..91e8de9710ea 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java @@ -6,6 +6,7 @@ import java.util.Map; public class FunctionRegistry { + private final Map functions; @Inject @@ -17,4 +18,12 @@ public FunctionRegistry(Map functions) { public Function resolve(String name) { return functions.get(name); } + + public Function resolveOrError(String name) { + final Function function = resolve(name); + if (function == null) { + return Function.ERROR_FUNCTION; + } + return function; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java new file mode 100644 index 000000000000..92336d7a2731 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java @@ -0,0 +1,28 @@ +package org.graylog.plugins.messageprocessor.parser; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; + +public class IncompatibleTypes extends ParseError { + private final RuleLangParser.ExpressionContext ctx; + private final BinaryExpression binaryExpr; + + public IncompatibleTypes(RuleLangParser.ExpressionContext ctx, BinaryExpression binaryExpr) { + super("incompatible_types", ctx); + this.ctx = ctx; + this.binaryExpr = binaryExpr; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); + } + + private String exprString(Expression e) { + return "(" + e.toString() + ") : " + e.getType().getSimpleName(); + } + + +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java index bdfbdbdbf443..5c12b1758a2b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.antlr.v4.runtime.ParserRuleContext; +import java.util.Objects; + public abstract class ParseError { @JsonProperty @@ -32,4 +34,18 @@ protected String positionString() { " line " + line() + " pos " + positionInLine(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ParseError)) return false; + ParseError that = (ParseError) o; + return Objects.equals(type, that.type) && + Objects.equals(ctx, that.ctx); + } + + @Override + public int hashCode() { + return Objects.hash(type, ctx); + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java index 8cbdd9f98236..efb2d36b74c0 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java @@ -1,15 +1,15 @@ package org.graylog.plugins.messageprocessor.parser; -import java.util.List; +import java.util.Set; public class ParseException extends RuntimeException { - private final List errors; + private final Set errors; - public ParseException(List errors) { + public ParseException(Set errors) { this.errors = errors; } - public List getErrors() { + public Set getErrors() { return errors; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 5bdc2372f597..9805703a5613 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -5,10 +5,14 @@ import com.google.common.collect.Sets; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTreeProperty; import org.antlr.v4.runtime.tree.ParseTreeWalker; +import org.apache.mina.util.IdentityHashSet; import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BooleanExpression; import org.graylog.plugins.messageprocessor.ast.expressions.ComparisonExpression; import org.graylog.plugins.messageprocessor.ast.expressions.DoubleExpression; @@ -62,7 +66,7 @@ public Rule parseRule(String rule) throws ParseException { // 3. optimizer: TODO WALKER.walk(new AstBuilder(parseContext), ruleDeclaration); - WALKER.walk(new SanityCheck(parseContext), ruleDeclaration); + WALKER.walk(new TypeChecker(parseContext), ruleDeclaration); if (parseContext.getErrors().isEmpty()) { return parseContext.getRule(); @@ -106,11 +110,13 @@ public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); final Map args = this.args.get(ctx.arguments()); - final FunctionExpression expr = new FunctionExpression(name, args); if (functionRegistry.resolve(name) == null) { parseContext.addError(new UndeclaredFunction(ctx)); } + + final FunctionExpression expr = new FunctionExpression(name, args, functionRegistry.resolveOrError(name)); + log.info("FUNC: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -252,12 +258,14 @@ public void exitBoolean(RuleLangParser.BooleanContext ctx) { public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { // nothing to do, just propagate the ConstantExpression exprs.put(ctx, exprs.get(ctx.literal())); + parseContext.addInnerNode(ctx); } @Override public void exitParenExpr(RuleLangParser.ParenExprContext ctx) { // nothing to do, just propagate exprs.put(ctx, exprs.get(ctx.expression())); + parseContext.addInnerNode(ctx); } @Override @@ -287,12 +295,14 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { public void exitPrimaryExpression(RuleLangParser.PrimaryExpressionContext ctx) { // nothing to do, just propagate exprs.put(ctx, exprs.get(ctx.primary())); + parseContext.addInnerNode(ctx); } @Override public void exitFunc(RuleLangParser.FuncContext ctx) { // nothing to do, just propagate exprs.put(ctx, exprs.get(ctx.functionCall())); + parseContext.addInnerNode(ctx); } @Override @@ -302,13 +312,66 @@ public void exitNull(RuleLangParser.NullContext ctx) { } } - private class SanityCheck extends RuleLangBaseListener { + private class TypeChecker extends RuleLangBaseListener { private final ParseContext parseContext; - - public SanityCheck(ParseContext parseContext) { + StringBuffer sb = new StringBuffer(); + public TypeChecker(ParseContext parseContext) { this.parseContext = parseContext; } + @Override + public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { + log.info("Type tree {}", sb.toString()); + } + + @Override + public void exitAnd(RuleLangParser.AndContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitOr(RuleLangParser.OrContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitComparison(RuleLangParser.ComparisonContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitEquality(RuleLangParser.EqualityContext ctx) { + checkBinaryExpression(ctx); + } + + private void checkBinaryExpression(RuleLangParser.ExpressionContext ctx) { + final BinaryExpression binaryExpr = (BinaryExpression) parseContext.expressions().get(ctx); + final Class leftType = binaryExpr.left().getType(); + final Class rightType = binaryExpr.right().getType(); + + if (!leftType.equals(rightType)) { + parseContext.addError(new IncompatibleTypes(ctx, binaryExpr)); + } + } + + @Override + public void enterEveryRule(ParserRuleContext ctx) { + final Expression expression = parseContext.expressions().get(ctx); + if (expression != null && !parseContext.isInnerNode(ctx)) { + sb.append(" ( "); + sb.append(expression.getClass().getSimpleName()); + sb.append(":").append(ctx.getClass().getSimpleName()).append(" "); + sb.append(" <").append(expression.getType().getSimpleName()).append(">"); + } + } + + @Override + public void exitEveryRule(ParserRuleContext ctx) { + final Expression expression = parseContext.expressions().get(ctx); + if (expression != null && !parseContext.isInnerNode(ctx)) { + sb.append(" ) "); + } + } } @@ -320,8 +383,9 @@ public SanityCheck(ParseContext parseContext) { private static class ParseContext { private final ParseTreeProperty exprs = new ParseTreeProperty<>(); private final ParseTreeProperty> args = new ParseTreeProperty<>(); - private List errors = Lists.newArrayList(); - + private Set errors = Sets.newHashSet(); + // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information + private Set innerNodes = new IdentityHashSet<>(); public List statements = Lists.newArrayList(); public Rule rule; @@ -341,13 +405,21 @@ public void setRule(Rule rule) { this.rule = rule; } - public List getErrors() { + public Set getErrors() { return errors; } public void addError(ParseError error) { errors.add(error); } + + public void addInnerNode(RuleContext node) { + innerNodes.add(node); + } + + public boolean isInnerNode(RuleContext node) { + return innerNodes.contains(node); + } } } diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 241f83158bd5..762f8b21a1d1 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -1,6 +1,7 @@ package org.graylog.plugins.messageprocessor.parser; import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import org.graylog.plugins.messageprocessor.EvaluationContext; @@ -8,6 +9,7 @@ import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; import org.graylog2.plugin.Message; import org.junit.After; import org.junit.Assert; @@ -39,16 +41,48 @@ public class RuleParserTest { @BeforeClass public static void registerFunctions() { final Map functions = Maps.newHashMap(); - functions.put("nein", new Function() { + functions.put("nein", new Function() { @Override - public Object evaluate(Map args, EvaluationContext context, Message message) { + public Boolean evaluate(Map args, EvaluationContext context, Message message) { return false; } @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() .name("nein") + .returnType(Boolean.class) + .params(ImmutableList.of()) + .build(); + } + }); + functions.put("double_valued_func", new Function() { + @Override + public Double evaluate(Map args, EvaluationContext context, Message message) { + return 0d; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("double_valued_func") + .returnType(Double.class) + .params(ImmutableList.of()) + .build(); + } + }); + functions.put("one_arg", new Function() { + @Override + public String evaluate(Map args, EvaluationContext context, Message message) { + return "return"; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("one_arg") + .returnType(String.class) + .params(ImmutableList.of(ParameterDescriptor.string("one"))) .build(); } }); @@ -97,8 +131,9 @@ public void undeclaredFunction() throws Exception { parser.parseRule(ruleForTest()); fail("should throw error: undeclared function 'unknown'"); } catch (ParseException e) { - assertEquals(1, e.getErrors().size()); - assertTrue("Should find error UndeclaredFunction", Iterables.getOnlyElement(e.getErrors()) instanceof UndeclaredFunction); + assertEquals(2, e.getErrors().size()); + assertTrue("Should find error UndeclaredFunction", Iterables.getFirst(e.getErrors(), null) instanceof UndeclaredFunction); + assertTrue("Should find error IncompatibleTypes", Iterables.get(e.getErrors(), 1, null) instanceof IncompatibleTypes); } } private String ruleForTest() { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt index f17a8f8e6bc6..5d4362b2a9d3 100644 --- a/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt @@ -2,5 +2,5 @@ rule "something" during stage 0 when double_valued_func() > 1.0d AND false == true then -some_func(); +double_valued_func(); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt index ad032bc2e23f..b0e11a3f5e75 100644 --- a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt @@ -1,5 +1,5 @@ rule "undeclared variable" when true then - some_func(x); + one_arg(one: x); end \ No newline at end of file From 3ca611b4644c1824da0f26ac190b57896533dc43 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 12:36:59 +0100 Subject: [PATCH 008/521] syntactic sugar for single parameter functions: func("arg") == func(parameterName: "arg") - add test --- .../ast/expressions/FunctionExpression.java | 6 +- .../parser/FunctionRegistry.java | 10 +-- .../messageprocessor/parser/RuleParser.java | 12 +++- .../parser/RuleParserTest.java | 64 +++++++++++++++++-- .../parser/singleArgFunction.txt | 5 ++ 5 files changed, 86 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/singleArgFunction.txt diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index 4e192743ec75..a8d10cbf85db 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -21,6 +21,10 @@ public FunctionExpression(String name, Map args, Function fu this.descriptor = function.descriptor(); } + public Function getFunction() { + return function; + } + @Override public boolean isConstant() { return false; @@ -42,6 +46,6 @@ public String toString() { if (args != null) { join = Joiner.on(", ").withKeyValueSeparator(": ").join(args); // TODO order arg names } - return name + "(" + join + ")"; + return descriptor.name() + "(" + join + ")"; } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java index 91e8de9710ea..a8d5a1cd2571 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java @@ -7,20 +7,20 @@ public class FunctionRegistry { - private final Map functions; + private final Map> functions; @Inject - public FunctionRegistry(Map functions) { + public FunctionRegistry(Map> functions) { this.functions = functions; } - public Function resolve(String name) { + public Function resolve(String name) { return functions.get(name); } - public Function resolveOrError(String name) { - final Function function = resolve(name); + public Function resolveOrError(String name) { + final Function function = resolve(name); if (function == null) { return Function.ERROR_FUNCTION; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 9805703a5613..392bc1bcb5fb 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -28,6 +28,8 @@ import org.graylog.plugins.messageprocessor.ast.expressions.OrExpression; import org.graylog.plugins.messageprocessor.ast.expressions.StringExpression; import org.graylog.plugins.messageprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.messageprocessor.ast.statements.FunctionStatement; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.ast.statements.VarAssignStatement; @@ -111,8 +113,16 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); final Map args = this.args.get(ctx.arguments()); - if (functionRegistry.resolve(name) == null) { + final Function function = functionRegistry.resolve(name); + if (function == null) { parseContext.addError(new UndeclaredFunction(ctx)); + } else { + // convert null key, single arg style to default named args for single arg functions + if (args != null && args.containsKey(null) && function.descriptor().params().size() == 1) { + final Expression argExpr = args.remove(null); + final ParameterDescriptor param = function.descriptor().params().get(0); + args.put(param.name(), argExpr); + } } final FunctionExpression expr = new FunctionExpression(name, args, functionRegistry.resolveOrError(name)); diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 762f8b21a1d1..0aedd8c45446 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -10,7 +10,9 @@ import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog2.plugin.Message; +import org.joda.time.DateTime; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -18,6 +20,7 @@ import org.junit.Test; import org.junit.rules.TestName; +import javax.annotation.Nullable; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; @@ -25,8 +28,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -38,9 +43,11 @@ public class RuleParserTest { private RuleParser parser; private static FunctionRegistry functionRegistry; + private static final AtomicBoolean actionsTriggered = new AtomicBoolean(false); + @BeforeClass public static void registerFunctions() { - final Map functions = Maps.newHashMap(); + final Map> functions = Maps.newHashMap(); functions.put("nein", new Function() { @Override public Boolean evaluate(Map args, EvaluationContext context, Message message) { @@ -74,7 +81,7 @@ public FunctionDescriptor descriptor() { functions.put("one_arg", new Function() { @Override public String evaluate(Map args, EvaluationContext context, Message message) { - return "return"; + return args.get("one").toString(); } @Override @@ -86,12 +93,30 @@ public FunctionDescriptor descriptor() { .build(); } }); + functions.put("trigger_test", new Function() { + @Override + public Void evaluate(Map args, EvaluationContext context, Message message) { + actionsTriggered.set(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("trigger_test") + .returnType(Void.class) + .params(ImmutableList.of()) + .build(); + } + }); functionRegistry = new FunctionRegistry(functions); } @Before public void setup() { parser = new RuleParser(functionRegistry); + // initialize before every test! + actionsTriggered.set(false); } @After @@ -132,10 +157,41 @@ public void undeclaredFunction() throws Exception { fail("should throw error: undeclared function 'unknown'"); } catch (ParseException e) { assertEquals(2, e.getErrors().size()); - assertTrue("Should find error UndeclaredFunction", Iterables.getFirst(e.getErrors(), null) instanceof UndeclaredFunction); - assertTrue("Should find error IncompatibleTypes", Iterables.get(e.getErrors(), 1, null) instanceof IncompatibleTypes); + assertTrue("Should find error UndeclaredFunction", + e.getErrors().stream().anyMatch(input -> input instanceof UndeclaredFunction)); + assertTrue("Should find error IncompatibleTypes", + e.getErrors().stream().anyMatch(input -> input instanceof IncompatibleTypes)); + } + } + + @Test + public void singleArgFunction() throws Exception { + try { + final Rule rule = parser.parseRule(ruleForTest()); + final Message message = evaluateRule(rule); + + assertNotNull(message); + assertTrue("actions should have triggered", actionsTriggered.get()); + } catch (ParseException e) { + fail("Should not fail to parse"); } } + + @Nullable + private Message evaluateRule(Rule rule) { + final EvaluationContext context = new EvaluationContext(functionRegistry); + final Message message = new Message("hello test", "source", DateTime.now()); + if (rule.when().evaluateBool(context, message)) { + + for (Statement statement : rule.then()) { + statement.evaluate(context, message); + } + return message; + } else { + return null; + } + } + private String ruleForTest() { try { final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/singleArgFunction.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/singleArgFunction.txt new file mode 100644 index 000000000000..5c8d9d718661 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/singleArgFunction.txt @@ -0,0 +1,5 @@ +rule "single arg" +when one_arg("arg") == one_arg(one: "arg") +then + trigger_test(); +end \ No newline at end of file From cb2096f6b3bf970a5a3e33833ab91eeb97dc1386 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 13:32:16 +0100 Subject: [PATCH 009/521] type annotator compilation phase, type inference for variables, func argument type checking --- .../messageprocessor/EvaluationContext.java | 38 +++++++----- .../ast/expressions/FunctionExpression.java | 10 +++- .../ast/expressions/VarRefExpression.java | 21 ++++++- .../ast/statements/Statement.java | 1 + .../ast/statements/VarAssignStatement.java | 4 +- .../parser/IncompatibleArgumentType.java | 32 ++++++++++ .../messageprocessor/parser/RuleParser.java | 60 ++++++++++++++++++- .../processors/NaiveRuleProcessor.java | 2 +- .../parser/RuleParserTest.java | 31 ++++++++-- .../parser/inferVariableType.txt | 6 ++ .../parser/invalidArgType.txt | 6 ++ 11 files changed, 182 insertions(+), 29 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/inferVariableType.txt create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/invalidArgType.txt diff --git a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java index 0fc48149e541..eb2d112c6621 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java @@ -1,30 +1,40 @@ package org.graylog.plugins.messageprocessor; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.parser.FunctionRegistry; -import org.graylog2.plugin.Message; +import com.google.common.collect.Maps; import java.util.Map; public class EvaluationContext { - private final FunctionRegistry functionRegistry; + private Map ruleVars; - public EvaluationContext(FunctionRegistry functionRegistry) { - this.functionRegistry = functionRegistry; + public EvaluationContext() { + ruleVars = Maps.newHashMap(); } - public void define(String name, Object result) { + public void define(String identifier, Class type, Object value) { + ruleVars.put(identifier, new TypedValue(type, value)); } - public Object get(String identifier) { - return null; + public TypedValue get(String identifier) { + return ruleVars.get(identifier); } - public Object invokeFunction(EvaluationContext context, Message message, String functionName, Map args) { - final Function function = functionRegistry.resolve(functionName); - // TODO missing function - return function.evaluate(args, context, message); + public class TypedValue { + private final Class type; + private final Object value; + + public TypedValue(Class type, Object value) { + this.type = type; + this.value = value; + } + + public Class getType() { + return type; + } + + public Object getValue() { + return value; + } } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index a8d10cbf85db..cb9ab7f97d22 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -11,20 +11,24 @@ public class FunctionExpression implements Expression { private final String name; private final Map args; - private final Function function; + private final Function function; private final FunctionDescriptor descriptor; - public FunctionExpression(String name, Map args, Function function) { + public FunctionExpression(String name, Map args, Function function) { this.name = name; this.args = args; this.function = function; this.descriptor = function.descriptor(); } - public Function getFunction() { + public Function getFunction() { return function; } + public Map getArgs() { + return args; + } + @Override public boolean isConstant() { return false; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java index 105434e03b38..63d04539fc0e 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java @@ -2,9 +2,13 @@ import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog2.plugin.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class VarRefExpression implements Expression { + private static final Logger log = LoggerFactory.getLogger(VarRefExpression.class); private final String identifier; + private Class type = Object.class; public VarRefExpression(String identifier) { this.identifier = identifier; @@ -17,16 +21,29 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context, Message message) { - return context.get(identifier); + final EvaluationContext.TypedValue typedValue = context.get(identifier); + if (typedValue != null) { + return typedValue.getValue(); + } + log.error("Unable to retrieve value for variable {}", identifier); + return null; } @Override public Class getType() { - return Object.class; + return type; } @Override public String toString() { return identifier; } + + public String varName() { + return identifier; + } + + public void setType(Class type) { + this.type = type; + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java index 5e4429b4bc71..251993474f39 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java @@ -5,5 +5,6 @@ public interface Statement { + // TODO should this have a return value at all? Object evaluate(EvaluationContext context, Message message); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java index 94d740c36dd5..20ceb64c798d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java @@ -14,9 +14,9 @@ public VarAssignStatement(String name, Expression expr) { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Void evaluate(EvaluationContext context, Message message) { final Object result = expr.evaluate(context, message); - context.define(name, result); + context.define(name, expr.getType(), result); return null; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java new file mode 100644 index 000000000000..955875b06c19 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java @@ -0,0 +1,32 @@ +package org.graylog.plugins.messageprocessor.parser; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; + +public class IncompatibleArgumentType extends ParseError { + private final FunctionExpression functionExpression; + private final ParameterDescriptor p; + private final Expression argExpr; + + public IncompatibleArgumentType(RuleLangParser.FunctionCallContext ctx, + FunctionExpression functionExpression, + ParameterDescriptor p, + Expression argExpr) { + super("incompatible_argument_type", ctx); + this.functionExpression = functionExpression; + this.p = p; + this.argExpr = argExpr; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + p.type().getSimpleName() + + " for argument " + p.name() + + " but found " + argExpr.getType().getSimpleName() + + " in call to function " + functionExpression.getFunction().descriptor().name() + + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 392bc1bcb5fb..5821dc087614 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -29,6 +29,7 @@ import org.graylog.plugins.messageprocessor.ast.expressions.StringExpression; import org.graylog.plugins.messageprocessor.ast.expressions.VarRefExpression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.messageprocessor.ast.statements.FunctionStatement; import org.graylog.plugins.messageprocessor.ast.statements.Statement; @@ -64,10 +65,12 @@ public Rule parseRule(String rule) throws ParseException { // parsing stages: // 1. build AST nodes, checks for invalid var, function refs - // 2. checker: static type check w/ coercion nodes - // 3. optimizer: TODO + // 2. type annotator: infer type information from var refs, func refs + // 3. checker: static type check w/ coercion nodes + // 4. optimizer: TODO WALKER.walk(new AstBuilder(parseContext), ruleDeclaration); + WALKER.walk(new TypeAnnotator(parseContext), ruleDeclaration); WALKER.walk(new TypeChecker(parseContext), ruleDeclaration); if (parseContext.getErrors().isEmpty()) { @@ -104,6 +107,7 @@ public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { final String name = ctx.varName.getText(); final Expression expr = exprs.get(ctx.expression()); + parseContext.defineVar(name, expr); definedVars.add(name); parseContext.statements.add(new VarAssignStatement(name, expr)); } @@ -322,6 +326,30 @@ public void exitNull(RuleLangParser.NullContext ctx) { } } + private class TypeAnnotator extends RuleLangBaseListener { + private final ParseContext parseContext; + + public TypeAnnotator(ParseContext parseContext) { + this.parseContext = parseContext; + } + + @Override + public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { + final Expression expr = parseContext.expressions().get(ctx); + if (expr instanceof VarRefExpression) { + final VarRefExpression varRefExpression = (VarRefExpression) expr; + final String name = varRefExpression.varName(); + final Expression expression = parseContext.getDefinedVar(name); + if (expression == null) { + log.error("Unable to retrieve expression for variable {}, this is a bug", name); + return; + } + log.info("Inferred type of variable {} to {}", name, expression.getType().getSimpleName()); + varRefExpression.setType(expression.getType()); + } + } + } + private class TypeChecker extends RuleLangBaseListener { private final ParseContext parseContext; StringBuffer sb = new StringBuffer(); @@ -364,6 +392,19 @@ private void checkBinaryExpression(RuleLangParser.ExpressionContext ctx) { } } + @Override + public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { + final FunctionExpression expr = (FunctionExpression) parseContext.expressions().get(ctx); + final FunctionDescriptor descriptor = expr.getFunction().descriptor(); + final Map args = expr.getArgs(); + for (ParameterDescriptor p : descriptor.params()) { + final Expression argExpr = args.get(p.name()); + if (!argExpr.getType().equals(p.type())) { + parseContext.addError(new IncompatibleArgumentType(ctx, expr, p, argExpr)); + } + } + } + @Override public void enterEveryRule(ParserRuleContext ctx) { final Expression expression = parseContext.expressions().get(ctx); @@ -398,6 +439,7 @@ private static class ParseContext { private Set innerNodes = new IdentityHashSet<>(); public List statements = Lists.newArrayList(); public Rule rule; + private Map varDecls = Maps.newHashMap(); public ParseTreeProperty expressions() { return exprs; @@ -430,6 +472,20 @@ public void addInnerNode(RuleContext node) { public boolean isInnerNode(RuleContext node) { return innerNodes.contains(node); } + + /** + * Links the declared var to its expression. + * @param name var name + * @param expr expression + * @return true if successful, false if previously declared + */ + public boolean defineVar(String name, Expression expr) { + return varDecls.put(name, expr) == null; + } + + public Expression getDefinedVar(String name) { + return varDecls.get(name); + } } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index fe0239b87e04..2f3226a9874c 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -58,7 +58,7 @@ public Messages process(Messages messages) { for (Message message : messages) { try { - final EvaluationContext context = new EvaluationContext(functionRegistry); + final EvaluationContext context = new EvaluationContext(); if (rule.when().evaluateBool(context, message)) { log.info("[✓] Message {} matches condition", message.getId()); diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 0aedd8c45446..286be367d6ae 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -2,7 +2,6 @@ import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; @@ -81,7 +80,7 @@ public FunctionDescriptor descriptor() { functions.put("one_arg", new Function() { @Override public String evaluate(Map args, EvaluationContext context, Message message) { - return args.get("one").toString(); + return (String) args.get("one").evaluate(context, message); } @Override @@ -136,8 +135,8 @@ public void undeclaredIdentifier() throws Exception { parser.parseRule(ruleForTest()); fail("should throw error: undeclared variable x"); } catch (ParseException e) { - assertEquals(1, e.getErrors().size()); - assertTrue("Should find error UndeclaredVariable", Iterables.getOnlyElement(e.getErrors()) instanceof UndeclaredVariable); + assertEquals(2, e.getErrors().size()); // undeclared var and incompatible type, but we only care about the undeclared one here + assertTrue("Should find error UndeclaredVariable", e.getErrors().stream().anyMatch(error -> error instanceof UndeclaredVariable)); } } @@ -177,9 +176,31 @@ public void singleArgFunction() throws Exception { } } + @Test + public void inferVariableType() throws Exception { + try { + final Rule rule = parser.parseRule(ruleForTest()); + + evaluateRule(rule); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + + @Test + public void invalidArgType() throws Exception { + try { + parser.parseRule(ruleForTest()); + } catch (ParseException e) { + assertEquals(2, e.getErrors().size()); + assertTrue("Should only find IncompatibleArgumentType errors", + e.getErrors().stream().allMatch(input -> input instanceof IncompatibleArgumentType)); + } + } + @Nullable private Message evaluateRule(Rule rule) { - final EvaluationContext context = new EvaluationContext(functionRegistry); + final EvaluationContext context = new EvaluationContext(); final Message message = new Message("hello test", "source", DateTime.now()); if (rule.when().evaluateBool(context, message)) { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/inferVariableType.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/inferVariableType.txt new file mode 100644 index 000000000000..d72935a40e0d --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/inferVariableType.txt @@ -0,0 +1,6 @@ +rule "infer" +when true +then + let x = one_arg("string"); + one_arg(x); +end diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/invalidArgType.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/invalidArgType.txt new file mode 100644 index 000000000000..373df5300893 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/invalidArgType.txt @@ -0,0 +1,6 @@ +rule "invalid arg type" +when one_arg(0d) == "0" // one_arg needs a String argument, but 0d is Double +then + let x = double_valued_func(); + one_arg(x); // this needs a String argument, but x resolves to Double +end \ No newline at end of file From e4d89582b3dc5810e85c4a1175fe9fe63e0f4da4 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 13:34:04 +0100 Subject: [PATCH 010/521] remote unneeded parameter --- .../messageprocessor/processors/NaiveRuleProcessor.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index 2f3226a9874c..258ea7c8af37 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -6,7 +6,6 @@ import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.db.RuleSourceService; -import org.graylog.plugins.messageprocessor.parser.FunctionRegistry; import org.graylog.plugins.messageprocessor.parser.ParseException; import org.graylog.plugins.messageprocessor.parser.RuleParser; import org.graylog.plugins.messageprocessor.rest.RuleSource; @@ -27,19 +26,16 @@ public class NaiveRuleProcessor implements MessageProcessor { private final RuleSourceService ruleSourceService; private final RuleParser ruleParser; - private final FunctionRegistry functionRegistry; private final Journal journal; private final Meter filteredOutMessages; @Inject public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser ruleParser, - FunctionRegistry functionRegistry, Journal journal, MetricRegistry metricRegistry) { this.ruleSourceService = ruleSourceService; this.ruleParser = ruleParser; - this.functionRegistry = functionRegistry; this.journal = journal; this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); } From 7b1b53ff9ceffc0c26c7600c8f05ccbcbe2bb1f3 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 15:58:30 +0100 Subject: [PATCH 011/521] add has_field function - fix type literal for injection --- .../ProcessorPluginModule.java | 11 ++++--- .../ast/functions/builtin/HasField.java | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index 440db0d0c77b..3d185f1c8d8d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -1,9 +1,11 @@ package org.graylog.plugins.messageprocessor; import com.google.inject.Binder; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.builtin.DropMessageFunction; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.HasField; import org.graylog.plugins.messageprocessor.ast.functions.builtin.InputFunction; import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; @@ -26,19 +28,20 @@ protected void configure() { addRestResource(MessageProcessorRuleResource.class); // built-in functions + addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); } - protected void addMessageProcessorFunction(String name, Class functionClass) { + protected void addMessageProcessorFunction(String name, Class> functionClass) { addMessageProcessorFunction(binder(), name, functionClass); } - public static MapBinder processorFunctionBinder(Binder binder) { - return MapBinder.newMapBinder(binder, String.class, Function.class); + public static MapBinder> processorFunctionBinder(Binder binder) { + return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral>() {}); } - public static void addMessageProcessorFunction(Binder binder, String name, Class functionClass) { + public static void addMessageProcessorFunction(Binder binder, String name, Class> functionClass) { processorFunctionBinder(binder).addBinding(name).to(functionClass); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java new file mode 100644 index 000000000000..f3a7862e3744 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java @@ -0,0 +1,30 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +public class HasField implements Function { + + public static final String NAME = "has_field"; + + @Override + public Boolean evaluate(Map args, EvaluationContext context, Message message) { + return message.hasField((String) args.get("field").evaluate(context, message)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(ImmutableList.of(ParameterDescriptor.string("field"))) + .build(); + } +} From 33928f2e53c7a92e2df292707966c97b68f42076 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 15:59:06 +0100 Subject: [PATCH 012/521] implement lazy reloading/unloading of rules on cluster-wide changes --- .../events/RulesChangedEvent.java | 50 ++++++++++ .../processors/NaiveRuleProcessor.java | 97 ++++++++++++++++--- .../rest/MessageProcessorRuleResource.java | 16 ++- 3 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java diff --git a/src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java b/src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java new file mode 100644 index 000000000000..13b26c6d3569 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java @@ -0,0 +1,50 @@ +package org.graylog.plugins.messageprocessor.events; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import com.google.common.collect.Sets; + +import java.util.Set; + +import static java.util.Collections.emptySet; + +@AutoValue +public abstract class RulesChangedEvent { + + @JsonProperty + public abstract Set deletedRuleIds(); + + @JsonProperty + public abstract Set updatedRuleIds(); + + public static Builder builder() { + return new AutoValue_RulesChangedEvent.Builder().deletedRuleIds(emptySet()).updatedRuleIds(emptySet()); + } + + public static RulesChangedEvent updatedRuleId(String id) { + return builder().updatedRuleId(id).build(); + } + + public static RulesChangedEvent deletedRuleId(String id) { + return builder().deletedRuleId(id).build(); + } + + @JsonCreator + public static RulesChangedEvent create(@JsonProperty("deleted_rule_ids") Set deletedIds, @JsonProperty("updated_rule_ids") Set updatedIds) { + return builder().deletedRuleIds(deletedIds).updatedRuleIds(updatedIds).build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder deletedRuleIds(Set ids); + public Builder deletedRuleId(String id) { + return deletedRuleIds(Sets.newHashSet(id)); + } + public abstract Builder updatedRuleIds(Set ids); + public Builder updatedRuleId(String id) { + return updatedRuleIds(Sets.newHashSet(id)); + } + public abstract RulesChangedEvent build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index 258ea7c8af37..bafe74645fa9 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -2,13 +2,22 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.db.RuleSourceService; +import org.graylog.plugins.messageprocessor.events.RulesChangedEvent; import org.graylog.plugins.messageprocessor.parser.ParseException; import org.graylog.plugins.messageprocessor.parser.RuleParser; import org.graylog.plugins.messageprocessor.rest.RuleSource; +import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; import org.graylog2.plugin.messageprocessors.MessageProcessor; @@ -17,40 +26,48 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import javax.inject.Inject; +import javax.inject.Named; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; import static com.codahale.metrics.MetricRegistry.name; +import static com.google.common.cache.CacheLoader.asyncReloading; public class NaiveRuleProcessor implements MessageProcessor { private static final Logger log = LoggerFactory.getLogger(NaiveRuleProcessor.class); - private final RuleSourceService ruleSourceService; - private final RuleParser ruleParser; private final Journal journal; private final Meter filteredOutMessages; + private final LoadingCache ruleCache; @Inject public NaiveRuleProcessor(RuleSourceService ruleSourceService, RuleParser ruleParser, Journal journal, - MetricRegistry metricRegistry) { - this.ruleSourceService = ruleSourceService; - this.ruleParser = ruleParser; + MetricRegistry metricRegistry, + @Named("daemonScheduler") ScheduledExecutorService scheduledExecutorService, + @ClusterEventBus EventBus clusterBus) { this.journal = journal; this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); + clusterBus.register(this); + ruleCache = CacheBuilder.newBuilder() + .build(asyncReloading(new RuleLoader(ruleSourceService, ruleParser), scheduledExecutorService)); + // prime the cache with all presently stored rules + try { + ruleCache.getAll(Collections.emptyList()); + } catch (ExecutionException ignored) {} } @Override public Messages process(Messages messages) { - for (RuleSource ruleSource : ruleSourceService.loadAll()) { - final Rule rule; - try { - rule = ruleParser.parseRule(ruleSource.source()); - } catch (ParseException parseException) { - log.error("Unable to parse rule: " + parseException.getMessage()); - continue; - } - log.info("Evaluation rule {}", rule.name()); + for (Map.Entry entry : ruleCache.asMap().entrySet()) { + final Rule rule = entry.getValue(); + log.info("Evaluating rule {}", rule.name()); for (Message message : messages) { try { @@ -78,4 +95,56 @@ public Messages process(Messages messages) { } return messages; } + + @Subscribe + public void handleRuleChanges(RulesChangedEvent event) { + event.deletedRuleIds().forEach(id -> { + ruleCache.invalidate(id); + log.info("Invalidated rule {}", id); + }); + event.updatedRuleIds().forEach(id -> { + ruleCache.refresh(id); + log.info("Refreshing rule {}", id); + }); + } + + private static class RuleLoader extends CacheLoader { + private final RuleSourceService ruleSourceService; + private final RuleParser ruleParser; + + public RuleLoader(RuleSourceService ruleSourceService, RuleParser ruleParser) { + this.ruleSourceService = ruleSourceService; + this.ruleParser = ruleParser; + } + + @Override + public Map loadAll(Iterable keys) throws Exception { + final Map all = Maps.newHashMap(); + final HashSet keysToLoad = Sets.newHashSet(keys); + for (RuleSource ruleSource : ruleSourceService.loadAll()) { + if (!keysToLoad.isEmpty()) { + if (!keysToLoad.contains(ruleSource.id())) { + continue; + } + } + try { + all.put(ruleSource.id(), ruleParser.parseRule(ruleSource.source())); + } catch (ParseException e) { + log.error("Unable to parse rule: " + e.getMessage()); + } + } + return all; + } + + @Override + public Rule load(@Nullable String ruleId) throws Exception { + final RuleSource ruleSource = ruleSourceService.load(ruleId); + try { + return ruleParser.parseRule(ruleSource.source()); + } catch (ParseException e) { + log.error("Unable to parse rule: " + e.getMessage()); + throw e; + } + } + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java index 7cf150a01727..8c8e671a2139 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java @@ -1,12 +1,15 @@ package org.graylog.plugins.messageprocessor.rest; +import com.google.common.eventbus.EventBus; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; import org.graylog.plugins.messageprocessor.db.RuleSourceService; +import org.graylog.plugins.messageprocessor.events.RulesChangedEvent; import org.graylog.plugins.messageprocessor.parser.ParseException; import org.graylog.plugins.messageprocessor.parser.RuleParser; import org.graylog2.database.NotFoundException; +import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.shared.rest.resources.RestResource; import org.joda.time.DateTime; @@ -36,11 +39,15 @@ public class MessageProcessorRuleResource extends RestResource implements Plugin private final RuleSourceService ruleSourceService; private final RuleParser ruleParser; + private final EventBus clusterBus; @Inject - public MessageProcessorRuleResource(RuleSourceService ruleSourceService, RuleParser ruleParser) { + public MessageProcessorRuleResource(RuleSourceService ruleSourceService, + RuleParser ruleParser, + @ClusterEventBus EventBus clusterBus) { this.ruleSourceService = ruleSourceService; this.ruleParser = ruleParser; + this.clusterBus = clusterBus; } @@ -61,6 +68,7 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No .modifiedAt(DateTime.now()) .build(); final RuleSource save = ruleSourceService.save(newRuleSource); + clusterBus.post(RulesChangedEvent.updatedRuleId(save.id())); log.info("Created new rule {}", save); return save; } @@ -100,7 +108,10 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, .source(update.source()) .modifiedAt(DateTime.now()) .build(); - return ruleSourceService.save(toSave); + final RuleSource savedRule = ruleSourceService.save(toSave); + clusterBus.post(RulesChangedEvent.updatedRuleId(savedRule.id())); + + return savedRule; } @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") @@ -110,6 +121,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) { ruleSourceService.delete(id); + clusterBus.post(RulesChangedEvent.deletedRuleId(id)); } From cc332af3acab6abad4026d3bb8b0ab6509085235 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 16:13:01 +0100 Subject: [PATCH 013/521] allow using boolean-valued functions as sole top level conditions this allows to write rules like: ` rule "a" when has_field("something") then end ` instead of having to do `has_field("something") == true` --- .../BooleanValuedFunctionWrapper.java | 35 +++++++++++++++++++ .../messageprocessor/parser/RuleParser.java | 13 ++++++- .../parser/RuleParserTest.java | 27 ++++++++++++++ .../booleanValuedFunctionAsCondition.txt | 5 +++ 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/booleanValuedFunctionAsCondition.txt diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java new file mode 100644 index 000000000000..c6e136e8a4e6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -0,0 +1,35 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog2.plugin.Message; + +public class BooleanValuedFunctionWrapper implements LogicalExpression { + private final Expression expr; + + public BooleanValuedFunctionWrapper(Expression expr) { + this.expr = expr; + if (!expr.getType().equals(Boolean.class)) { + throw new IllegalArgumentException("expr must be of boolean type"); + } + } + + @Override + public boolean evaluateBool(EvaluationContext context, Message message) { + return (Boolean) expr.evaluate(context, message); + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context, Message message) { + return evaluateBool(context, message); + } + + @Override + public Class getType() { + return expr.getType(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 5821dc087614..758bc8ed67a9 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -14,6 +14,7 @@ import org.graylog.plugins.messageprocessor.ast.expressions.AndExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.BooleanValuedFunctionWrapper; import org.graylog.plugins.messageprocessor.ast.expressions.ComparisonExpression; import org.graylog.plugins.messageprocessor.ast.expressions.DoubleExpression; import org.graylog.plugins.messageprocessor.ast.expressions.EqualityExpression; @@ -164,7 +165,17 @@ public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { if (ctx.stage != null) { ruleBuilder.stage(Integer.parseInt(ctx.stage.getText())); } - ruleBuilder.when((LogicalExpression) exprs.get(ctx.condition)); + final Expression expr = exprs.get(ctx.condition); + + LogicalExpression condition; + if (expr instanceof LogicalExpression) { + condition = (LogicalExpression) expr; + } else if (expr.getType().equals(Boolean.class)) { + condition = new BooleanValuedFunctionWrapper(expr); + } else { + throw new RuntimeException("Unable to create condition, this is a bug"); + } + ruleBuilder.when(condition); ruleBuilder.then(parseContext.statements); final Rule rule = ruleBuilder.build(); parseContext.setRule(rule); diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 286be367d6ae..9798af244ebf 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -62,6 +62,21 @@ public FunctionDescriptor descriptor() { .build(); } }); + functions.put("doch", new Function() { + @Override + public Boolean evaluate(Map args, EvaluationContext context, Message message) { + return true; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("doch") + .returnType(Boolean.class) + .params(ImmutableList.of()) + .build(); + } + }); functions.put("double_valued_func", new Function() { @Override public Double evaluate(Map args, EvaluationContext context, Message message) { @@ -198,6 +213,18 @@ public void invalidArgType() throws Exception { } } + @Test + public void booleanValuedFunctionAsCondition() throws Exception { + try { + final Rule rule = parser.parseRule(ruleForTest()); + + evaluateRule(rule); + assertTrue("actions should have triggered", actionsTriggered.get()); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + @Nullable private Message evaluateRule(Rule rule) { final EvaluationContext context = new EvaluationContext(); diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/booleanValuedFunctionAsCondition.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/booleanValuedFunctionAsCondition.txt new file mode 100644 index 000000000000..6c226ae823fc --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/booleanValuedFunctionAsCondition.txt @@ -0,0 +1,5 @@ +rule "bool function as top level" +when doch() +then + trigger_test(); +end \ No newline at end of file From 790070f77f93524c71abc8c1ad52599e86445cfd Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 16:18:43 +0100 Subject: [PATCH 014/521] clean up readme --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 28a1c86ebfd6..17f270c64eeb 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,9 @@ -# Graylog ProcessorPlugin Plugin - -Welcome to your new Graylog plugin! - -Please refer to http://docs.graylog.org/en/latest/pages/plugins.html for documentation on how to write -plugins for Graylog. - +# Graylog Message Processor Plugin Getting started --------------- -This project is using Maven 3 and requires Java 7 or higher. The plugin will require Graylog 1.0.0 or higher. +This project is using Maven 3 and requires Java 8 or higher. The plugin will require Graylog 2.0.0 or higher. * Clone this repository. * Run `mvn package` to build a JAR file. From a17287e53f99af2eca3d3769fc59962844252b62 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 31 Dec 2015 17:47:07 +0100 Subject: [PATCH 015/521] improve message ref expression efficiency add type coercion functions for long, double, string, bool add set_field function to mutate message fields check for wrong number of arguments to functions relax type checks in function arguments and improve flexibility in comparison expressions --- .../messageprocessor/parser/RuleLang.g4 | 8 ++-- .../ProcessorPluginModule.java | 9 ++++ .../ast/expressions/ComparisonExpression.java | 17 +++---- .../ast/expressions/FunctionExpression.java | 10 +++-- .../ast/expressions/MessageRefExpression.java | 17 +++++-- .../ast/functions/ParameterDescriptor.java | 8 +++- .../functions/builtin/BooleanCoercion.java | 34 ++++++++++++++ .../ast/functions/builtin/DoubleCoercion.java | 42 ++++++++++++++++++ .../ast/functions/builtin/LongCoercion.java | 42 ++++++++++++++++++ .../ast/functions/builtin/SetField.java | 38 ++++++++++++++++ .../ast/functions/builtin/StringCoercion.java | 44 +++++++++++++++++++ .../parser/IncompatibleType.java | 20 +++++++++ .../messageprocessor/parser/RuleParser.java | 27 ++++++++++-- .../parser/WrongNumberOfArgs.java | 29 ++++++++++++ .../parser/RuleParserTest.java | 30 ++++++++++--- .../messageprocessor/parser/messageRef.txt | 5 +++ 16 files changed, 348 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index b454ce0e97ae..c5dd7a2207d8 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -44,6 +44,7 @@ ruleDeclaration expression : primary # PrimaryExpression + | MessageRef '.' field=expression # MessageRef | fieldSet=expression '.' field=expression # Nested | fieldSet=expression '[' expression ']' # Array | functionCall # Func @@ -55,10 +56,9 @@ expression ; primary - : '(' expression ')' # ParenExpr - | literal # LiteralPrimary - | Identifier # Identifier - | MessageRef # MessageRef + : '(' expression ')' # ParenExpr + | literal # LiteralPrimary + | Identifier # Identifier ; statement diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index 3d185f1c8d8d..61d185183291 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -4,9 +4,13 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.BooleanCoercion; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.DoubleCoercion; import org.graylog.plugins.messageprocessor.ast.functions.builtin.DropMessageFunction; import org.graylog.plugins.messageprocessor.ast.functions.builtin.HasField; import org.graylog.plugins.messageprocessor.ast.functions.builtin.InputFunction; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.LongCoercion; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.StringCoercion; import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; import org.graylog2.plugin.PluginConfigBean; @@ -28,6 +32,11 @@ protected void configure() { addRestResource(MessageProcessorRuleResource.class); // built-in functions + addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); + addMessageProcessorFunction(DoubleCoercion.NAME, DoubleCoercion.class); + addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); + addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); + addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java index 885b25bd85e3..8e6a0ab29a34 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java @@ -5,12 +5,9 @@ public class ComparisonExpression extends BinaryExpression implements LogicalExpression { private final String operator; - private final boolean numericArgs; public ComparisonExpression(Expression left, Expression right, String operator) { super(left, right); - this.numericArgs = left instanceof NumericExpression && right instanceof NumericExpression; - this.operator = operator; } @@ -26,15 +23,13 @@ public Class getType() { @Override public boolean evaluateBool(EvaluationContext context, Message message) { - if (!numericArgs) { - return false; - } - final NumericExpression numericLeft = (NumericExpression) this.left; - final NumericExpression numericRight = (NumericExpression) this.right; - if (numericLeft.isFloatingPoint() || numericRight.isFloatingPoint()) { - return compareDouble(operator, numericLeft.doubleValue(), numericRight.doubleValue()); + + final Object leftValue = this.left.evaluate(context, message); + final Object rightValue = this.right.evaluate(context, message); + if (leftValue instanceof Double || rightValue instanceof Double) { + return compareDouble(operator, (double) leftValue, (double) rightValue); } else { - return compareLong(operator, numericLeft.longValue(), numericRight.longValue()); + return compareLong(operator, (long) leftValue, (long) rightValue); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index cb9ab7f97d22..696e3cb99023 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -46,10 +46,14 @@ public Class getType() { @Override public String toString() { - String join = ""; + String argsString = ""; if (args != null) { - join = Joiner.on(", ").withKeyValueSeparator(": ").join(args); // TODO order arg names + if (args.containsKey(null)) { + argsString = args.get(null).toString(); + } else { + argsString = Joiner.on(", ").withKeyValueSeparator(": ").join(args); // TODO order arg names + } } - return descriptor.name() + "(" + join + ")"; + return descriptor.name() + "(" + argsString + ")"; } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java index 55c8e0ae6540..373671d002a2 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java @@ -4,6 +4,12 @@ import org.graylog2.plugin.Message; public class MessageRefExpression implements Expression { + private final Expression fieldExpr; + + public MessageRefExpression(Expression fieldExpr) { + this.fieldExpr = fieldExpr; + } + @Override public boolean isConstant() { return false; @@ -11,16 +17,21 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context, Message message) { - return message; + final Object fieldName = fieldExpr.evaluate(context, message); + return message.getField(fieldName.toString()); } @Override public Class getType() { - return Message.class; + return Object.class; } @Override public String toString() { - return "$message"; + return "$message." + fieldExpr.toString(); + } + + public Expression getFieldExpr() { + return fieldExpr; } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java index 5bccd06ffeeb..7d81c1857460 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java @@ -5,7 +5,7 @@ @AutoValue public abstract class ParameterDescriptor { - public abstract Class type(); + public abstract Class type(); public abstract String name(); @@ -17,9 +17,13 @@ public static ParameterDescriptor string(String name) { return builder().string(name).build(); } + public static ParameterDescriptor object(String name) { + return builder().name(name).type(Object.class).build(); + } + @AutoValue.Builder public static abstract class Builder { - public abstract Builder type(Class type); + public abstract Builder type(Class type); public abstract Builder name(String name); public abstract ParameterDescriptor build(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java new file mode 100644 index 000000000000..da9896e9b933 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java @@ -0,0 +1,34 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; + +public class BooleanCoercion implements Function { + public static final String NAME = "bool"; + + private static final String VALUE = "value"; + + @Override + public Boolean evaluate(Map args, EvaluationContext context, Message message) { + final Expression value = args.get(VALUE); + final Object evaluated = value.evaluate(context, message); + return Boolean.parseBoolean(evaluated.toString()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(object(VALUE))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java new file mode 100644 index 000000000000..3f6268583793 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java @@ -0,0 +1,42 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import com.google.common.primitives.Doubles; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.builder; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; + +public class DoubleCoercion implements Function { + + public static final String NAME = "double"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public Double evaluate(Map args, EvaluationContext context, Message message) { + final Expression value = args.get(VALUE); + final Object evaluated = value.evaluate(context, message); + return (Double) firstNonNull(Doubles.tryParse(evaluated.toString()), args.get(DEFAULT).evaluate(context, message)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Double.class) + .params(of( + object(VALUE), + builder().name(DEFAULT).type(Double.class).build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java new file mode 100644 index 000000000000..49d22a5902b2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java @@ -0,0 +1,42 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import com.google.common.primitives.Longs; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.builder; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; + +public class LongCoercion implements Function { + + public static final String NAME = "long"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public Long evaluate(Map args, EvaluationContext context, Message message) { + final Expression value = args.get(VALUE); + final Object evaluated = value.evaluate(context, message); + return (Long) firstNonNull(Longs.tryParse(evaluated.toString()), args.get(DEFAULT).evaluate(context, message)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Long.class) + .params(of( + object(VALUE), + builder().name(DEFAULT).type(Long.class).build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java new file mode 100644 index 000000000000..31ea144ec9f9 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java @@ -0,0 +1,38 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.string; + +public class SetField implements Function { + + public static final String NAME = "set_field"; + public static final String FIELD = "field"; + public static final String VALUE = "value"; + + @Override + public Void evaluate(Map args, EvaluationContext context, Message message) { + final Object field = args.get(FIELD).evaluate(context, message); + final Object value = args.get(VALUE).evaluate(context, message); + message.addField(field.toString(), value); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of(string(FIELD), + object(VALUE))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java new file mode 100644 index 000000000000..c741285d509a --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java @@ -0,0 +1,44 @@ +package org.graylog.plugins.messageprocessor.ast.functions.builtin; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.string; + +public class StringCoercion implements Function { + + public static final String NAME = "string"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public String evaluate(Map args, EvaluationContext context, Message message) { + final Expression value = args.get(VALUE); + final Object evaluated = value.evaluate(context, message); + if (evaluated instanceof String) { + return (String) evaluated; + } else { + return (String) args.get(DEFAULT).evaluate(context, message); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + object(VALUE), + string(DEFAULT) + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java new file mode 100644 index 000000000000..852f6a419fb3 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java @@ -0,0 +1,20 @@ +package org.graylog.plugins.messageprocessor.parser; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IncompatibleType extends ParseError { + private final Class expected; + private final Class actual; + + public IncompatibleType(RuleLangParser.MessageRefContext ctx, Class expected, Class actual) { + super("incompatible_type", ctx); + this.expected = expected; + this.actual = actual; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 758bc8ed67a9..25dd435c6c99 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -128,6 +128,10 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final ParameterDescriptor param = function.descriptor().params().get(0); args.put(param.name(), argExpr); } + // check for the right number of arguments to the function + if (args != null && function.descriptor().params().size() != args.size()) { + parseContext.addError(new WrongNumberOfArgs(ctx, function, args)); + } } final FunctionExpression expr = new FunctionExpression(name, args, functionRegistry.resolveOrError(name)); @@ -293,9 +297,17 @@ public void exitParenExpr(RuleLangParser.ParenExprContext ctx) { parseContext.addInnerNode(ctx); } + @Override + public void enterMessageRef(RuleLangParser.MessageRefContext ctx) { + // nested field access is ok, these are not rule variables + idIsFieldAccess = true; + } + @Override public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { - final MessageRefExpression expr = new MessageRefExpression(); + idIsFieldAccess = false; // reset for error checks + final Expression fieldExpr = exprs.get(ctx.field); + final MessageRefExpression expr = new MessageRefExpression(fieldExpr); log.info("$MSG: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -390,7 +402,8 @@ public void exitComparison(RuleLangParser.ComparisonContext ctx) { @Override public void exitEquality(RuleLangParser.EqualityContext ctx) { - checkBinaryExpression(ctx); + // TODO equality allows arbitrary types, in the future optimize by using specialized operators +// checkBinaryExpression(ctx, true); } private void checkBinaryExpression(RuleLangParser.ExpressionContext ctx) { @@ -410,12 +423,20 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final Map args = expr.getArgs(); for (ParameterDescriptor p : descriptor.params()) { final Expression argExpr = args.get(p.name()); - if (!argExpr.getType().equals(p.type())) { + if (argExpr != null && !p.type().isAssignableFrom(argExpr.getType())) { parseContext.addError(new IncompatibleArgumentType(ctx, expr, p, argExpr)); } } } + @Override + public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { + final MessageRefExpression expr = (MessageRefExpression) parseContext.expressions().get(ctx); + if (!expr.getFieldExpr().getType().equals(String.class)) { + parseContext.addError(new IncompatibleType(ctx, String.class, expr.getFieldExpr().getType())); + } + } + @Override public void enterEveryRule(ParserRuleContext ctx) { final Expression expression = parseContext.expressions().get(ctx); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java new file mode 100644 index 000000000000..f383d3af2fd2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java @@ -0,0 +1,29 @@ +package org.graylog.plugins.messageprocessor.parser; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.ast.functions.Function; + +import java.util.Map; + +public class WrongNumberOfArgs extends ParseError { + private final Function function; + private final Map args; + + public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, + Function function, + Map args) { + super("wrong_number_of_arguments", ctx); + this.function = function; + this.args = args; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected " + function.descriptor().params().size() + + " arguments but found " + args.size() + + " in call to function " + function.descriptor().name() + + positionString(); + } +} diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 9798af244ebf..d3f7bcaf07e5 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -9,6 +9,9 @@ import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.HasField; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.LongCoercion; +import org.graylog.plugins.messageprocessor.ast.functions.builtin.SetField; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog2.plugin.Message; import org.joda.time.DateTime; @@ -123,6 +126,9 @@ public FunctionDescriptor descriptor() { .build(); } }); + functions.put(LongCoercion.NAME, new LongCoercion()); + functions.put(SetField.NAME, new SetField()); + functions.put(HasField.NAME, new HasField()); functionRegistry = new FunctionRegistry(functions); } @@ -170,11 +176,8 @@ public void undeclaredFunction() throws Exception { parser.parseRule(ruleForTest()); fail("should throw error: undeclared function 'unknown'"); } catch (ParseException e) { - assertEquals(2, e.getErrors().size()); assertTrue("Should find error UndeclaredFunction", e.getErrors().stream().anyMatch(input -> input instanceof UndeclaredFunction)); - assertTrue("Should find error IncompatibleTypes", - e.getErrors().stream().anyMatch(input -> input instanceof IncompatibleTypes)); } } @@ -225,10 +228,19 @@ public void booleanValuedFunctionAsCondition() throws Exception { } } - @Nullable - private Message evaluateRule(Rule rule) { + @Test + public void messageRef() throws Exception { + final Rule rule = parser.parseRule(ruleForTest()); + Message message = new Message("hello test", "source", DateTime.now()); + message.addField("responseCode", 500); + final Message processedMsg = evaluateRule(rule, message); + + assertNotNull(processedMsg); + assertEquals("server_error", processedMsg.getField("response_category")); + } + + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(); - final Message message = new Message("hello test", "source", DateTime.now()); if (rule.when().evaluateBool(context, message)) { for (Statement statement : rule.then()) { @@ -240,6 +252,12 @@ private Message evaluateRule(Rule rule) { } } + @Nullable + private Message evaluateRule(Rule rule) { + final Message message = new Message("hello test", "source", DateTime.now()); + return evaluateRule(rule, message); + } + private String ruleForTest() { try { final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt new file mode 100644 index 000000000000..27c249cd6a14 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt @@ -0,0 +1,5 @@ +rule "message field ref" +when long(value: $message.responseCode, default: 200) >= 500 +then + set_field(field: "response_category", value: "server_error"); +end \ No newline at end of file From ea9487873c6e78ac37d9729bac031c37e17d9f75 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sun, 3 Jan 2016 18:55:00 +0100 Subject: [PATCH 016/521] allow function calls with positional arguments as specified in the descriptor cleanup: - move error classes to own packages - FunctionExpression does not need the name, it's available via function.descriptor().name() --- .../messageprocessor/parser/RuleLang.g4 | 1 + .../ast/expressions/FunctionExpression.java | 4 +- .../parser/ParseException.java | 2 + .../messageprocessor/parser/RuleParser.java | 57 +++++++++++++++---- .../IncompatibleArgumentType.java | 3 +- .../parser/{ => errors}/IncompatibleType.java | 3 +- .../{ => errors}/IncompatibleTypes.java | 3 +- .../parser/{ => errors}/ParseError.java | 2 +- .../{ => errors}/UndeclaredFunction.java | 3 +- .../{ => errors}/UndeclaredVariable.java | 3 +- .../{ => errors}/WrongNumberOfArgs.java | 14 ++--- .../parser/RuleParserTest.java | 37 ++++++++++++ .../parser/positionalArguments.txt | 5 ++ 13 files changed, 110 insertions(+), 27 deletions(-) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/IncompatibleArgumentType.java (91%) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/IncompatibleType.java (82%) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/IncompatibleTypes.java (87%) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/ParseError.java (95%) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/UndeclaredFunction.java (78%) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/UndeclaredVariable.java (81%) rename src/main/java/org/graylog/plugins/messageprocessor/parser/{ => errors}/WrongNumberOfArgs.java (66%) create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/positionalArguments.txt diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index c5dd7a2207d8..ac2861c3b821 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -73,6 +73,7 @@ functionCall arguments : Identifier ':' expression (',' Identifier ':' expression)* # NamedArgs + | expression (',' expression)+ # PositionalArgs | expression # SingleDefaultArg ; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index 696e3cb99023..ede34b380870 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -9,13 +9,11 @@ import java.util.Map; public class FunctionExpression implements Expression { - private final String name; private final Map args; private final Function function; private final FunctionDescriptor descriptor; - public FunctionExpression(String name, Map args, Function function) { - this.name = name; + public FunctionExpression(Function function, Map args) { this.args = args; this.function = function; this.descriptor = function.descriptor(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java index efb2d36b74c0..8f6d6104e4eb 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java @@ -1,5 +1,7 @@ package org.graylog.plugins.messageprocessor.parser; +import org.graylog.plugins.messageprocessor.parser.errors.ParseError; + import java.util.Set; public class ParseException extends RuntimeException { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 25dd435c6c99..fa71bf28fb9f 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -35,6 +35,13 @@ import org.graylog.plugins.messageprocessor.ast.statements.FunctionStatement; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.ast.statements.VarAssignStatement; +import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleType; +import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleTypes; +import org.graylog.plugins.messageprocessor.parser.errors.ParseError; +import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; +import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; +import org.graylog.plugins.messageprocessor.parser.errors.WrongNumberOfArgs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +50,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; public class RuleParser { @@ -84,6 +92,7 @@ private class AstBuilder extends RuleLangBaseListener { private final ParseContext parseContext; private final ParseTreeProperty> args; + private final ParseTreeProperty> argsList; private final ParseTreeProperty exprs; private final Set definedVars = Sets.newHashSet(); @@ -94,6 +103,7 @@ private class AstBuilder extends RuleLangBaseListener { public AstBuilder(ParseContext parseContext) { this.parseContext = parseContext; args = parseContext.arguments(); + argsList = parseContext.argumentLists(); exprs = parseContext.expressions(); } @@ -116,25 +126,40 @@ public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { @Override public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); - final Map args = this.args.get(ctx.arguments()); + Map args = this.args.get(ctx.arguments()); + final List positionalArgs = this.argsList.get(ctx.arguments()); final Function function = functionRegistry.resolve(name); if (function == null) { parseContext.addError(new UndeclaredFunction(ctx)); } else { // convert null key, single arg style to default named args for single arg functions - if (args != null && args.containsKey(null) && function.descriptor().params().size() == 1) { - final Expression argExpr = args.remove(null); - final ParameterDescriptor param = function.descriptor().params().get(0); - args.put(param.name(), argExpr); - } - // check for the right number of arguments to the function - if (args != null && function.descriptor().params().size() != args.size()) { - parseContext.addError(new WrongNumberOfArgs(ctx, function, args)); + if (args != null) { + if (args.containsKey(null) && function.descriptor().params().size() == 1) { + final Expression argExpr = args.remove(null); + final ParameterDescriptor param = function.descriptor().params().get(0); + args.put(param.name(), argExpr); + } + // check for the right number of arguments to the function + if (function.descriptor().params().size() != args.size()) { + parseContext.addError(new WrongNumberOfArgs(ctx, function, args.size())); + } + } else if (positionalArgs != null) { + // use descriptor to turn positional arguments into a map + args = Maps.newHashMap(); + if (positionalArgs.size() != function.descriptor().params().size()) { + parseContext.addError(new WrongNumberOfArgs(ctx, function, positionalArgs.size())); + } + int i = 0; + for (ParameterDescriptor p : function.descriptor().params()) { + final Expression argExpr = positionalArgs.get(i); + args.put(p.name(), argExpr); + i++; + } } } - final FunctionExpression expr = new FunctionExpression(name, args, functionRegistry.resolveOrError(name)); + final FunctionExpression expr = new FunctionExpression(functionRegistry.resolveOrError(name), args); log.info("FUNC: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -162,6 +187,13 @@ public void exitSingleDefaultArg(RuleLangParser.SingleDefaultArgContext ctx) { args.put(ctx, singleArg); } + @Override + public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { + List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); + expressions.addAll(ctx.expression().stream().map(exprs::get).collect(Collectors.toList())); + argsList.put(ctx, expressions); + } + @Override public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { final Rule.Builder ruleBuilder = Rule.builder(); @@ -466,6 +498,7 @@ public void exitEveryRule(ParserRuleContext ctx) { private static class ParseContext { private final ParseTreeProperty exprs = new ParseTreeProperty<>(); private final ParseTreeProperty> args = new ParseTreeProperty<>(); + private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); private Set errors = Sets.newHashSet(); // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information private Set innerNodes = new IdentityHashSet<>(); @@ -518,6 +551,10 @@ public boolean defineVar(String name, Expression expr) { public Expression getDefinedVar(String name) { return varDecls.get(name); } + + public ParseTreeProperty> argumentLists() { + return argsLists; + } } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java similarity index 91% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java index 955875b06c19..25831ac6cbb4 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleArgumentType.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java @@ -1,9 +1,10 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; public class IncompatibleArgumentType extends ParseError { private final FunctionExpression functionExpression; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java similarity index 82% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java index 852f6a419fb3..42ce4dbce311 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleType.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java @@ -1,6 +1,7 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; public class IncompatibleType extends ParseError { private final Class expected; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java similarity index 87% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java index 92336d7a2731..a92fb88e3f71 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/IncompatibleTypes.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java @@ -1,8 +1,9 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; public class IncompatibleTypes extends ParseError { private final RuleLangParser.ExpressionContext ctx; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/ParseError.java similarity index 95% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/ParseError.java index 5c12b1758a2b..3bae61442846 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseError.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/ParseError.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java similarity index 78% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java index d1e8d40d5bd8..99967392e2a6 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java @@ -1,6 +1,7 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; public class UndeclaredFunction extends ParseError { private final RuleLangParser.FunctionCallContext ctx; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java similarity index 81% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java index 3f26e693196a..3f513d6a04ca 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/UndeclaredVariable.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java @@ -1,7 +1,8 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; public class UndeclaredVariable extends ParseError { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java similarity index 66% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java index f383d3af2fd2..e97c8cd08785 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/WrongNumberOfArgs.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java @@ -1,28 +1,26 @@ -package org.graylog.plugins.messageprocessor.parser; +package org.graylog.plugins.messageprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; - -import java.util.Map; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; public class WrongNumberOfArgs extends ParseError { private final Function function; - private final Map args; + private final int argCount; public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, Function function, - Map args) { + int argCount) { super("wrong_number_of_arguments", ctx); this.function = function; - this.args = args; + this.argCount = argCount; } @JsonProperty("reason") @Override public String toString() { return "Expected " + function.descriptor().params().size() + - " arguments but found " + args.size() + + " arguments but found " + argCount + " in call to function " + function.descriptor().name() + positionString(); } diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index d3f7bcaf07e5..78cbb3b5eb26 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -13,6 +13,9 @@ import org.graylog.plugins.messageprocessor.ast.functions.builtin.LongCoercion; import org.graylog.plugins.messageprocessor.ast.functions.builtin.SetField; import org.graylog.plugins.messageprocessor.ast.statements.Statement; +import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; +import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; import org.graylog2.plugin.Message; import org.joda.time.DateTime; import org.junit.After; @@ -110,6 +113,28 @@ public FunctionDescriptor descriptor() { .build(); } }); + functions.put("concat", new Function() { + @Override + public String evaluate(Map args, EvaluationContext context, Message message) { + final Object one = args.get("one").evaluate(context, message); + final Object two = args.get("two").evaluate(context, message); + final Object three = args.get("three").evaluate(context, message); + return one.toString() + two.toString() + three.toString(); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("concat") + .returnType(String.class) + .params(ImmutableList.of( + ParameterDescriptor.string("one"), + ParameterDescriptor.object("two"), + ParameterDescriptor.object("three") + )) + .build(); + } + }); functions.put("trigger_test", new Function() { @Override public Void evaluate(Map args, EvaluationContext context, Message message) { @@ -194,6 +219,18 @@ public void singleArgFunction() throws Exception { } } + @Test + public void positionalArguments() throws Exception { + try { + final Rule rule = parser.parseRule(ruleForTest()); + evaluateRule(rule); + + assertTrue(actionsTriggered.get()); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + @Test public void inferVariableType() throws Exception { try { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/positionalArguments.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/positionalArguments.txt new file mode 100644 index 000000000000..ac834b085e21 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/positionalArguments.txt @@ -0,0 +1,5 @@ +rule "positional args" +when concat("a", 1, true) == concat(one: "a", two: 1, three: true) +then +trigger_test(); +end \ No newline at end of file From b7934be47a68560fc37155a3e8c2081591c5692e Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Tue, 5 Jan 2016 11:39:53 +0100 Subject: [PATCH 017/521] Add prerequisites for maven to pom.xml --- pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pom.xml b/pom.xml index 1a54a078ce74..dd3a82abf80b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,6 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + 3.0 + org.graylog.plugins message-processor From 00a85b65b9923bc913ba295821c56bedd1d79522 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 6 Jan 2016 18:27:59 +0100 Subject: [PATCH 018/521] move builtin function package --- .../messageprocessor/ProcessorPluginModule.java | 14 +++++++------- .../builtin => functions}/BooleanCoercion.java | 2 +- .../builtin => functions}/DoubleCoercion.java | 2 +- .../builtin => functions}/DropMessageFunction.java | 2 +- .../functions/builtin => functions}/HasField.java | 2 +- .../builtin => functions}/InputFunction.java | 2 +- .../builtin => functions}/LongCoercion.java | 2 +- .../functions/builtin => functions}/SetField.java | 2 +- .../builtin => functions}/StringCoercion.java | 2 +- .../messageprocessor/parser/RuleParserTest.java | 6 +++--- 10 files changed, 18 insertions(+), 18 deletions(-) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/BooleanCoercion.java (94%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/DoubleCoercion.java (96%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/DropMessageFunction.java (93%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/HasField.java (94%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/InputFunction.java (95%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/LongCoercion.java (96%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/SetField.java (95%) rename src/main/java/org/graylog/plugins/messageprocessor/{ast/functions/builtin => functions}/StringCoercion.java (95%) diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java index 61d185183291..cff35e294f03 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java @@ -4,13 +4,13 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.BooleanCoercion; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.DoubleCoercion; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.DropMessageFunction; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.HasField; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.InputFunction; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.LongCoercion; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.StringCoercion; +import org.graylog.plugins.messageprocessor.functions.BooleanCoercion; +import org.graylog.plugins.messageprocessor.functions.DoubleCoercion; +import org.graylog.plugins.messageprocessor.functions.DropMessageFunction; +import org.graylog.plugins.messageprocessor.functions.HasField; +import org.graylog.plugins.messageprocessor.functions.InputFunction; +import org.graylog.plugins.messageprocessor.functions.LongCoercion; +import org.graylog.plugins.messageprocessor.functions.StringCoercion; import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; import org.graylog2.plugin.PluginConfigBean; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java similarity index 94% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java index da9896e9b933..754c8b58c54d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/BooleanCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java similarity index 96% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java index 3f6268583793..a99c643350b2 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DoubleCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import com.google.common.primitives.Doubles; import org.graylog.plugins.messageprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java similarity index 93% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java index f0d5975f8939..2e6b250bf3ea 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/DropMessageFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import com.google.common.collect.ImmutableList; import org.graylog.plugins.messageprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java similarity index 94% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java index f3a7862e3744..a82228949253 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/HasField.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import com.google.common.collect.ImmutableList; import org.graylog.plugins.messageprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java similarity index 95% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java index 017885cc06a1..2866c84d7548 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/InputFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java similarity index 96% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java index 49d22a5902b2..b6cdc1a987ef 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/LongCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import com.google.common.primitives.Longs; import org.graylog.plugins.messageprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java similarity index 95% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java index 31ea144ec9f9..326bbe79ca1b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/SetField.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java similarity index 95% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java rename to src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java index c741285d509a..9359aef9f7db 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/builtin/StringCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.messageprocessor.ast.functions.builtin; +package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 78cbb3b5eb26..916e87d5432a 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -9,9 +9,9 @@ import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.HasField; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.LongCoercion; -import org.graylog.plugins.messageprocessor.ast.functions.builtin.SetField; +import org.graylog.plugins.messageprocessor.functions.HasField; +import org.graylog.plugins.messageprocessor.functions.LongCoercion; +import org.graylog.plugins.messageprocessor.functions.SetField; import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; From 17bc9749f37281c81f41f63f72085d8b5b4bd35a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 7 Jan 2016 06:16:35 +0100 Subject: [PATCH 019/521] allow quoting identifiers to contain arbitrary special characters remove unused FieldSet class --- .../messageprocessor/parser/RuleLang.g4 | 1 + .../plugins/messageprocessor/FieldSet.java | 70 ------------------- .../messageprocessor/parser/RuleParser.java | 11 +-- .../parser/RuleParserTest.java | 12 ++++ .../parser/messageRefQuotedField.txt | 5 ++ 5 files changed, 25 insertions(+), 74 deletions(-) delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index ac2861c3b821..5e9cf895153a 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -346,6 +346,7 @@ ZeroToThree Identifier : [a-zA-Z_] [a-zA-Z_0-9]* + | '`' ~['`']+ '`' ; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java b/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java deleted file mode 100644 index 539cfc4fc56d..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/FieldSet.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.graylog.plugins.messageprocessor; - -import com.google.common.collect.Maps; - -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.Set; - -public class FieldSet { - private static final FieldSet INSTANCE = new FieldSet(Collections.emptyMap()); - - private FieldSet(Map map) { - fields = map; - } - - public static FieldSet empty() { - return INSTANCE; - } - - private final Map fields; - - public FieldSet() { - this.fields = Maps.newHashMap(); - } - - public Object get(String key) { - return fields.get(key); - } - - public Object remove(String key) { - return fields.remove(key); - } - - public FieldSet put(String key, Object value) { - fields.put(key, value); - return this; - } - - public int size() { - return fields.size(); - } - - public boolean isEmpty() { - return fields.isEmpty(); - } - - @Override - public int hashCode() { - return fields.hashCode(); - } - - @Override - public boolean equals(Object o) { - return fields.equals(o); - } - - public Set> entrySet() { - return fields.entrySet(); - } - - public Set keySet() { - return fields.keySet(); - } - - public Collection values() { - return fields.values(); - } - -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index fa71bf28fb9f..b0f57eebc413 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -42,6 +42,7 @@ import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; import org.graylog.plugins.messageprocessor.parser.errors.WrongNumberOfArgs; +import org.graylog2.plugin.Tools; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -346,15 +347,17 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { - final String variableName = ctx.Identifier().getText(); - if (!idIsFieldAccess && !definedVars.contains(variableName)) { + // unquote identifier if necessary + final String identifierName = Tools.unquote(ctx.Identifier().getText(), '`'); + + if (!idIsFieldAccess && !definedVars.contains(identifierName)) { parseContext.addError(new UndeclaredVariable(ctx)); } final Expression expr; if (idIsFieldAccess) { - expr = new FieldRefExpression(variableName); + expr = new FieldRefExpression(identifierName); } else { - expr = new VarRefExpression(variableName); + expr = new VarRefExpression(identifierName); } log.info("VAR: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 916e87d5432a..1bf3054e53a3 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -13,6 +13,7 @@ import org.graylog.plugins.messageprocessor.functions.LongCoercion; import org.graylog.plugins.messageprocessor.functions.SetField; import org.graylog.plugins.messageprocessor.ast.statements.Statement; +import org.graylog.plugins.messageprocessor.functions.StringCoercion; import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; @@ -152,6 +153,7 @@ public FunctionDescriptor descriptor() { } }); functions.put(LongCoercion.NAME, new LongCoercion()); + functions.put(StringCoercion.NAME, new StringCoercion()); functions.put(SetField.NAME, new SetField()); functions.put(HasField.NAME, new HasField()); functionRegistry = new FunctionRegistry(functions); @@ -276,6 +278,16 @@ public void messageRef() throws Exception { assertEquals("server_error", processedMsg.getField("response_category")); } + @Test + public void messageRefQuotedField() throws Exception { + final Rule rule = parser.parseRule(ruleForTest()); + Message message = new Message("hello test", "source", DateTime.now()); + message.addField("@specialfieldname", "string"); + evaluateRule(rule, message); + + assertTrue(actionsTriggered.get()); + } + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(); if (rule.when().evaluateBool(context, message)) { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt new file mode 100644 index 000000000000..2f50c546d5b0 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt @@ -0,0 +1,5 @@ +rule "test" +when string($message.`@specialfieldname`, "empty") == "string" +then + trigger_test(); +end \ No newline at end of file From f4e2eca065091369ebce9268e9ff1431c0fd4345 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sun, 10 Jan 2016 16:50:32 +0100 Subject: [PATCH 020/521] allow optional paramters for functions - guard against null in BooleanValueFunctionWrapper - refactor ParamDescription builder to support more common types - new errors for parameter handling - test cases --- .../BooleanValuedFunctionWrapper.java | 5 +- .../ast/functions/ParameterDescriptor.java | 41 +++++++++++-- .../functions/DoubleCoercion.java | 4 +- .../functions/LongCoercion.java | 4 +- .../messageprocessor/parser/RuleParser.java | 51 ++++++++++++---- .../parser/errors/MissingRequiredParam.java | 28 +++++++++ .../errors/OptionalParametersMustBeNamed.java | 20 +++++++ .../parser/RuleParserTest.java | 58 ++++++++++++++++--- .../parser/optionalArguments.txt | 6 ++ .../parser/optionalParamsMustBeNamed.txt | 5 ++ 10 files changed, 193 insertions(+), 29 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalArguments.txt create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalParamsMustBeNamed.txt diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java index c6e136e8a4e6..5481eb38adff 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -15,12 +15,13 @@ public BooleanValuedFunctionWrapper(Expression expr) { @Override public boolean evaluateBool(EvaluationContext context, Message message) { - return (Boolean) expr.evaluate(context, message); + final Object value = expr.evaluate(context, message); + return value != null && (Boolean) value; } @Override public boolean isConstant() { - return false; + return expr.isConstant(); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java index 7d81c1857460..b27feb667a9a 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java @@ -9,26 +9,59 @@ public abstract class ParameterDescriptor { public abstract String name(); - public static Builder builder() { - return new AutoValue_ParameterDescriptor.Builder(); + public abstract boolean optional(); + + public static Builder param() { + return new AutoValue_ParameterDescriptor.Builder().optional(false); } public static ParameterDescriptor string(String name) { - return builder().string(name).build(); + return param().string(name).build(); } public static ParameterDescriptor object(String name) { - return builder().name(name).type(Object.class).build(); + return param().object(name).build(); + } + + public static ParameterDescriptor integer(String name) { + return param().integer(name).build(); + } + + public static ParameterDescriptor floating(String name) { + return param().floating(name).build(); + } + + public static ParameterDescriptor bool(String name) { + return param().bool(name).build(); } @AutoValue.Builder public static abstract class Builder { public abstract Builder type(Class type); public abstract Builder name(String name); + public abstract Builder optional(boolean optional); public abstract ParameterDescriptor build(); + public Builder string(String name) { return type(String.class).name(name); } + public Builder object(String name) { + return type(Object.class).name(name); + } + public Builder floating(String name) { + return type(Double.class).name(name); + } + public Builder integer(String name) { + return type(Long.class).name(name); + } + public Builder bool(String name) { + return type(Boolean.class).name(name); + } + + public Builder optional() { + return optional(true); + } + } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java index a99c643350b2..51beb04f45be 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java @@ -11,7 +11,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.builder; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; public class DoubleCoercion implements Function { @@ -35,7 +35,7 @@ public FunctionDescriptor descriptor() { .returnType(Double.class) .params(of( object(VALUE), - builder().name(DEFAULT).type(Double.class).build() + param().name(DEFAULT).type(Double.class).build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java index b6cdc1a987ef..bc16485f4d95 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java @@ -11,7 +11,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.builder; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; public class LongCoercion implements Function { @@ -35,7 +35,7 @@ public FunctionDescriptor descriptor() { .returnType(Long.class) .params(of( object(VALUE), - builder().name(DEFAULT).type(Long.class).build() + param().name(DEFAULT).type(Long.class).build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index b0f57eebc413..08e8a9dcac97 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -1,5 +1,6 @@ package org.graylog.plugins.messageprocessor.parser; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -38,6 +39,8 @@ import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleType; import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleTypes; +import org.graylog.plugins.messageprocessor.parser.errors.MissingRequiredParam; +import org.graylog.plugins.messageprocessor.parser.errors.OptionalParametersMustBeNamed; import org.graylog.plugins.messageprocessor.parser.errors.ParseError; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; @@ -134,28 +137,53 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { if (function == null) { parseContext.addError(new UndeclaredFunction(ctx)); } else { + final ImmutableList params = function.descriptor().params(); + final boolean hasOptionalParams = params.stream().anyMatch(ParameterDescriptor::optional); + // convert null key, single arg style to default named args for single arg functions if (args != null) { - if (args.containsKey(null) && function.descriptor().params().size() == 1) { + if (args.containsKey(null) && params.size() == 1) { final Expression argExpr = args.remove(null); - final ParameterDescriptor param = function.descriptor().params().get(0); + final ParameterDescriptor param = params.get(0); args.put(param.name(), argExpr); } - // check for the right number of arguments to the function - if (function.descriptor().params().size() != args.size()) { + + // check for the right number of arguments to the function if the function only has required params + if (!hasOptionalParams && params.size() != args.size()) { parseContext.addError(new WrongNumberOfArgs(ctx, function, args.size())); + } else { + // there are optional parameters, check that all required ones are present + final Map givenArguments = args; + final List missingParams = + params.stream() + .filter(p -> !p.optional()) + .map(p -> givenArguments.containsKey(p.name()) ? null : p) + .filter(p -> p != null) + .collect(Collectors.toList()); + for (ParameterDescriptor param : missingParams) { + parseContext.addError(new MissingRequiredParam(ctx, function, param)); + } + } } else if (positionalArgs != null) { // use descriptor to turn positional arguments into a map args = Maps.newHashMap(); - if (positionalArgs.size() != function.descriptor().params().size()) { + if (!hasOptionalParams && positionalArgs.size() != params.size()) { parseContext.addError(new WrongNumberOfArgs(ctx, function, positionalArgs.size())); } - int i = 0; - for (ParameterDescriptor p : function.descriptor().params()) { - final Expression argExpr = positionalArgs.get(i); - args.put(p.name(), argExpr); - i++; + if (hasOptionalParams) { + parseContext.addError(new OptionalParametersMustBeNamed(ctx, function)); + } else { + int i = 0; + for (ParameterDescriptor p : params) { + if (i >= params.size()) { + // avoid index out of bounds, we've added an error anyway + break; + } + final Expression argExpr = positionalArgs.get(i); + args.put(p.name(), argExpr); + i++; + } } } } @@ -210,7 +238,8 @@ public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { } else if (expr.getType().equals(Boolean.class)) { condition = new BooleanValuedFunctionWrapper(expr); } else { - throw new RuntimeException("Unable to create condition, this is a bug"); + condition = new BooleanExpression(false); + log.debug("Unable to create condition, replacing with 'false'"); } ruleBuilder.when(condition); ruleBuilder.then(parseContext.statements); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java new file mode 100644 index 000000000000..5c30b2e48324 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java @@ -0,0 +1,28 @@ +package org.graylog.plugins.messageprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; + +public class MissingRequiredParam extends ParseError { + private final Function function; + private final ParameterDescriptor param; + + public MissingRequiredParam(RuleLangParser.FunctionCallContext ctx, + Function function, + ParameterDescriptor param) { + super("missing_required_param", ctx); + this.function = function; + this.param = param; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Missing required parameter " + param.name() + + " of type " + param.type().getSimpleName() + + " in call to function " + function.descriptor().name() + + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java new file mode 100644 index 000000000000..769e17a1124c --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java @@ -0,0 +1,20 @@ +package org.graylog.plugins.messageprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.parser.RuleLangParser; + +public class OptionalParametersMustBeNamed extends ParseError { + private final Function function; + + public OptionalParametersMustBeNamed(RuleLangParser.FunctionCallContext ctx, Function function) { + super("must_name_optional_params", ctx); + this.function = function; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Function " + function.descriptor().name() + " has optional parameters, must use named parameters to call" + positionString(); + } +} diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 1bf3054e53a3..2033488f4136 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.parser; import com.google.common.base.Charsets; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; @@ -9,12 +8,13 @@ import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.functions.HasField; import org.graylog.plugins.messageprocessor.functions.LongCoercion; import org.graylog.plugins.messageprocessor.functions.SetField; -import org.graylog.plugins.messageprocessor.ast.statements.Statement; import org.graylog.plugins.messageprocessor.functions.StringCoercion; import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.messageprocessor.parser.errors.OptionalParametersMustBeNamed; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; import org.graylog2.plugin.Message; @@ -36,6 +36,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -65,7 +67,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("nein") .returnType(Boolean.class) - .params(ImmutableList.of()) + .params(of()) .build(); } }); @@ -80,7 +82,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("doch") .returnType(Boolean.class) - .params(ImmutableList.of()) + .params(of()) .build(); } }); @@ -95,7 +97,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("double_valued_func") .returnType(Double.class) - .params(ImmutableList.of()) + .params(of()) .build(); } }); @@ -110,7 +112,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("one_arg") .returnType(String.class) - .params(ImmutableList.of(ParameterDescriptor.string("one"))) + .params(of(ParameterDescriptor.string("one"))) .build(); } }); @@ -128,7 +130,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("concat") .returnType(String.class) - .params(ImmutableList.of( + .params(of( ParameterDescriptor.string("one"), ParameterDescriptor.object("two"), ParameterDescriptor.object("three") @@ -148,7 +150,27 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("trigger_test") .returnType(Void.class) - .params(ImmutableList.of()) + .params(of()) + .build(); + } + }); + functions.put("optional", new Function() { + @Override + public Boolean evaluate(Map args, EvaluationContext context, Message message) { + return true; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("optional") + .returnType(Boolean.class) + .params(of( + ParameterDescriptor.bool("a"), + ParameterDescriptor.string("b"), + param().floating("c").optional().build(), + ParameterDescriptor.integer("d") + )) .build(); } }); @@ -288,6 +310,26 @@ public void messageRefQuotedField() throws Exception { assertTrue(actionsTriggered.get()); } + @Test + public void optionalArguments() throws Exception { + final Rule rule = parser.parseRule(ruleForTest()); + + Message message = new Message("hello test", "source", DateTime.now()); + evaluateRule(rule, message); + assertTrue(actionsTriggered.get()); + } + + @Test + public void optionalParamsMustBeNamed() throws Exception { + try { + parser.parseRule(ruleForTest()); + } catch (ParseException e) { + assertEquals(1, e.getErrors().stream().count()); + assertTrue(e.getErrors().stream().allMatch(error -> error instanceof OptionalParametersMustBeNamed)); + } + + } + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(); if (rule.when().evaluateBool(context, message)) { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalArguments.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalArguments.txt new file mode 100644 index 000000000000..913b30f7bc0d --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalArguments.txt @@ -0,0 +1,6 @@ +rule "optional function arguments" +when + optional(d: 3, a: true, b: "string") +then + trigger_test(); +end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalParamsMustBeNamed.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalParamsMustBeNamed.txt new file mode 100644 index 000000000000..c0eb68be4bf7 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalParamsMustBeNamed.txt @@ -0,0 +1,5 @@ +rule "optionalParamsMustBeNamed" +when + optional(false, "string", 3) +then +end \ No newline at end of file From d9dc80874c86d8f4995c9d15f119c785273603fc Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 13 Jan 2016 15:40:45 +0100 Subject: [PATCH 021/521] remove '$' character from message field access - test object field access parsing/node --- .../messageprocessor/parser/RuleLang.g4 | 2 +- .../parser/RuleParserTest.java | 37 +++++++++++++++++++ .../messageprocessor/parser/messageRef.txt | 2 +- .../parser/messageRefQuotedField.txt | 2 +- .../parser/typedFieldAccess.txt | 6 +++ 5 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/typedFieldAccess.txt diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index 5e9cf895153a..288e0130fa79 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -98,7 +98,7 @@ When: W H E N; Then: T H E N; End: E N D; Let: L E T; -MessageRef: '$message'; +MessageRef: 'message'; Boolean : 'true'|'false' diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index 2033488f4136..e2801a6eab4b 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -174,6 +174,21 @@ public FunctionDescriptor descriptor() { .build(); } }); + functions.put("customObject", new Function() { + @Override + public CustomObject evaluate(Map args, EvaluationContext context, Message message) { + return new CustomObject((String) args.get("default").evaluate(context, message)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("customObject") + .returnType(CustomObject.class) + .params(of(ParameterDescriptor.string("default"))) + .build(); + } + }); functions.put(LongCoercion.NAME, new LongCoercion()); functions.put(StringCoercion.NAME, new StringCoercion()); functions.put(SetField.NAME, new SetField()); @@ -330,6 +345,17 @@ public void optionalParamsMustBeNamed() throws Exception { } + @Test + public void typedFieldAccess() throws Exception { + try { + final Rule rule = parser.parseRule(ruleForTest()); + evaluateRule(rule, new Message("hallo", "test", DateTime.now())); + assertTrue("condition should be true", actionsTriggered.get()); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(); if (rule.when().evaluateBool(context, message)) { @@ -360,4 +386,15 @@ private String ruleForTest() { } } + public static class CustomObject { + private final String id; + + public CustomObject(String id) { + this.id = id; + } + + public String getId() { + return id; + } + } } \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt index 27c249cd6a14..8285453124d2 100644 --- a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt @@ -1,5 +1,5 @@ rule "message field ref" -when long(value: $message.responseCode, default: 200) >= 500 +when long(value: message.responseCode, default: 200) >= 500 then set_field(field: "response_category", value: "server_error"); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt index 2f50c546d5b0..7e1067903ba6 100644 --- a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt @@ -1,5 +1,5 @@ rule "test" -when string($message.`@specialfieldname`, "empty") == "string" +when string(message.`@specialfieldname`, "empty") == "string" then trigger_test(); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/typedFieldAccess.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/typedFieldAccess.txt new file mode 100644 index 000000000000..727d06057b33 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/typedFieldAccess.txt @@ -0,0 +1,6 @@ +rule "typed field access" +when + long(customObject("1").id, 0) < 2 +then + trigger_test(); +end \ No newline at end of file From 8ac6659d0eecb1bb0cf053aececf52ae40813833 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sun, 17 Jan 2016 15:06:31 +0100 Subject: [PATCH 022/521] refactor function arguments - argument Map is a proper class now - supports more elegant access to argument expressions, with evaluate helper to reduce boilerplate - single place to handle expression value caching (per rule invocation) - fix tests - simplify single argument handling in parser --- pom.xml | 9 ++ .../messageprocessor/parser/RuleLang.g4 | 3 +- .../messageprocessor/EvaluationContext.java | 9 +- .../ast/expressions/AndExpression.java | 9 +- .../ast/expressions/BooleanExpression.java | 5 +- .../BooleanValuedFunctionWrapper.java | 9 +- .../ast/expressions/ComparisonExpression.java | 11 ++- .../ast/expressions/DoubleExpression.java | 3 +- .../ast/expressions/EqualityExpression.java | 9 +- .../ast/expressions/Expression.java | 3 +- .../expressions/FieldAccessExpression.java | 7 +- .../ast/expressions/FieldRefExpression.java | 3 +- .../ast/expressions/FunctionExpression.java | 22 ++--- .../ast/expressions/LogicalExpression.java | 3 +- .../ast/expressions/LongExpression.java | 3 +- .../ast/expressions/MessageRefExpression.java | 7 +- .../ast/expressions/NotExpression.java | 9 +- .../ast/expressions/OrExpression.java | 9 +- .../ast/expressions/StringExpression.java | 3 +- .../ast/expressions/VarRefExpression.java | 3 +- .../ast/functions/Function.java | 8 +- .../ast/functions/FunctionArgs.java | 47 ++++++++++ .../ast/statements/FunctionStatement.java | 5 +- .../ast/statements/Statement.java | 3 +- .../ast/statements/VarAssignStatement.java | 5 +- .../functions/BooleanCoercion.java | 10 +-- .../functions/DoubleCoercion.java | 16 ++-- .../functions/DropMessageFunction.java | 20 +++-- .../messageprocessor/functions/HasField.java | 10 +-- .../functions/InputFunction.java | 10 +-- .../functions/LongCoercion.java | 24 +++--- .../messageprocessor/functions/SetField.java | 15 ++-- .../functions/StringCoercion.java | 16 ++-- .../messageprocessor/parser/RuleParser.java | 86 ++++++++++++------- .../processors/NaiveRuleProcessor.java | 6 +- .../parser/RuleParserTest.java | 34 ++++---- 36 files changed, 251 insertions(+), 203 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java diff --git a/pom.xml b/pom.xml index dd3a82abf80b..dbd12f4c036c 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,11 @@ junit 4.12 + + org.jooq + jool + 0.9.9 + org.slf4j @@ -113,6 +118,10 @@ org.antlr antlr4-runtime + + org.jooq + jool + junit junit diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index 288e0130fa79..1c10ca64eec7 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -73,8 +73,7 @@ functionCall arguments : Identifier ':' expression (',' Identifier ':' expression)* # NamedArgs - | expression (',' expression)+ # PositionalArgs - | expression # SingleDefaultArg + | expression (',' expression)* # PositionalArgs ; literal diff --git a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java index eb2d112c6621..1df0087201ab 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java @@ -1,14 +1,17 @@ package org.graylog.plugins.messageprocessor; import com.google.common.collect.Maps; +import org.graylog2.plugin.Message; import java.util.Map; public class EvaluationContext { + private final Message message; private Map ruleVars; - public EvaluationContext() { + public EvaluationContext(Message message) { + this.message = message; ruleVars = Maps.newHashMap(); } @@ -16,6 +19,10 @@ public void define(String identifier, Class type, Object value) { ruleVars.put(identifier, new TypedValue(type, value)); } + public Message currentMessage() { + return message; + } + public TypedValue get(String identifier) { return ruleVars.get(identifier); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java index b2930634bb47..aeda5ad95ed0 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class AndExpression extends BinaryExpression implements LogicalExpression { public AndExpression(LogicalExpression left, @@ -10,13 +9,13 @@ public AndExpression(LogicalExpression left, } @Override - public Object evaluate(EvaluationContext context, Message message) { - return evaluateBool(context, message); + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { - return ((LogicalExpression)left).evaluateBool(context, message) && ((LogicalExpression)right).evaluateBool(context, message); + public boolean evaluateBool(EvaluationContext context) { + return ((LogicalExpression)left).evaluateBool(context) && ((LogicalExpression)right).evaluateBool(context); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java index 363a17bfe9bb..f73f501f8a66 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class BooleanExpression extends ConstantExpression implements LogicalExpression { private final boolean value; @@ -12,13 +11,13 @@ public BooleanExpression(boolean value) { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Object evaluate(EvaluationContext context) { return value; } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { + public boolean evaluateBool(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java index 5481eb38adff..a6c6818367d0 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class BooleanValuedFunctionWrapper implements LogicalExpression { private final Expression expr; @@ -14,8 +13,8 @@ public BooleanValuedFunctionWrapper(Expression expr) { } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { - final Object value = expr.evaluate(context, message); + public boolean evaluateBool(EvaluationContext context) { + final Object value = expr.evaluate(context); return value != null && (Boolean) value; } @@ -25,8 +24,8 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context, Message message) { - return evaluateBool(context, message); + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java index 8e6a0ab29a34..df6061082d3d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class ComparisonExpression extends BinaryExpression implements LogicalExpression { private final String operator; @@ -12,8 +11,8 @@ public ComparisonExpression(Expression left, Expression right, String operator) } @Override - public Object evaluate(EvaluationContext context, Message message) { - return evaluateBool(context, message); + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); } @Override @@ -22,10 +21,10 @@ public Class getType() { } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { + public boolean evaluateBool(EvaluationContext context) { - final Object leftValue = this.left.evaluate(context, message); - final Object rightValue = this.right.evaluate(context, message); + final Object leftValue = this.left.evaluate(context); + final Object rightValue = this.right.evaluate(context); if (leftValue instanceof Double || rightValue instanceof Double) { return compareDouble(operator, (double) leftValue, (double) rightValue); } else { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java index c75ea007f142..97dc29c87745 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class DoubleExpression extends ConstantExpression implements NumericExpression { private final double value; @@ -12,7 +11,7 @@ public DoubleExpression(double value) { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Object evaluate(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java index aeb4c9dc0c04..16b483f6cea4 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class EqualityExpression extends BinaryExpression implements LogicalExpression { private final boolean checkEquality; @@ -12,8 +11,8 @@ public EqualityExpression(Expression left, Expression right, boolean checkEquali } @Override - public Object evaluate(EvaluationContext context, Message message) { - return evaluateBool(context, message); + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); } @Override @@ -22,8 +21,8 @@ public Class getType() { } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { - final boolean equals = left.evaluate(context, message).equals(right.evaluate(context, message)); + public boolean evaluateBool(EvaluationContext context) { + final boolean equals = left.evaluate(context).equals(right.evaluate(context)); if (checkEquality) { return equals; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java index ba6b8428c7ad..e09b64317b4d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java @@ -1,13 +1,12 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public interface Expression { boolean isConstant(); - Object evaluate(EvaluationContext context, Message message); + Object evaluate(EvaluationContext context); Class getType(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java index 57450298a09d..36de1741e8f9 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java @@ -2,7 +2,6 @@ import org.apache.commons.beanutils.PropertyUtils; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,9 +24,9 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context, Message message) { - final Object bean = this.object.evaluate(context, message); - final String fieldName = field.evaluate(context, message).toString(); + public Object evaluate(EvaluationContext context) { + final Object bean = this.object.evaluate(context); + final String fieldName = field.evaluate(context).toString(); try { final Object property = PropertyUtils.getProperty(bean, fieldName); log.debug("[field access] property {} of bean {}: {}", fieldName, bean.getClass().getTypeName(), property); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java index b60f00554c5f..93b5c4a2b84f 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class FieldRefExpression implements Expression { private final String variableName; @@ -16,7 +15,7 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Object evaluate(EvaluationContext context) { return variableName; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index ede34b380870..8c7de9c3b47d 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -3,17 +3,15 @@ import com.google.common.base.Joiner; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; - -import java.util.Map; public class FunctionExpression implements Expression { - private final Map args; + private final FunctionArgs args; private final Function function; private final FunctionDescriptor descriptor; - public FunctionExpression(Function function, Map args) { + public FunctionExpression(Function function, FunctionArgs args) { this.args = args; this.function = function; this.descriptor = function.descriptor(); @@ -23,7 +21,7 @@ public Function getFunction() { return function; } - public Map getArgs() { + public FunctionArgs getArgs() { return args; } @@ -33,8 +31,8 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context, Message message) { - return descriptor.returnType().cast(function.evaluate(args, context, message)); + public Object evaluate(EvaluationContext context) { + return descriptor.returnType().cast(function.evaluate(args, context)); } @Override @@ -46,12 +44,8 @@ public Class getType() { public String toString() { String argsString = ""; if (args != null) { - if (args.containsKey(null)) { - argsString = args.get(null).toString(); - } else { - argsString = Joiner.on(", ").withKeyValueSeparator(": ").join(args); // TODO order arg names - } + argsString = Joiner.on(", ").withKeyValueSeparator(": ").join(args.getArgs()); // TODO order arg names } - return descriptor.name() + "(" + argsString + ")"; + return descriptor.name() + "(" + argsString + ")"; } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java index 75becef2c789..b92e0098dd2e 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java @@ -1,9 +1,8 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public interface LogicalExpression extends Expression { - boolean evaluateBool(EvaluationContext context, Message message); + boolean evaluateBool(EvaluationContext context); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java index a44d58bd4d6a..c260ccb1563f 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class LongExpression extends ConstantExpression implements NumericExpression { private final long value; @@ -12,7 +11,7 @@ public LongExpression(long value) { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Object evaluate(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java index 373671d002a2..b4197a2dfdcb 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class MessageRefExpression implements Expression { private final Expression fieldExpr; @@ -16,9 +15,9 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context, Message message) { - final Object fieldName = fieldExpr.evaluate(context, message); - return message.getField(fieldName.toString()); + public Object evaluate(EvaluationContext context) { + final Object fieldName = fieldExpr.evaluate(context); + return context.currentMessage().getField(fieldName.toString()); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java index 294d0d7a6221..1e4eff5e4a1b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class NotExpression extends UnaryExpression implements LogicalExpression { public NotExpression(LogicalExpression right) { @@ -9,13 +8,13 @@ public NotExpression(LogicalExpression right) { } @Override - public Object evaluate(EvaluationContext context, Message message) { - return !evaluateBool(context, message); + public Object evaluate(EvaluationContext context) { + return !evaluateBool(context); } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { - return !((LogicalExpression)right).evaluateBool(context, message); + public boolean evaluateBool(EvaluationContext context) { + return !((LogicalExpression)right).evaluateBool(context); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java index 42d1e0e546fe..72806a2cb98a 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class OrExpression extends BinaryExpression implements LogicalExpression { public OrExpression(LogicalExpression left, @@ -10,13 +9,13 @@ public OrExpression(LogicalExpression left, } @Override - public Object evaluate(EvaluationContext context, Message message) { - return evaluateBool(context, message); + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); } @Override - public boolean evaluateBool(EvaluationContext context, Message message) { - return ((LogicalExpression)left).evaluateBool(context, message) || ((LogicalExpression)right).evaluateBool(context, message); + public boolean evaluateBool(EvaluationContext context) { + return ((LogicalExpression)left).evaluateBool(context) || ((LogicalExpression)right).evaluateBool(context); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java index 6022b2a8219a..a79248f31983 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public class StringExpression extends ConstantExpression { @@ -13,7 +12,7 @@ public StringExpression(String value) { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Object evaluate(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java index 63d04539fc0e..f3d17d6eb447 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java @@ -1,7 +1,6 @@ package org.graylog.plugins.messageprocessor.ast.expressions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +19,7 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context, Message message) { + public Object evaluate(EvaluationContext context) { final EvaluationContext.TypedValue typedValue = context.get(identifier); if (typedValue != null) { return typedValue.getValue(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java index 92bc0552deae..584249ab3ef6 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java @@ -2,16 +2,12 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog2.plugin.Message; - -import java.util.Map; public interface Function { Function ERROR_FUNCTION = new Function() { @Override - public Void evaluate(Map args, EvaluationContext context, Message message) { + public Void evaluate(FunctionArgs args, EvaluationContext context) { return null; } @@ -25,7 +21,7 @@ public FunctionDescriptor descriptor() { } }; - T evaluate(Map args, EvaluationContext context, Message message); + T evaluate(FunctionArgs args, EvaluationContext context); FunctionDescriptor descriptor(); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java new file mode 100644 index 000000000000..4aae4b0afdd7 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java @@ -0,0 +1,47 @@ +package org.graylog.plugins.messageprocessor.ast.functions; + +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.expressions.Expression; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class FunctionArgs { + + @Nonnull + private final Map args; + + public FunctionArgs(Map args) { + this.args = firstNonNull(args, Collections.emptyMap()); + } + + @Nonnull + public Map getArgs() { + return args; + } + + @Nonnull + public Optional evaluated(String name, EvaluationContext context, Class argumentType) { + final Expression valueExpr = expression(name); + if (valueExpr == null) { + return Optional.empty(); + } + final Object value = valueExpr.evaluate(context); + return Optional.of(argumentType.cast(value)); + } + + public boolean isPresent(String key) { + return args.containsKey(key); + } + + @Nullable + public Expression expression(String key) { + return args.get(key); + } + +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java index 9f74a6f7551e..40ff9b093739 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java @@ -2,7 +2,6 @@ import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog2.plugin.Message; public class FunctionStatement implements Statement { private final Expression functionExpression; @@ -12,8 +11,8 @@ public FunctionStatement(Expression functionExpression) { } @Override - public Object evaluate(EvaluationContext context, Message message) { - return functionExpression.evaluate(context, message); + public Object evaluate(EvaluationContext context) { + return functionExpression.evaluate(context); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java index 251993474f39..39443886e176 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java @@ -1,10 +1,9 @@ package org.graylog.plugins.messageprocessor.ast.statements; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog2.plugin.Message; public interface Statement { // TODO should this have a return value at all? - Object evaluate(EvaluationContext context, Message message); + Object evaluate(EvaluationContext context); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java index 20ceb64c798d..62570bad6e94 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java @@ -2,7 +2,6 @@ import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog2.plugin.Message; public class VarAssignStatement implements Statement { private final String name; @@ -14,8 +13,8 @@ public VarAssignStatement(String name, Expression expr) { } @Override - public Void evaluate(EvaluationContext context, Message message) { - final Object result = expr.evaluate(context, message); + public Void evaluate(EvaluationContext context) { + final Object result = expr.evaluate(context); context.define(name, expr.getType(), result); return null; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java index 754c8b58c54d..2767c54308c8 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java @@ -1,12 +1,9 @@ package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; - -import java.util.Map; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; @@ -17,9 +14,8 @@ public class BooleanCoercion implements Function { private static final String VALUE = "value"; @Override - public Boolean evaluate(Map args, EvaluationContext context, Message message) { - final Expression value = args.get(VALUE); - final Object evaluated = value.evaluate(context, message); + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse("false"); return Boolean.parseBoolean(evaluated.toString()); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java index 51beb04f45be..86acb03bee37 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java @@ -2,17 +2,14 @@ import com.google.common.primitives.Doubles; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; - -import java.util.Map; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; public class DoubleCoercion implements Function { @@ -22,10 +19,9 @@ public class DoubleCoercion implements Function { private static final String DEFAULT = "default"; @Override - public Double evaluate(Map args, EvaluationContext context, Message message) { - final Expression value = args.get(VALUE); - final Object evaluated = value.evaluate(context, message); - return (Double) firstNonNull(Doubles.tryParse(evaluated.toString()), args.get(DEFAULT).evaluate(context, message)); + public Double evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); + return (Double) firstNonNull(Doubles.tryParse(evaluated.toString()), args.evaluated(DEFAULT, context, Double.class).orElse(0d)); } @Override @@ -35,7 +31,7 @@ public FunctionDescriptor descriptor() { .returnType(Double.class) .params(of( object(VALUE), - param().name(DEFAULT).type(Double.class).build() + param().optional().name(DEFAULT).type(Double.class).build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java index 2e6b250bf3ea..5d169f5ab7b8 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java @@ -2,20 +2,28 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog2.plugin.Message; -import java.util.Map; +import java.util.Optional; + +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; public class DropMessageFunction implements Function { public static final String NAME = "drop_message"; @Override - public Void evaluate(Map args, EvaluationContext context, Message message) { - message.setFilterOut(true); + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Optional message; + if (args.isPresent("message")) { + message = args.evaluated("message", context, Message.class); + } else { + message = Optional.of(context.currentMessage()); + } + message.get().setFilterOut(true); return null; } @@ -25,7 +33,9 @@ public FunctionDescriptor descriptor() { .name(NAME) .pure(true) .returnType(Void.class) - .params(ImmutableList.of()) + .params(ImmutableList.of( + param().type(Message.class).optional().name("message").build() + )) .build(); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java index a82228949253..349ba744adcf 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java @@ -2,21 +2,21 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog2.plugin.Message; -import java.util.Map; +import java.util.Optional; public class HasField implements Function { public static final String NAME = "has_field"; @Override - public Boolean evaluate(Map args, EvaluationContext context, Message message) { - return message.hasField((String) args.get("field").evaluate(context, message)); + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Optional field = args.evaluated("field", context, String.class); + return context.currentMessage().hasField(field.orElse(null)); } @Override diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java index 2866c84d7548..65c968d49643 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java @@ -1,17 +1,15 @@ package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; import org.graylog2.plugin.IOState; -import org.graylog2.plugin.Message; import org.graylog2.plugin.inputs.MessageInput; import org.graylog2.shared.inputs.InputRegistry; import javax.inject.Inject; -import java.util.Map; import static com.google.common.collect.ImmutableList.of; @@ -27,9 +25,9 @@ public InputFunction(InputRegistry inputRegistry) { } @Override - public MessageInput evaluate(Map args, EvaluationContext context, Message message) { - final Object id = args.get("id").evaluate(context, message); - final IOState inputState = inputRegistry.getInputState(id.toString()); + public MessageInput evaluate(FunctionArgs args, EvaluationContext context) { + final String id = args.evaluated("id", context, String.class).orElse(""); + final IOState inputState = inputRegistry.getInputState(id); return inputState.getStoppable(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java index bc16485f4d95..a8670f930abf 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java @@ -1,18 +1,15 @@ package org.graylog.plugins.messageprocessor.functions; -import com.google.common.primitives.Longs; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; - -import java.util.Map; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; +import static com.google.common.primitives.Longs.tryParse; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; public class LongCoercion implements Function { @@ -22,11 +19,12 @@ public class LongCoercion implements Function { private static final String DEFAULT = "default"; @Override - public Long evaluate(Map args, EvaluationContext context, Message message) { - final Expression value = args.get(VALUE); - final Object evaluated = value.evaluate(context, message); - return (Long) firstNonNull(Longs.tryParse(evaluated.toString()), args.get(DEFAULT).evaluate(context, message)); - } + public Long evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); + final Long defaultValue = args.evaluated(DEFAULT, context, Long.class).orElse(0L); + + return firstNonNull(tryParse(evaluated.toString()), defaultValue); + } @Override public FunctionDescriptor descriptor() { @@ -35,8 +33,8 @@ public FunctionDescriptor descriptor() { .returnType(Long.class) .params(of( object(VALUE), - param().name(DEFAULT).type(Long.class).build() - )) + param().optional().integer(DEFAULT).build() + )) .build(); } } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java index 326bbe79ca1b..3db64c382175 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java @@ -1,12 +1,11 @@ package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; -import java.util.Map; +import java.util.Optional; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; @@ -19,10 +18,12 @@ public class SetField implements Function { public static final String VALUE = "value"; @Override - public Void evaluate(Map args, EvaluationContext context, Message message) { - final Object field = args.get(FIELD).evaluate(context, message); - final Object value = args.get(VALUE).evaluate(context, message); - message.addField(field.toString(), value); + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Optional field = args.evaluated(FIELD, context, Object.class); + final Optional value = args.evaluated(VALUE, context, Object.class); + + context.currentMessage().addField(field.get().toString(), value.get()); + return null; } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java index 9359aef9f7db..ec44f73f7c3c 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java @@ -1,16 +1,13 @@ package org.graylog.plugins.messageprocessor.functions; import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; - -import java.util.Map; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; public class StringCoercion implements Function { @@ -20,13 +17,12 @@ public class StringCoercion implements Function { private static final String DEFAULT = "default"; @Override - public String evaluate(Map args, EvaluationContext context, Message message) { - final Expression value = args.get(VALUE); - final Object evaluated = value.evaluate(context, message); + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); if (evaluated instanceof String) { return (String) evaluated; } else { - return (String) args.get(DEFAULT).evaluate(context, message); + return args.evaluated(DEFAULT, context, String.class).orElse(""); } } @@ -37,7 +33,7 @@ public FunctionDescriptor descriptor() { .returnType(String.class) .params(of( object(VALUE), - string(DEFAULT) + param().optional().string(DEFAULT).build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 08e8a9dcac97..664ec4a9c335 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -31,6 +31,7 @@ import org.graylog.plugins.messageprocessor.ast.expressions.StringExpression; import org.graylog.plugins.messageprocessor.ast.expressions.VarRefExpression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.messageprocessor.ast.statements.FunctionStatement; @@ -50,7 +51,6 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -130,7 +130,7 @@ public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { @Override public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); - Map args = this.args.get(ctx.arguments()); + Map argsMap = this.args.get(ctx.arguments()); final List positionalArgs = this.argsList.get(ctx.arguments()); final Function function = functionRegistry.resolve(name); @@ -140,20 +140,13 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final ImmutableList params = function.descriptor().params(); final boolean hasOptionalParams = params.stream().anyMatch(ParameterDescriptor::optional); - // convert null key, single arg style to default named args for single arg functions - if (args != null) { - if (args.containsKey(null) && params.size() == 1) { - final Expression argExpr = args.remove(null); - final ParameterDescriptor param = params.get(0); - args.put(param.name(), argExpr); - } - + if (argsMap != null) { // check for the right number of arguments to the function if the function only has required params - if (!hasOptionalParams && params.size() != args.size()) { - parseContext.addError(new WrongNumberOfArgs(ctx, function, args.size())); + if (!hasOptionalParams && params.size() != argsMap.size()) { + parseContext.addError(new WrongNumberOfArgs(ctx, function, argsMap.size())); } else { // there are optional parameters, check that all required ones are present - final Map givenArguments = args; + final Map givenArguments = argsMap; final List missingParams = params.stream() .filter(p -> !p.optional()) @@ -163,32 +156,57 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { for (ParameterDescriptor param : missingParams) { parseContext.addError(new MissingRequiredParam(ctx, function, param)); } - } } else if (positionalArgs != null) { // use descriptor to turn positional arguments into a map - args = Maps.newHashMap(); + argsMap = Maps.newHashMap(); + // if we only have required parameters and the number doesn't match, complain if (!hasOptionalParams && positionalArgs.size() != params.size()) { parseContext.addError(new WrongNumberOfArgs(ctx, function, positionalArgs.size())); } + // if optional parameters precede any required ones, the function must used named parameters + boolean hasError = false; if (hasOptionalParams) { - parseContext.addError(new OptionalParametersMustBeNamed(ctx, function)); - } else { + // find the index of the first optional parameter + // then check if any non-optional come after it, if so, complain + int firstOptional = Integer.MAX_VALUE; + boolean requiredAfterOptional = false; + int i = 0; + for (ParameterDescriptor param : params) { + i++; + if (param.optional()) { + firstOptional = Math.min(firstOptional, i); + } else { + if (i > firstOptional) { + requiredAfterOptional = true; + } + } + } + if (requiredAfterOptional) { + parseContext.addError(new OptionalParametersMustBeNamed(ctx, function)); + hasError = true; + } + } + + if (!hasError) { + // only try to assign params if we didn't encounter a problem with position optional params above int i = 0; for (ParameterDescriptor p : params) { - if (i >= params.size()) { + if (i >= positionalArgs.size()) { // avoid index out of bounds, we've added an error anyway + // the remaining parameters were optional, so we can safely skip them break; } final Expression argExpr = positionalArgs.get(i); - args.put(p.name(), argExpr); + argsMap.put(p.name(), argExpr); i++; } } } } - final FunctionExpression expr = new FunctionExpression(functionRegistry.resolveOrError(name), args); + final FunctionExpression expr = new FunctionExpression(functionRegistry.resolveOrError(name), + new FunctionArgs(argsMap)); log.info("FUNC: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -206,15 +224,15 @@ public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { args.put(ctx, argMap); } - @Override - public void exitSingleDefaultArg(RuleLangParser.SingleDefaultArgContext ctx) { - final Expression expr = exprs.get(ctx.expression()); - final HashMap singleArg = Maps.newHashMap(); - // null key means to use the single declared argument for this function, it's syntactic sugar - // this gets validated and expanded in a later parsing stage - singleArg.put(null, expr); - args.put(ctx, singleArg); - } +// @Override +// public void exitSingleDefaultArg(RuleLangParser.SingleDefaultArgContext ctx) { +// final Expression expr = exprs.get(ctx.expression()); +// final HashMap singleArg = Maps.newHashMap(); +// // null key means to use the single declared argument for this function, it's syntactic sugar +// // this gets validated and expanded in a later parsing stage +// singleArg.put(null, expr); +// args.put(ctx, singleArg); +// } @Override public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { @@ -260,7 +278,7 @@ public void exitNested(RuleLangParser.NestedContext ctx) { final Expression object = exprs.get(ctx.fieldSet); final Expression field = exprs.get(ctx.field); final FieldAccessExpression expr = new FieldAccessExpression(object, field); - log.info("NOT: ctx {} => {}", ctx, expr); + log.info("FIELDACCESS: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -333,7 +351,7 @@ public void exitChar(RuleLangParser.CharContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { final String text = ctx.getText(); - final StringExpression expr = new StringExpression(text.substring(1, text.length()-1)); + final StringExpression expr = new StringExpression(text.substring(1, text.length() - 1)); log.info("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -440,6 +458,7 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { private class TypeChecker extends RuleLangBaseListener { private final ParseContext parseContext; StringBuffer sb = new StringBuffer(); + public TypeChecker(ParseContext parseContext) { this.parseContext = parseContext; } @@ -484,9 +503,9 @@ private void checkBinaryExpression(RuleLangParser.ExpressionContext ctx) { public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final FunctionExpression expr = (FunctionExpression) parseContext.expressions().get(ctx); final FunctionDescriptor descriptor = expr.getFunction().descriptor(); - final Map args = expr.getArgs(); + final FunctionArgs args = expr.getArgs(); for (ParameterDescriptor p : descriptor.params()) { - final Expression argExpr = args.get(p.name()); + final Expression argExpr = args.expression(p.name()); if (argExpr != null && !p.type().isAssignableFrom(argExpr.getType())) { parseContext.addError(new IncompatibleArgumentType(ctx, expr, p, argExpr)); } @@ -572,6 +591,7 @@ public boolean isInnerNode(RuleContext node) { /** * Links the declared var to its expression. + * * @param name var name * @param expr expression * @return true if successful, false if previously declared diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index bafe74645fa9..0d9ece57c92c 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -71,12 +71,12 @@ public Messages process(Messages messages) { for (Message message : messages) { try { - final EvaluationContext context = new EvaluationContext(); - if (rule.when().evaluateBool(context, message)) { + final EvaluationContext context = new EvaluationContext(message); + if (rule.when().evaluateBool(context)) { log.info("[✓] Message {} matches condition", message.getId()); for (Statement statement : rule.then()) { - statement.evaluate(context, message); + statement.evaluate(context); } } else { diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index e2801a6eab4b..b79a0e187f28 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -4,8 +4,8 @@ import com.google.common.collect.Maps; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; import org.graylog.plugins.messageprocessor.ast.functions.Function; +import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.messageprocessor.ast.statements.Statement; @@ -58,7 +58,7 @@ public static void registerFunctions() { final Map> functions = Maps.newHashMap(); functions.put("nein", new Function() { @Override - public Boolean evaluate(Map args, EvaluationContext context, Message message) { + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { return false; } @@ -73,7 +73,7 @@ public FunctionDescriptor descriptor() { }); functions.put("doch", new Function() { @Override - public Boolean evaluate(Map args, EvaluationContext context, Message message) { + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { return true; } @@ -88,7 +88,7 @@ public FunctionDescriptor descriptor() { }); functions.put("double_valued_func", new Function() { @Override - public Double evaluate(Map args, EvaluationContext context, Message message) { + public Double evaluate(FunctionArgs args, EvaluationContext context) { return 0d; } @@ -103,8 +103,8 @@ public FunctionDescriptor descriptor() { }); functions.put("one_arg", new Function() { @Override - public String evaluate(Map args, EvaluationContext context, Message message) { - return (String) args.get("one").evaluate(context, message); + public String evaluate(FunctionArgs args, EvaluationContext context) { + return args.evaluated("one", context, String.class).orElse(""); } @Override @@ -118,10 +118,10 @@ public FunctionDescriptor descriptor() { }); functions.put("concat", new Function() { @Override - public String evaluate(Map args, EvaluationContext context, Message message) { - final Object one = args.get("one").evaluate(context, message); - final Object two = args.get("two").evaluate(context, message); - final Object three = args.get("three").evaluate(context, message); + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object one = args.evaluated("one", context, Object.class).orElse(""); + final Object two = args.evaluated("two", context, Object.class).orElse(""); + final Object three = args.evaluated("three", context, Object.class).orElse(""); return one.toString() + two.toString() + three.toString(); } @@ -140,7 +140,7 @@ public FunctionDescriptor descriptor() { }); functions.put("trigger_test", new Function() { @Override - public Void evaluate(Map args, EvaluationContext context, Message message) { + public Void evaluate(FunctionArgs args, EvaluationContext context) { actionsTriggered.set(true); return null; } @@ -156,7 +156,7 @@ public FunctionDescriptor descriptor() { }); functions.put("optional", new Function() { @Override - public Boolean evaluate(Map args, EvaluationContext context, Message message) { + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { return true; } @@ -176,8 +176,8 @@ public FunctionDescriptor descriptor() { }); functions.put("customObject", new Function() { @Override - public CustomObject evaluate(Map args, EvaluationContext context, Message message) { - return new CustomObject((String) args.get("default").evaluate(context, message)); + public CustomObject evaluate(FunctionArgs args, EvaluationContext context) { + return new CustomObject(args.evaluated("default", context, String.class).orElse("")); } @Override @@ -357,11 +357,11 @@ public void typedFieldAccess() throws Exception { } private Message evaluateRule(Rule rule, Message message) { - final EvaluationContext context = new EvaluationContext(); - if (rule.when().evaluateBool(context, message)) { + final EvaluationContext context = new EvaluationContext(message); + if (rule.when().evaluateBool(context)) { for (Statement statement : rule.then()) { - statement.evaluate(context, message); + statement.evaluate(context); } return message; } else { From cc34e2b7a14e7447d5ff7821103302e5409db210 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 18 Jan 2016 10:41:50 +0100 Subject: [PATCH 023/521] remove null from rule language --- .../org/graylog/plugins/messageprocessor/parser/RuleLang.g4 | 1 - .../graylog/plugins/messageprocessor/parser/RuleParser.java | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index 1c10ca64eec7..802526b55b51 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -82,7 +82,6 @@ literal | Char # Char | String # String | Boolean # Boolean - | 'null' # Null ; // Lexer diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index 664ec4a9c335..f9c9591da034 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -423,12 +423,6 @@ public void exitFunc(RuleLangParser.FuncContext ctx) { exprs.put(ctx, exprs.get(ctx.functionCall())); parseContext.addInnerNode(ctx); } - - @Override - public void exitNull(RuleLangParser.NullContext ctx) { - // TODO - super.exitNull(ctx); - } } private class TypeAnnotator extends RuleLangBaseListener { From 52bed0a2b223dcc1185b27c607c7b597d19cfef7 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 18 Jan 2016 17:21:05 +0100 Subject: [PATCH 024/521] add map and array literals - fix identifier text handling in some places, this needs a little refactoring to get rid of duplicate code - maps and arrays work, but generics aren't supported due to type erasure. looking to use TypeLiteral from guice instead, needs even more changes --- .../messageprocessor/parser/RuleLang.g4 | 40 ++++++++++++---- .../ast/expressions/ArrayExpression.java | 37 +++++++++++++++ .../ast/expressions/MapExpression.java | 40 ++++++++++++++++ .../messageprocessor/parser/RuleParser.java | 46 ++++++++++--------- .../parser/RuleParserTest.java | 46 +++++++++++++++++++ .../parser/mapArrayLiteral.txt | 5 ++ 6 files changed, 182 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/mapArrayLiteral.txt diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index 802526b55b51..f5064b8281b1 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -33,18 +33,35 @@ grammar RuleLang; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; } +file + : ruleDeclaration* pipelineDecl* EOF + ; + +pipelineDecl + : 'pipeline' name=Identifier + (When condition=expression)? + 'run' + ruleRef* + End + ; + +ruleRef + : Identifier ';' + | ';' + ; + ruleDeclaration - : Rule name=String + : Rule name=Identifier + title=String? + description=String? (During Stage stage=Integer)? When condition=expression Then actions=statement* End - EOF ; expression - : primary # PrimaryExpression - | MessageRef '.' field=expression # MessageRef + : MessageRef '.' field=expression # MessageRef | fieldSet=expression '.' field=expression # Nested | fieldSet=expression '[' expression ']' # Array | functionCall # Func @@ -53,12 +70,15 @@ expression | left=expression and=And right=expression # And | left=expression or=Or right=expression # Or | not=Not expression # Not + | Identifier # Identifier + | literal # LiteralPrimary + | '[' (expression (',' expression)*)* ']' # ArrayLiteralExpr + | '{' (propAssignment (',' propAssignment)*)* '}' # MapLiteralExpr + | '(' expression ')' # ParenExpr ; -primary - : '(' expression ')' # ParenExpr - | literal # LiteralPrimary - | Identifier # Identifier +propAssignment + : Identifier ':' expression ; statement @@ -72,8 +92,8 @@ functionCall ; arguments - : Identifier ':' expression (',' Identifier ':' expression)* # NamedArgs - | expression (',' expression)* # PositionalArgs + : propAssignment (',' propAssignment)* # NamedArgs + | expression (',' expression)* # PositionalArgs ; literal diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java new file mode 100644 index 000000000000..a8676bcd146e --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java @@ -0,0 +1,37 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import com.google.common.base.Joiner; +import org.graylog.plugins.messageprocessor.EvaluationContext; + +import java.util.List; +import java.util.stream.Collectors; + +public class ArrayExpression implements Expression { + private final List elements; + + public ArrayExpression(List elements) { + this.elements = elements; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context) { + return elements.stream() + .map(expression -> expression.evaluate(context)) + .collect(Collectors.toList()); + } + + @Override + public Class getType() { + return List.class; + } + + @Override + public String toString() { + return "[" + Joiner.on(", ").join(elements) + "]"; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java new file mode 100644 index 000000000000..0d55facb6f5c --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java @@ -0,0 +1,40 @@ +package org.graylog.plugins.messageprocessor.ast.expressions; + +import com.google.common.base.Joiner; +import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.jooq.lambda.Seq; +import org.jooq.lambda.tuple.Tuple2; + +import java.util.HashMap; +import java.util.Map; + +public class MapExpression implements Expression { + private final HashMap map; + + public MapExpression(HashMap map) { + this.map = map; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context) { + // evaluate all values for each key and return the resulting map + return Seq.seq(map) + .map(entry -> entry.map2(value -> value.evaluate(context))) + .toMap(Tuple2::v1, Tuple2::v2); + } + + @Override + public Class getType() { + return Map.class; + } + + @Override + public String toString() { + return "{" + Joiner.on(", ").withKeyValueSeparator(":").join(map) + "}"; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java index f9c9591da034..fe70a329b2c9 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java @@ -11,8 +11,10 @@ import org.antlr.v4.runtime.tree.ParseTreeProperty; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.apache.mina.util.IdentityHashSet; +import org.graylog.plugins.messageprocessor.ast.expressions.MapExpression; import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.ArrayExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BooleanExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BooleanValuedFunctionWrapper; @@ -51,6 +53,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -120,7 +123,7 @@ public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { @Override public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { - final String name = ctx.varName.getText(); + final String name = Tools.unquote(ctx.varName.getText(), '`'); final Expression expr = exprs.get(ctx.expression()); parseContext.defineVar(name, expr); definedVars.add(name); @@ -215,25 +218,14 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { @Override public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { final Map argMap = Maps.newHashMap(); - final int argCount = ctx.Identifier().size(); - for (int i = 0; i < argCount; i++) { - final String argName = ctx.Identifier(i).getText(); - final Expression argValue = exprs.get(ctx.expression(i)); + for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { + final String argName = Tools.unquote(propAssignmentContext.Identifier().getText(), '`'); + final Expression argValue = exprs.get(propAssignmentContext.expression()); argMap.put(argName, argValue); } args.put(ctx, argMap); } -// @Override -// public void exitSingleDefaultArg(RuleLangParser.SingleDefaultArgContext ctx) { -// final Expression expr = exprs.get(ctx.expression()); -// final HashMap singleArg = Maps.newHashMap(); -// // null key means to use the single declared argument for this function, it's syntactic sugar -// // this gets validated and expanded in a later parsing stage -// singleArg.put(null, expr); -// args.put(ctx, singleArg); -// } - @Override public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); @@ -370,6 +362,23 @@ public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { parseContext.addInnerNode(ctx); } + @Override + public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { + final List elements = ctx.expression().stream().map(exprs::get).collect(Collectors.toList()); + exprs.put(ctx, new ArrayExpression(elements)); + } + + @Override + public void exitMapLiteralExpr(RuleLangParser.MapLiteralExprContext ctx) { + final HashMap map = Maps.newHashMap(); + for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { + final String key = Tools.unquote(propAssignmentContext.Identifier().getText(), '`'); + final Expression value = exprs.get(propAssignmentContext.expression()); + map.put(key, value); + } + exprs.put(ctx, new MapExpression(map)); + } + @Override public void exitParenExpr(RuleLangParser.ParenExprContext ctx) { // nothing to do, just propagate @@ -410,13 +419,6 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { exprs.put(ctx, expr); } - @Override - public void exitPrimaryExpression(RuleLangParser.PrimaryExpressionContext ctx) { - // nothing to do, just propagate - exprs.put(ctx, exprs.get(ctx.primary())); - parseContext.addInnerNode(ctx); - } - @Override public void exitFunc(RuleLangParser.FuncContext ctx) { // nothing to do, just propagate diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java index b79a0e187f28..cd698eaf6425 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java @@ -1,7 +1,9 @@ package org.graylog.plugins.messageprocessor.parser; import com.google.common.base.Charsets; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; import org.graylog.plugins.messageprocessor.EvaluationContext; import org.graylog.plugins.messageprocessor.ast.Rule; import org.graylog.plugins.messageprocessor.ast.functions.Function; @@ -33,7 +35,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.collect.ImmutableList.of; @@ -189,6 +195,38 @@ public FunctionDescriptor descriptor() { .build(); } }); + functions.put("keys", new Function() { + @Override + public List evaluate(FunctionArgs args, EvaluationContext context) { + final Optional map = args.evaluated("map", context, Map.class); + return Lists.newArrayList(map.orElse(Collections.emptyMap()).keySet()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("keys") + .returnType(List.class) + .params(of(param().name("map").type(Map.class).build())) + .build(); + } + }); + functions.put("sort", new Function() { + @Override + public Collection evaluate(FunctionArgs args, EvaluationContext context) { + final Collection collection = args.evaluated("collection", context, Collection.class).orElse(Collections.emptyList()); + return Ordering.natural().sortedCopy(collection); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("sort") + .returnType(Collection.class) + .params(of(param().name("collection").type(Collection.class).build())) + .build(); + } + }); functions.put(LongCoercion.NAME, new LongCoercion()); functions.put(StringCoercion.NAME, new StringCoercion()); functions.put(SetField.NAME, new SetField()); @@ -345,6 +383,14 @@ public void optionalParamsMustBeNamed() throws Exception { } + @Test + public void mapArrayLiteral() { + final Rule rule = parser.parseRule(ruleForTest()); + Message message = new Message("hello test", "source", DateTime.now()); + evaluateRule(rule, message); + assertTrue(actionsTriggered.get()); + } + @Test public void typedFieldAccess() throws Exception { try { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/mapArrayLiteral.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/mapArrayLiteral.txt new file mode 100644 index 000000000000..a8fba5bfdb70 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/mapArrayLiteral.txt @@ -0,0 +1,5 @@ +rule "mapliteral" +when sort(keys({some_identifier: 1, `something with spaces`: "some expression"})) == ["some_identifier", "something with spaces"] +then + trigger_test(); +end \ No newline at end of file From 4801937f28e439981ea24bd392192f8931cfa0ba Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 18 Jan 2016 17:21:54 +0100 Subject: [PATCH 025/521] sort named parameters by name during toString --- .../ast/expressions/FunctionExpression.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java index 8c7de9c3b47d..eca6ff64935f 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java @@ -44,7 +44,11 @@ public Class getType() { public String toString() { String argsString = ""; if (args != null) { - argsString = Joiner.on(", ").withKeyValueSeparator(": ").join(args.getArgs()); // TODO order arg names + argsString = Joiner.on(", ") + .withKeyValueSeparator(": ") + .join(args.getArgs().entrySet().stream() + .sorted((e1, e2) -> e1.getKey().compareTo(e2.getKey())) + .iterator()); } return descriptor.name() + "(" + argsString + ")"; } From 487e06f79ce43e00d305f672cc6a4c5bedc7f69d Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 18 Jan 2016 17:24:08 +0100 Subject: [PATCH 026/521] revert erroneous changes to grammar --- .../messageprocessor/parser/RuleLang.g4 | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index f5064b8281b1..c97250cef809 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -33,31 +33,13 @@ grammar RuleLang; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; } -file - : ruleDeclaration* pipelineDecl* EOF - ; - -pipelineDecl - : 'pipeline' name=Identifier - (When condition=expression)? - 'run' - ruleRef* - End - ; - -ruleRef - : Identifier ';' - | ';' - ; - ruleDeclaration - : Rule name=Identifier - title=String? - description=String? + : Rule name=String (During Stage stage=Integer)? When condition=expression Then actions=statement* End + EOF ; expression From 0a00c25eba2576103437471c2b2abbb5244dcfbc Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 19 Jan 2016 14:07:46 +0100 Subject: [PATCH 027/521] add pipeline and stage parsing - pipelines contain stages, which reference rules by name - fix unquoting strings in rule, pipeline and rule ref names --- .../messageprocessor/parser/RuleLang.g4 | 38 ++++- .../messageprocessor/ast/Pipeline.java | 25 ++++ .../plugins/messageprocessor/ast/Rule.java | 5 +- .../plugins/messageprocessor/ast/Stage.java | 28 ++++ ...uleParser.java => PipelineRuleParser.java} | 133 ++++++++++++------ .../processors/NaiveRuleProcessor.java | 16 +-- .../rest/MessageProcessorRuleResource.java | 12 +- ...rTest.java => PipelineRuleParserTest.java} | 40 +++++- .../parser/pipelineDeclaration.txt | 11 ++ 9 files changed, 240 insertions(+), 68 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java create mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java rename src/main/java/org/graylog/plugins/messageprocessor/parser/{RuleParser.java => PipelineRuleParser.java} (89%) rename src/test/java/org/graylog/plugins/messageprocessor/parser/{RuleParserTest.java => PipelineRuleParserTest.java} (89%) create mode 100644 src/test/resources/org/graylog/plugins/messageprocessor/parser/pipelineDeclaration.txt diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 index c97250cef809..44e0d876fe54 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 @@ -33,13 +33,41 @@ grammar RuleLang; import org.graylog.plugins.messageprocessor.ast.expressions.Expression; } +file + : ( ruleDeclaration + | pipelineDeclaration + )+ + EOF + ; + +pipelineDecls + : pipelineDeclaration+ EOF + ; + +pipelineDeclaration + : Pipeline name=String + stageDeclaration+ + End + ; + +stageDeclaration + : Stage stage=Integer Match modifier=(All|Either) + ruleRef+ + ; + +ruleRef + : Rule name=String + ; + +ruleDecls + : ruleDeclaration+ EOF + ; + ruleDeclaration : Rule name=String - (During Stage stage=Integer)? When condition=expression - Then actions=statement* + (Then actions=statement*)? End - EOF ; expression @@ -88,9 +116,12 @@ literal // Lexer +All : A L L; +Either: E I T H E R; And : A N D | '&&'; Or: O R | '||'; Not: N O T | '!'; +Pipeline: P I P E L I N E; Rule: R U L E; During: D U R I N G; Stage: S T A G E; @@ -98,6 +129,7 @@ When: W H E N; Then: T H E N; End: E N D; Let: L E T; +Match: M A T C H; MessageRef: 'message'; Boolean diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java new file mode 100644 index 000000000000..fadf025883c7 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.messageprocessor.ast; + +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +public abstract class Pipeline { + + public abstract String name(); + public abstract List stages(); + + public static Builder builder() { + return new AutoValue_Pipeline.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Pipeline build(); + + public abstract Builder name(String name); + + public abstract Builder stages(List stages); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java index 08c1eb44d579..d00e131dbabd 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java @@ -11,21 +11,18 @@ public abstract class Rule { public abstract String name(); - public abstract int stage(); - public abstract LogicalExpression when(); public abstract Collection then(); public static Builder builder() { - return new AutoValue_Rule.Builder().stage(0); + return new AutoValue_Rule.Builder(); } @AutoValue.Builder public abstract static class Builder { public abstract Builder name(String name); - public abstract Builder stage(int stage); public abstract Builder when(LogicalExpression condition); public abstract Builder then(Collection actions); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java new file mode 100644 index 000000000000..08ed51cf7802 --- /dev/null +++ b/src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java @@ -0,0 +1,28 @@ +package org.graylog.plugins.messageprocessor.ast; + +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +public abstract class Stage { + + public abstract int stage(); + public abstract boolean matchAll(); + public abstract List ruleReferences(); + + public static Builder builder() { + return new AutoValue_Stage.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Stage build(); + + public abstract Builder stage(int stageNumber); + + public abstract Builder matchAll(boolean mustMatchAll); + + public abstract Builder ruleReferences(List ruleRefs); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParser.java similarity index 89% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java rename to src/main/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParser.java index fe70a329b2c9..bda58e054daf 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/RuleParser.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParser.java @@ -11,8 +11,9 @@ import org.antlr.v4.runtime.tree.ParseTreeProperty; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.apache.mina.util.IdentityHashSet; -import org.graylog.plugins.messageprocessor.ast.expressions.MapExpression; +import org.graylog.plugins.messageprocessor.ast.Pipeline; import org.graylog.plugins.messageprocessor.ast.Rule; +import org.graylog.plugins.messageprocessor.ast.Stage; import org.graylog.plugins.messageprocessor.ast.expressions.AndExpression; import org.graylog.plugins.messageprocessor.ast.expressions.ArrayExpression; import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; @@ -27,6 +28,7 @@ import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; import org.graylog.plugins.messageprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.messageprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.messageprocessor.ast.expressions.MapExpression; import org.graylog.plugins.messageprocessor.ast.expressions.MessageRefExpression; import org.graylog.plugins.messageprocessor.ast.expressions.NotExpression; import org.graylog.plugins.messageprocessor.ast.expressions.OrExpression; @@ -57,18 +59,19 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; -public class RuleParser { +import static java.util.stream.Collectors.toList; + +public class PipelineRuleParser { private final FunctionRegistry functionRegistry; @Inject - public RuleParser(FunctionRegistry functionRegistry) { + public PipelineRuleParser(FunctionRegistry functionRegistry) { this.functionRegistry = functionRegistry; } - private static final Logger log = LoggerFactory.getLogger(RuleParser.class); + private static final Logger log = LoggerFactory.getLogger(PipelineRuleParser.class); public static final ParseTreeWalker WALKER = ParseTreeWalker.DEFAULT; public Rule parseRule(String rule) throws ParseException { @@ -90,7 +93,22 @@ public Rule parseRule(String rule) throws ParseException { WALKER.walk(new TypeChecker(parseContext), ruleDeclaration); if (parseContext.getErrors().isEmpty()) { - return parseContext.getRule(); + return parseContext.getRules().get(0); + } + throw new ParseException(parseContext.getErrors()); + } + + public List parsePipelines(String pipeline) throws ParseException { + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(pipeline)); + final RuleLangParser parser = new RuleLangParser(new CommonTokenStream(lexer)); + + final RuleLangParser.PipelineDeclsContext pipelineDeclsContext = parser.pipelineDecls(); + final ParseContext parseContext = new ParseContext(); + + WALKER.walk(new AstBuilder(parseContext), pipelineDeclsContext); + + if (parseContext.getErrors().isEmpty()) { + return parseContext.pipelines; } throw new ParseException(parseContext.getErrors()); } @@ -114,6 +132,55 @@ public AstBuilder(ParseContext parseContext) { exprs = parseContext.expressions(); } + @Override + public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ctx) { + final Pipeline.Builder builder = Pipeline.builder(); + + builder.name(Tools.unquote(ctx.name.getText(), '"')); + List stages = Lists.newArrayList(); + for (RuleLangParser.StageDeclarationContext stage : ctx.stageDeclaration()) { + final Stage.Builder stageBuilder = Stage.builder(); + + final int stageNumber = Integer.parseInt(stage.stage.getText()); + stageBuilder.stage(stageNumber); + + final boolean isAllModifier = stage.modifier.getText().equalsIgnoreCase("all"); + stageBuilder.matchAll(isAllModifier); + + final List ruleRefs = stage.ruleRef().stream() + .map(ruleRefContext -> Tools.unquote(ruleRefContext.name.getText(), '"')) + .collect(toList()); + stageBuilder.ruleReferences(ruleRefs); + + stages.add(stageBuilder.build()); + } + + builder.stages(stages); + parseContext.pipelines.add(builder.build()); + } + + @Override + public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { + final Rule.Builder ruleBuilder = Rule.builder(); + ruleBuilder.name(Tools.unquote(ctx.name.getText(), '"')); + final Expression expr = exprs.get(ctx.condition); + + LogicalExpression condition; + if (expr instanceof LogicalExpression) { + condition = (LogicalExpression) expr; + } else if (expr.getType().equals(Boolean.class)) { + condition = new BooleanValuedFunctionWrapper(expr); + } else { + condition = new BooleanExpression(false); + log.debug("Unable to create condition, replacing with 'false'"); + } + ruleBuilder.when(condition); + ruleBuilder.then(parseContext.statements); + final Rule rule = ruleBuilder.build(); + parseContext.addRule(rule); + log.info("Declaring rule {}", rule); + } + @Override public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { final Expression expr = exprs.get(ctx.functionCall()); @@ -155,7 +222,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { .filter(p -> !p.optional()) .map(p -> givenArguments.containsKey(p.name()) ? null : p) .filter(p -> p != null) - .collect(Collectors.toList()); + .collect(toList()); for (ParameterDescriptor param : missingParams) { parseContext.addError(new MissingRequiredParam(ctx, function, param)); } @@ -229,35 +296,10 @@ public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { @Override public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); - expressions.addAll(ctx.expression().stream().map(exprs::get).collect(Collectors.toList())); + expressions.addAll(ctx.expression().stream().map(exprs::get).collect(toList())); argsList.put(ctx, expressions); } - @Override - public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { - final Rule.Builder ruleBuilder = Rule.builder(); - ruleBuilder.name(ctx.name.getText()); - if (ctx.stage != null) { - ruleBuilder.stage(Integer.parseInt(ctx.stage.getText())); - } - final Expression expr = exprs.get(ctx.condition); - - LogicalExpression condition; - if (expr instanceof LogicalExpression) { - condition = (LogicalExpression) expr; - } else if (expr.getType().equals(Boolean.class)) { - condition = new BooleanValuedFunctionWrapper(expr); - } else { - condition = new BooleanExpression(false); - log.debug("Unable to create condition, replacing with 'false'"); - } - ruleBuilder.when(condition); - ruleBuilder.then(parseContext.statements); - final Rule rule = ruleBuilder.build(); - parseContext.setRule(rule); - log.info("Declaring rule {}", rule); - } - @Override public void enterNested(RuleLangParser.NestedContext ctx) { // nested field access is ok, these are not rule variables @@ -322,6 +364,7 @@ public void exitComparison(RuleLangParser.ComparisonContext ctx) { @Override public void exitInteger(RuleLangParser.IntegerContext ctx) { + // TODO handle different radix and length final LongExpression expr = new LongExpression(Long.parseLong(ctx.getText())); log.info("INT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -342,8 +385,8 @@ public void exitChar(RuleLangParser.CharContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { - final String text = ctx.getText(); - final StringExpression expr = new StringExpression(text.substring(1, text.length() - 1)); + final String text = Tools.unquote(ctx.getText(), '\"'); + final StringExpression expr = new StringExpression(text); log.info("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -364,7 +407,7 @@ public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { @Override public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { - final List elements = ctx.expression().stream().map(exprs::get).collect(Collectors.toList()); + final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); exprs.put(ctx, new ArrayExpression(elements)); } @@ -550,8 +593,9 @@ private static class ParseContext { // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information private Set innerNodes = new IdentityHashSet<>(); public List statements = Lists.newArrayList(); - public Rule rule; + public List rules = Lists.newArrayList(); private Map varDecls = Maps.newHashMap(); + public List pipelines = Lists.newArrayList(); public ParseTreeProperty expressions() { return exprs; @@ -561,12 +605,19 @@ public ParseTreeProperty> arguments() { return args; } - public Rule getRule() { - return rule; + public List getRules() { + return rules; + } + + public void setRules(List rules) { + this.rules = rules; + } + public void addRule(Rule rule) { + this.rules.add(rule); } - public void setRule(Rule rule) { - this.rule = rule; + public List getPipelines() { + return pipelines; } public Set getErrors() { diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java index 0d9ece57c92c..e98560b36cec 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java @@ -15,7 +15,7 @@ import org.graylog.plugins.messageprocessor.db.RuleSourceService; import org.graylog.plugins.messageprocessor.events.RulesChangedEvent; import org.graylog.plugins.messageprocessor.parser.ParseException; -import org.graylog.plugins.messageprocessor.parser.RuleParser; +import org.graylog.plugins.messageprocessor.parser.PipelineRuleParser; import org.graylog.plugins.messageprocessor.rest.RuleSource; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; @@ -47,7 +47,7 @@ public class NaiveRuleProcessor implements MessageProcessor { @Inject public NaiveRuleProcessor(RuleSourceService ruleSourceService, - RuleParser ruleParser, + PipelineRuleParser pipelineRuleParser, Journal journal, MetricRegistry metricRegistry, @Named("daemonScheduler") ScheduledExecutorService scheduledExecutorService, @@ -56,7 +56,7 @@ public NaiveRuleProcessor(RuleSourceService ruleSourceService, this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); clusterBus.register(this); ruleCache = CacheBuilder.newBuilder() - .build(asyncReloading(new RuleLoader(ruleSourceService, ruleParser), scheduledExecutorService)); + .build(asyncReloading(new RuleLoader(ruleSourceService, pipelineRuleParser), scheduledExecutorService)); // prime the cache with all presently stored rules try { ruleCache.getAll(Collections.emptyList()); @@ -110,11 +110,11 @@ public void handleRuleChanges(RulesChangedEvent event) { private static class RuleLoader extends CacheLoader { private final RuleSourceService ruleSourceService; - private final RuleParser ruleParser; + private final PipelineRuleParser pipelineRuleParser; - public RuleLoader(RuleSourceService ruleSourceService, RuleParser ruleParser) { + public RuleLoader(RuleSourceService ruleSourceService, PipelineRuleParser pipelineRuleParser) { this.ruleSourceService = ruleSourceService; - this.ruleParser = ruleParser; + this.pipelineRuleParser = pipelineRuleParser; } @Override @@ -128,7 +128,7 @@ public Map loadAll(Iterable keys) throws Excepti } } try { - all.put(ruleSource.id(), ruleParser.parseRule(ruleSource.source())); + all.put(ruleSource.id(), pipelineRuleParser.parseRule(ruleSource.source())); } catch (ParseException e) { log.error("Unable to parse rule: " + e.getMessage()); } @@ -140,7 +140,7 @@ public Map loadAll(Iterable keys) throws Excepti public Rule load(@Nullable String ruleId) throws Exception { final RuleSource ruleSource = ruleSourceService.load(ruleId); try { - return ruleParser.parseRule(ruleSource.source()); + return pipelineRuleParser.parseRule(ruleSource.source()); } catch (ParseException e) { log.error("Unable to parse rule: " + e.getMessage()); throw e; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java index 8c8e671a2139..6c9b3627704b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java +++ b/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java @@ -7,7 +7,7 @@ import org.graylog.plugins.messageprocessor.db.RuleSourceService; import org.graylog.plugins.messageprocessor.events.RulesChangedEvent; import org.graylog.plugins.messageprocessor.parser.ParseException; -import org.graylog.plugins.messageprocessor.parser.RuleParser; +import org.graylog.plugins.messageprocessor.parser.PipelineRuleParser; import org.graylog2.database.NotFoundException; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.rest.PluginRestResource; @@ -38,15 +38,15 @@ public class MessageProcessorRuleResource extends RestResource implements Plugin private static final Logger log = LoggerFactory.getLogger(MessageProcessorRuleResource.class); private final RuleSourceService ruleSourceService; - private final RuleParser ruleParser; + private final PipelineRuleParser pipelineRuleParser; private final EventBus clusterBus; @Inject public MessageProcessorRuleResource(RuleSourceService ruleSourceService, - RuleParser ruleParser, + PipelineRuleParser pipelineRuleParser, @ClusterEventBus EventBus clusterBus) { this.ruleSourceService = ruleSourceService; - this.ruleParser = ruleParser; + this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; } @@ -58,7 +58,7 @@ public MessageProcessorRuleResource(RuleSourceService ruleSourceService, @POST public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull String ruleSource) throws ParseException { try { - ruleParser.parseRule(ruleSource); + pipelineRuleParser.parseRule(ruleSource); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -100,7 +100,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { final RuleSource ruleSource = ruleSourceService.load(id); try { - ruleParser.parseRule(update.source()); + pipelineRuleParser.parseRule(update.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java b/src/test/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParserTest.java similarity index 89% rename from src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java rename to src/test/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParserTest.java index cd698eaf6425..ca8fc89a47ed 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/RuleParserTest.java +++ b/src/test/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParserTest.java @@ -1,11 +1,14 @@ package org.graylog.plugins.messageprocessor.parser; import com.google.common.base.Charsets; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.messageprocessor.ast.Pipeline; import org.graylog.plugins.messageprocessor.ast.Rule; +import org.graylog.plugins.messageprocessor.ast.Stage; import org.graylog.plugins.messageprocessor.ast.functions.Function; import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; @@ -44,17 +47,18 @@ import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class RuleParserTest { +public class PipelineRuleParserTest { @org.junit.Rule public TestName name = new TestName(); - private RuleParser parser; + private PipelineRuleParser parser; private static FunctionRegistry functionRegistry; private static final AtomicBoolean actionsTriggered = new AtomicBoolean(false); @@ -214,7 +218,9 @@ public FunctionDescriptor descriptor() { functions.put("sort", new Function() { @Override public Collection evaluate(FunctionArgs args, EvaluationContext context) { - final Collection collection = args.evaluated("collection", context, Collection.class).orElse(Collections.emptyList()); + final Collection collection = args.evaluated("collection", + context, + Collection.class).orElse(Collections.emptyList()); return Ordering.natural().sortedCopy(collection); } @@ -236,7 +242,7 @@ public FunctionDescriptor descriptor() { @Before public void setup() { - parser = new RuleParser(functionRegistry); + parser = new PipelineRuleParser(functionRegistry); // initialize before every test! actionsTriggered.set(false); } @@ -258,8 +264,10 @@ public void undeclaredIdentifier() throws Exception { parser.parseRule(ruleForTest()); fail("should throw error: undeclared variable x"); } catch (ParseException e) { - assertEquals(2, e.getErrors().size()); // undeclared var and incompatible type, but we only care about the undeclared one here - assertTrue("Should find error UndeclaredVariable", e.getErrors().stream().anyMatch(error -> error instanceof UndeclaredVariable)); + assertEquals(2, + e.getErrors().size()); // undeclared var and incompatible type, but we only care about the undeclared one here + assertTrue("Should find error UndeclaredVariable", + e.getErrors().stream().anyMatch(error -> error instanceof UndeclaredVariable)); } } @@ -402,6 +410,26 @@ public void typedFieldAccess() throws Exception { } } + @Test + public void pipelineDeclaration() throws Exception { + final List pipelines = parser.parsePipelines(ruleForTest()); + assertEquals(1, pipelines.size()); + final Pipeline pipeline = Iterables.getOnlyElement(pipelines); + assertEquals("cisco", pipeline.name()); + assertEquals(2, pipeline.stages().size()); + final Stage stage1 = pipeline.stages().get(0); + final Stage stage2 = pipeline.stages().get(1); + + assertEquals(true, stage1.matchAll()); + assertEquals(1, stage1.stage()); + assertArrayEquals(new Object[]{"check_ip_whitelist", "cisco_device"}, stage1.ruleReferences().toArray()); + + assertEquals(false, stage2.matchAll()); + assertEquals(2, stage2.stage()); + assertArrayEquals(new Object[]{"parse_cisco_time", "extract_src_dest", "normalize_src_dest", "lookup_ips", "resolve_ips"}, + stage2.ruleReferences().toArray()); + } + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(message); if (rule.when().evaluateBool(context)) { diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/pipelineDeclaration.txt b/src/test/resources/org/graylog/plugins/messageprocessor/parser/pipelineDeclaration.txt new file mode 100644 index 000000000000..93f2037a2989 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/messageprocessor/parser/pipelineDeclaration.txt @@ -0,0 +1,11 @@ +pipeline "cisco" +stage 1 match all + rule "check_ip_whitelist" + rule "cisco_device" +stage 2 match either + rule "parse_cisco_time" + rule "extract_src_dest" + rule "normalize_src_dest" + rule "lookup_ips" + rule "resolve_ips" +end From 3dfcd5621c3f9f558671259401571291b9548bf6 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 19 Jan 2016 14:32:20 +0100 Subject: [PATCH 028/521] initial import --- .gitignore | 12 ++ .travis.yml | 26 +++ GETTING-STARTED.md | 26 +++ README.md | 58 ++++++ package.json | 33 +++ pom.xml | 191 ++++++++++++++++++ src/deb/control/control | 8 + .../pipelineprocessor/PipelineProcessor.java | 8 + .../PipelineProcessorMetaData.java | 57 ++++++ .../PipelineProcessorModule.java | 44 ++++ .../PipelineProcessorPlugin.java | 23 +++ .../services/org.graylog2.plugin.Plugin | 1 + src/web/index.jsx | 21 ++ webpack.config.js | 14 ++ 14 files changed, 522 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 GETTING-STARTED.md create mode 100644 README.md create mode 100644 package.json create mode 100644 pom.xml create mode 100644 src/deb/control/control create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java create mode 100644 src/main/resources/META-INF/services/org.graylog2.plugin.Plugin create mode 100644 src/web/index.jsx create mode 100644 webpack.config.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..97f247cbb1bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.idea/ +*.iml +*.ipr +*.iws +.classpath +.project +.settings/ +target/ +dependency-reduced-pom.xml +node_modules +node +build diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..b6dd9f083839 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +sudo: false +language: java +jdk: + - oraclejdk8 +addons: + apt: + packages: + - rpm +before_deploy: + - mvn jdeb:jdeb && export RELEASE_DEB_FILE=$(ls target/*.deb) + - mvn rpm:rpm && export RELEASE_RPM_FILE=$(find target/ -name '*.rpm' | tail -1) + - rm -f target/original-*.jar + - export RELEASE_PKG_FILE=$(ls target/*.jar) + - echo "Deploying release to GitHub releases" +deploy: + provider: releases + api_key: + secure: + file: + - "${RELEASE_PKG_FILE}" + - "${RELEASE_DEB_FILE}" + - "${RELEASE_RPM_FILE}" + skip_cleanup: true + on: + tags: true + jdk: oraclejdk8 diff --git a/GETTING-STARTED.md b/GETTING-STARTED.md new file mode 100644 index 000000000000..dd82c9dbe190 --- /dev/null +++ b/GETTING-STARTED.md @@ -0,0 +1,26 @@ +Getting started with your new Graylog plugin +============================================ + +Welcome to your new Graylog plugin! + +Please refer to http://docs.graylog.org/en/latest/pages/plugins.html for documentation on how to write +plugins for Graylog. + +Travis CI +--------- + +There is a `.travis.yml` template in this project which is prepared to automatically +deploy the plugin artifacts (JAR, DEB, RPM) to GitHub releases. + +You just have to add your encrypted GitHub access token to the `.travis.yml`. +The token can be generated in your [GitHub personal access token settings](https://github.com/settings/tokens). + +Before Travis CI works, you have to enable it. Install the Travis CI command line +application and execute `travis enable`. + +To encrypt your GitHub access token you can use `travis encrypt`. + +Alternatively you can use `travis setup -f releases` to automatically create a GitHub +access token and add it to the `.travis.yml` file. **Attention:** doing this +will replace some parts of the `.travis.yml` file and you have to restore previous +settings. diff --git a/README.md b/README.md new file mode 100644 index 000000000000..a15ca2c8290a --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# PipelineProcessor Plugin for Graylog + +[![Build Status](https://travis-ci.org/graylog-plugin-pipeline-processor.svg?branch=master)](https://travis-ci.org/graylog-plugin-pipeline-processor) + +__Use this paragraph to enter a description of your plugin.__ + +**Required Graylog version:** 2.0 and later + +Installation +------------ + +[Download the plugin](https://github.com/graylog-plugin-pipeline-processor/releases) +and place the `.jar` file in your Graylog plugin directory. The plugin directory +is the `plugins/` folder relative from your `graylog-server` directory by default +and can be configured in your `graylog.conf` file. + +Restart `graylog-server` and you are done. + +Development +----------- + +You can improve your development experience for the web interface part of your plugin +dramatically by making use of hot reloading. To do this, do the following: + +* `git clone https://github.com/Graylog2/graylog2-server.git` +* `cd graylog2-server/graylog2-web-interface` +* `ln -s $YOURPLUGIN plugin/` +* `npm install && npm start` + +Usage +----- + +__Use this paragraph to document the usage of your plugin__ + + +Getting started +--------------- + +This project is using Maven 3 and requires Java 7 or higher. + +* Clone this repository. +* Run `mvn package` to build a JAR file. +* Optional: Run `mvn jdeb:jdeb` and `mvn rpm:rpm` to create a DEB and RPM package respectively. +* Copy generated JAR file in target directory to your Graylog plugin directory. +* Restart the Graylog. + +Plugin Release +-------------- + +We are using the maven release plugin: + +``` +$ mvn release:prepare +[...] +$ mvn release:perform +``` + +This sets the version numbers, creates a tag and pushes to GitHub. Travis CI will build the release artifacts and upload to GitHub automatically. diff --git a/package.json b/package.json new file mode 100644 index 000000000000..5d9b4747319d --- /dev/null +++ b/package.json @@ -0,0 +1,33 @@ +{ + "name": "PipelineProcessor", + "version": "1.0.0-SNAPSHOT", + "description": "", + "repository": { + "type": "git", + "url": "graylog-plugin-pipeline-processor" + }, + "scripts": { + "start": "./node_modules/.bin/webpack-dev-server", + "build": "./node_modules/.bin/webpack" + }, + "keywords": [ + "graylog" + ], + "author": "Kay Roepke ", + "license": "MIT", + "dependencies": { + "react": "0.14.x", + "react-bootstrap": "^0.28.1", + "react-dom": "^0.14.5", + "react-router": "~1.0.0", + "react-router-bootstrap": "^0.19.0" + }, + "devDependencies": { + "babel-core": "^5.8.25", + "babel-loader": "^5.3.2", + "graylog-web-plugin": "~0.0.15", + "graylog-web-manifests": "*", + "json-loader": "^0.5.4", + "webpack": "^1.12.2" + } +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000000..3b46ef58cb98 --- /dev/null +++ b/pom.xml @@ -0,0 +1,191 @@ + + + 4.0.0 + + 3.0 + + + org.graylog.plugins + pipeline-processor + 1.0.0-SNAPSHOT + jar + + ${project.artifactId} + Graylog ${project.artifactId} plugin. + https://www.graylog.org + + + + Kay Roepke + kay@graylog.com + + + + + scm:git:git@github.com:graylog-plugin-pipeline-processor.git + scm:git:git@github.com:graylog-plugin-pipeline-processor.git + https://github.com/graylog-plugin-pipeline-processor + HEAD + + + + UTF-8 + 1.8 + 1.8 + 2.0.0-SNAPSHOT + /usr/share/graylog-server/plugin + + + + + org.graylog2 + graylog2-plugin + ${graylog2.version} + provided + + + + + + build + + src/main/resources + + **/*.properties + + + + + + com.github.eirslett + frontend-maven-plugin + 0.0.26 + + + + install node and npm + + install-node-and-npm + + + v0.12.5 + 2.11.2 + + + + + npm install + + npm + + + + install + + + + + npm run build + + npm + + + run build + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + false + + + + package + + shade + + + + + + + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5.2 + + true + forked-path + @{project.version} + clean test + package + + + + + jdeb + org.vafer + 1.3 + + ${project.build.directory}/${project.artifactId}-${project.version}.deb + + + ${project.build.directory}/${project.build.finalName}.jar + file + + perm + ${graylog2.plugin-dir} + 644 + root + root + + + + + + + + org.codehaus.mojo + rpm-maven-plugin + 2.1.2 + + Application/Internet + + /usr + + + _unpackaged_files_terminate_build 0 + _binaries_in_noarch_packages_terminate_build 0 + + 644 + 755 + root + root + + + ${graylog2.plugin-dir} + + + ${project.build.directory}/ + + ${project.build.finalName}.jar + + + + + + + + + + diff --git a/src/deb/control/control b/src/deb/control/control new file mode 100644 index 000000000000..2ce51eb96e63 --- /dev/null +++ b/src/deb/control/control @@ -0,0 +1,8 @@ +Package: [[name]] +Version: [[version]] +Architecture: all +Maintainer: Kay Roepke +Section: web +Priority: optional +Depends: graylog-server | graylog-radio +Description: [[description]] diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java new file mode 100644 index 000000000000..f771d63f8744 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java @@ -0,0 +1,8 @@ +package org.graylog.plugins.pipelineprocessor; + +/** + * This is the plugin. Your class should implement one of the existing plugin + * interfaces. (i.e. AlarmCallback, MessageInput, MessageOutput) + */ +public class PipelineProcessor { +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java new file mode 100644 index 000000000000..c06db70bc0b8 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -0,0 +1,57 @@ +package org.graylog.plugins.pipelineprocessor; + +import org.graylog2.plugin.PluginMetaData; +import org.graylog2.plugin.ServerStatus; +import org.graylog2.plugin.Version; + +import java.net.URI; +import java.util.Collections; +import java.util.Set; + +/** + * Implement the PluginMetaData interface here. + */ +public class PipelineProcessorMetaData implements PluginMetaData { + @Override + public String getUniqueId() { + return "org.graylog.plugins.pipelineprocessor.PipelineProcessorPlugin"; + } + + @Override + public String getName() { + return "PipelineProcessor"; + } + + @Override + public String getAuthor() { + // TODO Insert author name + return "PipelineProcessor author"; + } + + @Override + public URI getURL() { + // TODO Insert correct plugin website + return URI.create("https://www.graylog.org/"); + } + + @Override + public Version getVersion() { + return new Version(1, 0, 0); + } + + @Override + public String getDescription() { + // TODO Insert correct plugin description + return "Description of PipelineProcessor plugin"; + } + + @Override + public Version getRequiredVersion() { + return new Version(1, 2, 0); + } + + @Override + public Set getRequiredCapabilities() { + return Collections.emptySet(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java new file mode 100644 index 000000000000..416ddaacd929 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -0,0 +1,44 @@ +package org.graylog.plugins.pipelineprocessor; + +import org.graylog2.plugin.PluginConfigBean; +import org.graylog2.plugin.PluginModule; + +import java.util.Collections; +import java.util.Set; + +/** + * Extend the PluginModule abstract class here to add you plugin to the system. + */ +public class PipelineProcessorModule extends PluginModule { + /** + * Returns all configuration beans required by this plugin. + * + * Implementing this method is optional. The default method returns an empty {@link Set}. + */ + @Override + public Set getConfigBeans() { + return Collections.emptySet(); + } + + @Override + protected void configure() { + /* + * Register your plugin types here. + * + * Examples: + * + * addMessageInput(Class); + * addMessageFilter(Class); + * addMessageOutput(Class); + * addPeriodical(Class); + * addAlarmCallback(Class); + * addInitializer(Class); + * addRestResource(Class); + * + * + * Add all configuration beans returned by getConfigBeans(): + * + * addConfigBeans(); + */ + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java new file mode 100644 index 000000000000..c445feb1fcff --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java @@ -0,0 +1,23 @@ +package org.graylog.plugins.pipelineprocessor; + +import org.graylog2.plugin.Plugin; +import org.graylog2.plugin.PluginMetaData; +import org.graylog2.plugin.PluginModule; + +import java.util.Collection; +import java.util.Collections; + +/** + * Implement the Plugin interface here. + */ +public class PipelineProcessorPlugin implements Plugin { + @Override + public PluginMetaData metadata() { + return new PipelineProcessorMetaData(); + } + + @Override + public Collection modules () { + return Collections.singletonList(new PipelineProcessorModule()); + } +} diff --git a/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin b/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin new file mode 100644 index 000000000000..349ce00d83df --- /dev/null +++ b/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin @@ -0,0 +1 @@ +org.graylog.plugins.pipelineprocessor.PipelineProcessorPlugin \ No newline at end of file diff --git a/src/web/index.jsx b/src/web/index.jsx new file mode 100644 index 000000000000..1bf813e31222 --- /dev/null +++ b/src/web/index.jsx @@ -0,0 +1,21 @@ +import packageJson from '../../package.json'; +import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; + +PluginStore.register(new PluginManifest(packageJson, { + /* This is the place where you define which entities you are providing to the web interface. + Right now you can add routes and navigation elements to it. + + Examples: */ + + // Adding a route to /sample, rendering YourReactComponent when called: + + // routes: [ + // { path: '/sample', component: YourReactComponent }, + // ], + + // Adding an element to the top navigation pointing to /sample named "Sample": + + // navigation: [ + // { path: '/sample', description: 'Sample' }, + // ] +})); diff --git a/webpack.config.js b/webpack.config.js new file mode 100644 index 000000000000..4ac151d85982 --- /dev/null +++ b/webpack.config.js @@ -0,0 +1,14 @@ +const PluginWebpackConfig = require('graylog-web-plugin').PluginWebpackConfig; +const path = require('path'); +const ROOT_PATH = path.resolve(__dirname); +const BUILD_PATH = path.resolve(ROOT_PATH, 'build'); +const ENTRY_PATH = path.resolve(ROOT_PATH, 'src/web/index.jsx'); + +module.exports = new PluginWebpackConfig('org.graylog.plugins.pipelineprocessor.PipelineProcessor', { + root_path: ROOT_PATH, + build_path: BUILD_PATH, + entry_path: ENTRY_PATH +}, +{ + // Here goes your additional webpack configuration. +}); From 92fdbc5eba3a656bf37cef83b4ff06bbd47d4830 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 21 Jan 2016 16:58:36 +0100 Subject: [PATCH 029/521] move to new web-plugin archetype version, [wip] --- package.json | 8 +- pom.xml | 118 +++++++++++------ .../parser/RuleLang.g4 | 2 +- .../messageprocessor/ProcessorPlugin.java | 20 --- .../ProcessorPluginMetaData.java | 51 ------- .../messageprocessor/ast/Pipeline.java | 25 ---- .../plugins/messageprocessor/ast/Rule.java | 32 ----- .../plugins/messageprocessor/ast/Stage.java | 28 ---- .../ast/expressions/AndExpression.java | 30 ----- .../ast/expressions/ArrayExpression.java | 37 ------ .../ast/expressions/BinaryExpression.java | 20 --- .../ast/expressions/BooleanExpression.java | 28 ---- .../BooleanValuedFunctionWrapper.java | 35 ----- .../ast/expressions/ConstantExpression.java | 21 --- .../ast/expressions/DoubleExpression.java | 42 ------ .../ast/expressions/Expression.java | 12 -- .../ast/expressions/FieldRefExpression.java | 31 ----- .../ast/expressions/LogicalExpression.java | 8 -- .../ast/expressions/LongExpression.java | 42 ------ .../ast/expressions/MessageRefExpression.java | 36 ----- .../ast/expressions/NotExpression.java | 29 ---- .../ast/expressions/NumericExpression.java | 12 -- .../ast/expressions/OrExpression.java | 30 ----- .../ast/expressions/StringExpression.java | 23 ---- .../ast/expressions/UnaryExpression.java | 24 ---- .../ast/functions/Function.java | 28 ---- .../ast/statements/FunctionStatement.java | 22 ---- .../ast/statements/Statement.java | 9 -- .../ast/statements/VarAssignStatement.java | 26 ---- .../functions/BooleanCoercion.java | 30 ----- .../functions/DoubleCoercion.java | 38 ------ .../functions/DropMessageFunction.java | 41 ------ .../messageprocessor/functions/HasField.java | 30 ----- .../functions/InputFunction.java | 42 ------ .../functions/LongCoercion.java | 40 ------ .../messageprocessor/functions/SetField.java | 39 ------ .../functions/StringCoercion.java | 40 ------ .../parser/FunctionRegistry.java | 29 ---- .../parser/ParseException.java | 26 ---- .../errors/IncompatibleArgumentType.java | 33 ----- .../parser/errors/IncompatibleType.java | 21 --- .../parser/errors/IncompatibleTypes.java | 29 ---- .../parser/errors/MissingRequiredParam.java | 28 ---- .../errors/OptionalParametersMustBeNamed.java | 20 --- .../parser/errors/UndeclaredFunction.java | 19 --- .../parser/errors/UndeclaredVariable.java | 23 ---- .../parser/errors/WrongNumberOfArgs.java | 27 ---- .../EvaluationContext.java | 18 ++- .../pipelineprocessor/PipelineProcessor.java | 8 -- .../PipelineProcessorMetaData.java | 16 +++ .../PipelineProcessorModule.java | 16 +++ .../PipelineProcessorPlugin.java | 16 +++ .../pipelineprocessor/ProcessorPlugin.java | 36 +++++ .../ProcessorPluginMetaData.java | 67 ++++++++++ .../ProcessorPluginModule.java | 40 ++++-- .../pipelineprocessor/ast/Pipeline.java | 41 ++++++ .../plugins/pipelineprocessor/ast/Rule.java | 48 +++++++ .../plugins/pipelineprocessor/ast/Stage.java | 44 +++++++ .../ast/expressions/AndExpression.java | 46 +++++++ .../ast/expressions/ArrayExpression.java | 53 ++++++++ .../ast/expressions/BinaryExpression.java | 36 +++++ .../ast/expressions/BooleanExpression.java | 44 +++++++ .../BooleanValuedFunctionWrapper.java | 51 +++++++ .../ast/expressions/ComparisonExpression.java | 20 ++- .../ast/expressions/ConstantExpression.java | 37 ++++++ .../ast/expressions/DoubleExpression.java | 58 ++++++++ .../ast/expressions/EqualityExpression.java | 20 ++- .../ast/expressions/Expression.java | 28 ++++ .../expressions/FieldAccessExpression.java | 20 ++- .../ast/expressions/FieldRefExpression.java | 47 +++++++ .../ast/expressions/FunctionExpression.java | 26 +++- .../ast/expressions/LogicalExpression.java | 24 ++++ .../ast/expressions/LongExpression.java | 58 ++++++++ .../ast/expressions/MapExpression.java | 20 ++- .../ast/expressions/MessageRefExpression.java | 52 ++++++++ .../ast/expressions/NotExpression.java | 45 +++++++ .../ast/expressions/NumericExpression.java | 28 ++++ .../ast/expressions/OrExpression.java | 46 +++++++ .../ast/expressions/StringExpression.java | 39 ++++++ .../ast/expressions/UnaryExpression.java | 40 ++++++ .../ast/expressions/VarRefExpression.java | 20 ++- .../ast/functions/Function.java | 44 +++++++ .../ast/functions/FunctionArgs.java | 24 +++- .../ast/functions/FunctionDescriptor.java | 18 ++- .../ast/functions/ParameterDescriptor.java | 18 ++- .../ast/statements/FunctionStatement.java | 38 ++++++ .../ast/statements/Statement.java | 25 ++++ .../ast/statements/VarAssignStatement.java | 42 ++++++ .../db/PipelineSourceService.java | 77 +++++++++++ .../db/RuleSourceService.java | 22 +++- .../events/PipelinesChangedEvent.java | 66 ++++++++++ .../events/RulesChangedEvent.java | 18 ++- .../functions/BooleanCoercion.java | 46 +++++++ .../functions/DoubleCoercion.java | 54 ++++++++ .../functions/DropMessageFunction.java | 57 ++++++++ .../pipelineprocessor/functions/HasField.java | 46 +++++++ .../functions/InputFunction.java | 58 ++++++++ .../functions/LongCoercion.java | 56 ++++++++ .../pipelineprocessor/functions/SetField.java | 55 ++++++++ .../functions/StringCoercion.java | 56 ++++++++ .../parser/FunctionRegistry.java | 45 +++++++ .../parser/ParseException.java | 42 ++++++ .../parser/PipelineRuleParser.java | 121 ++++++++++------- .../errors/IncompatibleArgumentType.java | 49 +++++++ .../parser/errors/IncompatibleType.java | 37 ++++++ .../parser/errors/IncompatibleTypes.java | 45 +++++++ .../parser/errors/MissingRequiredParam.java | 44 +++++++ .../errors/OptionalParametersMustBeNamed.java | 36 +++++ .../parser/errors/ParseError.java | 18 ++- .../parser/errors/UndeclaredFunction.java | 35 +++++ .../parser/errors/UndeclaredVariable.java | 39 ++++++ .../parser/errors/WrongNumberOfArgs.java | 43 ++++++ .../processors/NaiveRuleProcessor.java | 34 +++-- .../rest/PipelineResource.java | 124 ++++++++++++++++++ .../rest/PipelineSource.java | 61 +++++++++ .../rest/RuleResource.java} | 48 ++++--- .../rest/RuleSource.java | 22 +++- .../parser/PipelineRuleParserTest.java | 54 +++++--- .../parser/basicRule.txt | 0 .../booleanValuedFunctionAsCondition.txt | 0 .../parser/declaredFunction.txt | 0 .../parser/inferVariableType.txt | 0 .../parser/invalidArgType.txt | 0 .../parser/mapArrayLiteral.txt | 0 .../parser/messageRef.txt | 0 .../parser/messageRefQuotedField.txt | 0 .../parser/optionalArguments.txt | 0 .../parser/optionalParamsMustBeNamed.txt | 0 .../parser/pipelineDeclaration.txt | 0 .../parser/positionalArguments.txt | 0 .../parser/singleArgFunction.txt | 0 .../parser/typedFieldAccess.txt | 0 .../parser/undeclaredFunction.txt | 0 .../parser/undeclaredIdentifier.txt | 0 src/web/PipelinesActions.jsx | 11 ++ src/web/PipelinesPage.jsx | 50 +++++++ src/web/PipelinesStore.js | 60 +++++++++ src/web/index.jsx | 22 +--- 138 files changed, 2989 insertions(+), 1463 deletions(-) rename src/main/antlr4/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/RuleLang.g4 (99%) delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java delete mode 100644 src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/EvaluationContext.java (56%) delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ProcessorPluginModule.java (53%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/expressions/ComparisonExpression.java (69%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/expressions/EqualityExpression.java (50%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/expressions/FieldAccessExpression.java (62%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/expressions/FunctionExpression.java (54%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/expressions/MapExpression.java (51%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/expressions/VarRefExpression.java (55%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/functions/FunctionArgs.java (53%) rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/functions/FunctionDescriptor.java (51%) rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/ast/functions/ParameterDescriptor.java (68%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/db/RuleSourceService.java (66%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/events/RulesChangedEvent.java (66%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/PipelineRuleParser.java (84%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/errors/ParseError.java (59%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/processors/NaiveRuleProcessor.java (80%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java rename src/main/java/org/graylog/plugins/{messageprocessor/rest/MessageProcessorRuleResource.java => pipelineprocessor/rest/RuleResource.java} (72%) rename src/main/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/rest/RuleSource.java (67%) rename src/test/java/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/PipelineRuleParserTest.java (88%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/basicRule.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/booleanValuedFunctionAsCondition.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/declaredFunction.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/inferVariableType.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/invalidArgType.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/mapArrayLiteral.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/messageRef.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/messageRefQuotedField.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/optionalArguments.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/optionalParamsMustBeNamed.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/pipelineDeclaration.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/positionalArguments.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/singleArgFunction.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/typedFieldAccess.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/undeclaredFunction.txt (100%) rename src/test/resources/org/graylog/plugins/{messageprocessor => pipelineprocessor}/parser/undeclaredIdentifier.txt (100%) create mode 100644 src/web/PipelinesActions.jsx create mode 100644 src/web/PipelinesPage.jsx create mode 100644 src/web/PipelinesStore.js diff --git a/package.json b/package.json index 5d9b4747319d..42a0201cd56d 100644 --- a/package.json +++ b/package.json @@ -14,20 +14,22 @@ "graylog" ], "author": "Kay Roepke ", - "license": "MIT", + "license": "GPL-3.0", "dependencies": { "react": "0.14.x", "react-bootstrap": "^0.28.1", "react-dom": "^0.14.5", "react-router": "~1.0.0", - "react-router-bootstrap": "^0.19.0" + "react-router-bootstrap": "^0.19.0", + "reflux": "^0.2.12" }, "devDependencies": { "babel-core": "^5.8.25", "babel-loader": "^5.3.2", + "graylog-web-manifests": "2.0.0-SNAPSHOT-1", "graylog-web-plugin": "~0.0.15", - "graylog-web-manifests": "*", "json-loader": "^0.5.4", + "react-hot-loader": "^1.3.0", "webpack": "^1.12.2" } } diff --git a/pom.xml b/pom.xml index c7a81508098b..7a636f931142 100644 --- a/pom.xml +++ b/pom.xml @@ -170,12 +170,14 @@ - build - src/main/resources - - **/*.properties - + build + + + src/main/resources + + **/*.properties + @@ -185,49 +187,81 @@ antlr4-maven-plugin 4.5 + + com.mycila + license-maven-plugin + 2.11 + +
com/mycila/maven/plugin/license/templates/GPL-3.txt
+ + ${project.organization.name} + Graylog Pipeline Processor + + + **/src/main/java/** + **/src/test/java/** + + + + + + check + + + + - - com.github.eirslett - frontend-maven-plugin - 0.0.26 + + maven-assembly-plugin + + true + + + + com.mycila + license-maven-plugin + + + com.github.eirslett + frontend-maven-plugin + 0.0.26 - - - install node and npm - - install-node-and-npm - - - v0.12.5 - 2.11.2 - - + + + install node and npm + + install-node-and-npm + + + v0.12.5 + 2.11.2 + + - - npm install - - npm - - - - install - - + + npm install + + npm + + + + install + + - - npm run build - - npm - - - run build - - - - - + + npm run build + + npm + + + run build + + + + org.apache.maven.plugins maven-shade-plugin diff --git a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 similarity index 99% rename from src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 rename to src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 index 44e0d876fe54..c9a4818a31f4 100644 --- a/src/main/antlr4/org/graylog/plugins/messageprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 @@ -30,7 +30,7 @@ grammar RuleLang; @header { -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; } file diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java deleted file mode 100644 index 6a7b840237b9..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPlugin.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.graylog.plugins.messageprocessor; - -import org.graylog2.plugin.Plugin; -import org.graylog2.plugin.PluginMetaData; -import org.graylog2.plugin.PluginModule; - -import java.util.Collection; -import java.util.Collections; - -public class ProcessorPlugin implements Plugin { - @Override - public PluginMetaData metadata() { - return new ProcessorPluginMetaData(); - } - - @Override - public Collection modules () { - return Collections.singletonList(new ProcessorPluginModule()); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java b/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java deleted file mode 100644 index 35934a5451d2..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginMetaData.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.graylog.plugins.messageprocessor; - -import com.google.common.collect.Sets; -import org.graylog2.plugin.PluginMetaData; -import org.graylog2.plugin.ServerStatus; -import org.graylog2.plugin.Version; - -import java.net.URI; -import java.util.Set; - -public class ProcessorPluginMetaData implements PluginMetaData { - @Override - public String getUniqueId() { - return "org.graylog.plugins.messageprocessor.ProcessorPlugin"; - } - - @Override - public String getName() { - return "Message Processor Plugin"; - } - - @Override - public String getAuthor() { - return "Graylog, Inc"; - } - - @Override - public URI getURL() { - return URI.create("https://www.graylog.org/"); - } - - @Override - public Version getVersion() { - return new Version(1, 0, 0); - } - - @Override - public String getDescription() { - return "Pluggable message processing framework"; - } - - @Override - public Version getRequiredVersion() { - return new Version(2, 0, 0); - } - - @Override - public Set getRequiredCapabilities() { - return Sets.newHashSet(ServerStatus.Capability.SERVER); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java deleted file mode 100644 index fadf025883c7..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/Pipeline.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast; - -import com.google.auto.value.AutoValue; - -import java.util.List; - -@AutoValue -public abstract class Pipeline { - - public abstract String name(); - public abstract List stages(); - - public static Builder builder() { - return new AutoValue_Pipeline.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Pipeline build(); - - public abstract Builder name(String name); - - public abstract Builder stages(List stages); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java deleted file mode 100644 index d00e131dbabd..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/Rule.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast; - -import com.google.auto.value.AutoValue; -import org.graylog.plugins.messageprocessor.ast.expressions.LogicalExpression; -import org.graylog.plugins.messageprocessor.ast.statements.Statement; - -import java.util.Collection; - -@AutoValue -public abstract class Rule { - - public abstract String name(); - - public abstract LogicalExpression when(); - - public abstract Collection then(); - - public static Builder builder() { - return new AutoValue_Rule.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder name(String name); - public abstract Builder when(LogicalExpression condition); - public abstract Builder then(Collection actions); - - public abstract Rule build(); - } - -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java deleted file mode 100644 index 08ed51cf7802..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/Stage.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast; - -import com.google.auto.value.AutoValue; - -import java.util.List; - -@AutoValue -public abstract class Stage { - - public abstract int stage(); - public abstract boolean matchAll(); - public abstract List ruleReferences(); - - public static Builder builder() { - return new AutoValue_Stage.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Stage build(); - - public abstract Builder stage(int stageNumber); - - public abstract Builder matchAll(boolean mustMatchAll); - - public abstract Builder ruleReferences(List ruleRefs); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java deleted file mode 100644 index aeda5ad95ed0..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/AndExpression.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class AndExpression extends BinaryExpression implements LogicalExpression { - public AndExpression(LogicalExpression left, - LogicalExpression right) { - super(left, right); - } - - @Override - public Object evaluate(EvaluationContext context) { - return evaluateBool(context); - } - - @Override - public boolean evaluateBool(EvaluationContext context) { - return ((LogicalExpression)left).evaluateBool(context) && ((LogicalExpression)right).evaluateBool(context); - } - - @Override - public Class getType() { - return Boolean.class; - } - - @Override - public String toString() { - return left.toString() + " AND " + right.toString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java deleted file mode 100644 index a8676bcd146e..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ArrayExpression.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import com.google.common.base.Joiner; -import org.graylog.plugins.messageprocessor.EvaluationContext; - -import java.util.List; -import java.util.stream.Collectors; - -public class ArrayExpression implements Expression { - private final List elements; - - public ArrayExpression(List elements) { - this.elements = elements; - } - - @Override - public boolean isConstant() { - return false; - } - - @Override - public Object evaluate(EvaluationContext context) { - return elements.stream() - .map(expression -> expression.evaluate(context)) - .collect(Collectors.toList()); - } - - @Override - public Class getType() { - return List.class; - } - - @Override - public String toString() { - return "[" + Joiner.on(", ").join(elements) + "]"; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java deleted file mode 100644 index dcbb8dbcd635..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BinaryExpression.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -public abstract class BinaryExpression extends UnaryExpression { - - protected final Expression left; - - public BinaryExpression(Expression left, Expression right) { - super(right); - this.left = left; - } - - @Override - public boolean isConstant() { - return left.isConstant() && right.isConstant(); - } - - public Expression left() { - return left; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java deleted file mode 100644 index f73f501f8a66..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanExpression.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class BooleanExpression extends ConstantExpression implements LogicalExpression { - private final boolean value; - - public BooleanExpression(boolean value) { - super(Boolean.class); - this.value = value; - } - - @Override - public Object evaluate(EvaluationContext context) { - return value; - } - - - @Override - public boolean evaluateBool(EvaluationContext context) { - return value; - } - - @Override - public String toString() { - return Boolean.toString(value); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java deleted file mode 100644 index a6c6818367d0..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/BooleanValuedFunctionWrapper.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class BooleanValuedFunctionWrapper implements LogicalExpression { - private final Expression expr; - - public BooleanValuedFunctionWrapper(Expression expr) { - this.expr = expr; - if (!expr.getType().equals(Boolean.class)) { - throw new IllegalArgumentException("expr must be of boolean type"); - } - } - - @Override - public boolean evaluateBool(EvaluationContext context) { - final Object value = expr.evaluate(context); - return value != null && (Boolean) value; - } - - @Override - public boolean isConstant() { - return expr.isConstant(); - } - - @Override - public Object evaluate(EvaluationContext context) { - return evaluateBool(context); - } - - @Override - public Class getType() { - return expr.getType(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java deleted file mode 100644 index 8f39fb19e5bc..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ConstantExpression.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -public abstract class ConstantExpression implements Expression { - - private final Class type; - - protected ConstantExpression(Class type) { - this.type = type; - } - - @Override - public boolean isConstant() { - return true; - } - - @Override - public Class getType() { - return type; - } - -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java deleted file mode 100644 index 97dc29c87745..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/DoubleExpression.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class DoubleExpression extends ConstantExpression implements NumericExpression { - private final double value; - - public DoubleExpression(double value) { - super(Double.class); - this.value = value; - } - - @Override - public Object evaluate(EvaluationContext context) { - return value; - } - - @Override - public String toString() { - return Double.toString(value); - } - - @Override - public boolean isIntegral() { - return false; - } - - @Override - public boolean isFloatingPoint() { - return true; - } - - @Override - public long longValue() { - return (long) value; - } - - @Override - public double doubleValue() { - return value; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java deleted file mode 100644 index e09b64317b4d..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/Expression.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public interface Expression { - - boolean isConstant(); - - Object evaluate(EvaluationContext context); - - Class getType(); -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java deleted file mode 100644 index 93b5c4a2b84f..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldRefExpression.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class FieldRefExpression implements Expression { - private final String variableName; - - public FieldRefExpression(String variableName) { - this.variableName = variableName; - } - - @Override - public boolean isConstant() { - return true; - } - - @Override - public Object evaluate(EvaluationContext context) { - return variableName; - } - - @Override - public Class getType() { - return String.class; - } - - @Override - public String toString() { - return variableName; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java deleted file mode 100644 index b92e0098dd2e..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LogicalExpression.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public interface LogicalExpression extends Expression { - - boolean evaluateBool(EvaluationContext context); -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java deleted file mode 100644 index c260ccb1563f..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/LongExpression.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class LongExpression extends ConstantExpression implements NumericExpression { - private final long value; - - public LongExpression(long value) { - super(Long.class); - this.value = value; - } - - @Override - public Object evaluate(EvaluationContext context) { - return value; - } - - @Override - public String toString() { - return Long.toString(value); - } - - @Override - public boolean isIntegral() { - return true; - } - - @Override - public boolean isFloatingPoint() { - return false; - } - - @Override - public long longValue() { - return value; - } - - @Override - public double doubleValue() { - return value; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java deleted file mode 100644 index b4197a2dfdcb..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MessageRefExpression.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class MessageRefExpression implements Expression { - private final Expression fieldExpr; - - public MessageRefExpression(Expression fieldExpr) { - this.fieldExpr = fieldExpr; - } - - @Override - public boolean isConstant() { - return false; - } - - @Override - public Object evaluate(EvaluationContext context) { - final Object fieldName = fieldExpr.evaluate(context); - return context.currentMessage().getField(fieldName.toString()); - } - - @Override - public Class getType() { - return Object.class; - } - - @Override - public String toString() { - return "$message." + fieldExpr.toString(); - } - - public Expression getFieldExpr() { - return fieldExpr; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java deleted file mode 100644 index 1e4eff5e4a1b..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NotExpression.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class NotExpression extends UnaryExpression implements LogicalExpression { - public NotExpression(LogicalExpression right) { - super(right); - } - - @Override - public Object evaluate(EvaluationContext context) { - return !evaluateBool(context); - } - - @Override - public boolean evaluateBool(EvaluationContext context) { - return !((LogicalExpression)right).evaluateBool(context); - } - - @Override - public Class getType() { - return Boolean.class; - } - - @Override - public String toString() { - return "NOT " + right.toString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java deleted file mode 100644 index 4fa650ba723f..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/NumericExpression.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -public interface NumericExpression extends Expression { - - boolean isIntegral(); - - boolean isFloatingPoint(); - - long longValue(); - - double doubleValue(); -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java deleted file mode 100644 index 72806a2cb98a..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/OrExpression.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class OrExpression extends BinaryExpression implements LogicalExpression { - public OrExpression(LogicalExpression left, - LogicalExpression right) { - super(left, right); - } - - @Override - public Object evaluate(EvaluationContext context) { - return evaluateBool(context); - } - - @Override - public boolean evaluateBool(EvaluationContext context) { - return ((LogicalExpression)left).evaluateBool(context) || ((LogicalExpression)right).evaluateBool(context); - } - - @Override - public Class getType() { - return Boolean.class; - } - - @Override - public String toString() { - return left.toString() + " OR " + right.toString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java deleted file mode 100644 index a79248f31983..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/StringExpression.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public class StringExpression extends ConstantExpression { - - private final String value; - - public StringExpression(String value) { - super(String.class); - this.value = value; - } - - @Override - public Object evaluate(EvaluationContext context) { - return value; - } - - @Override - public String toString() { - return '"' + value + '"'; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java deleted file mode 100644 index 65115b276924..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/UnaryExpression.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; - -public abstract class UnaryExpression implements Expression { - - protected final Expression right; - - public UnaryExpression(Expression right) { - this.right = right; - } - - @Override - public boolean isConstant() { - return right.isConstant(); - } - - @Override - public Class getType() { - return right.getType(); - } - - public Expression right() { - return right; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java deleted file mode 100644 index 584249ab3ef6..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/Function.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.functions; - -import com.google.common.collect.ImmutableList; -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public interface Function { - - Function ERROR_FUNCTION = new Function() { - @Override - public Void evaluate(FunctionArgs args, EvaluationContext context) { - return null; - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name("__unresolved_function") - .returnType(Void.class) - .params(ImmutableList.of()) - .build(); - } - }; - - T evaluate(FunctionArgs args, EvaluationContext context); - - FunctionDescriptor descriptor(); - -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java deleted file mode 100644 index 40ff9b093739..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/FunctionStatement.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.statements; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; - -public class FunctionStatement implements Statement { - private final Expression functionExpression; - - public FunctionStatement(Expression functionExpression) { - this.functionExpression = functionExpression; - } - - @Override - public Object evaluate(EvaluationContext context) { - return functionExpression.evaluate(context); - } - - @Override - public String toString() { - return functionExpression.toString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java deleted file mode 100644 index 39443886e176..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/Statement.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.statements; - -import org.graylog.plugins.messageprocessor.EvaluationContext; - -public interface Statement { - - // TODO should this have a return value at all? - Object evaluate(EvaluationContext context); -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java deleted file mode 100644 index 62570bad6e94..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/statements/VarAssignStatement.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.graylog.plugins.messageprocessor.ast.statements; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; - -public class VarAssignStatement implements Statement { - private final String name; - private final Expression expr; - - public VarAssignStatement(String name, Expression expr) { - this.name = name; - this.expr = expr; - } - - @Override - public Void evaluate(EvaluationContext context) { - final Object result = expr.evaluate(context); - context.define(name, expr.getType(), result); - return null; - } - - @Override - public String toString() { - return "let " + name + " = " + expr.toString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java deleted file mode 100644 index 2767c54308c8..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/BooleanCoercion.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; - -import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; - -public class BooleanCoercion implements Function { - public static final String NAME = "bool"; - - private static final String VALUE = "value"; - - @Override - public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse("false"); - return Boolean.parseBoolean(evaluated.toString()); - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(Boolean.class) - .params(of(object(VALUE))) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java deleted file mode 100644 index 86acb03bee37..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/DoubleCoercion.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import com.google.common.primitives.Doubles; -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; - -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; - -public class DoubleCoercion implements Function { - - public static final String NAME = "double"; - - private static final String VALUE = "value"; - private static final String DEFAULT = "default"; - - @Override - public Double evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); - return (Double) firstNonNull(Doubles.tryParse(evaluated.toString()), args.evaluated(DEFAULT, context, Double.class).orElse(0d)); - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(Double.class) - .params(of( - object(VALUE), - param().optional().name(DEFAULT).type(Double.class).build() - )) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java deleted file mode 100644 index 5d169f5ab7b8..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/DropMessageFunction.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import com.google.common.collect.ImmutableList; -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog2.plugin.Message; - -import java.util.Optional; - -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; - -public class DropMessageFunction implements Function { - - public static final String NAME = "drop_message"; - - @Override - public Void evaluate(FunctionArgs args, EvaluationContext context) { - final Optional message; - if (args.isPresent("message")) { - message = args.evaluated("message", context, Message.class); - } else { - message = Optional.of(context.currentMessage()); - } - message.get().setFilterOut(true); - return null; - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .pure(true) - .returnType(Void.class) - .params(ImmutableList.of( - param().type(Message.class).optional().name("message").build() - )) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java deleted file mode 100644 index 349ba744adcf..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/HasField.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import com.google.common.collect.ImmutableList; -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; - -import java.util.Optional; - -public class HasField implements Function { - - public static final String NAME = "has_field"; - - @Override - public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final Optional field = args.evaluated("field", context, String.class); - return context.currentMessage().hasField(field.orElse(null)); - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(Boolean.class) - .params(ImmutableList.of(ParameterDescriptor.string("field"))) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java deleted file mode 100644 index 65c968d49643..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/InputFunction.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog2.plugin.IOState; -import org.graylog2.plugin.inputs.MessageInput; -import org.graylog2.shared.inputs.InputRegistry; - -import javax.inject.Inject; - -import static com.google.common.collect.ImmutableList.of; - -public class InputFunction implements Function { - - public static final String NAME = "input"; - - private final InputRegistry inputRegistry; - - @Inject - public InputFunction(InputRegistry inputRegistry) { - this.inputRegistry = inputRegistry; - } - - @Override - public MessageInput evaluate(FunctionArgs args, EvaluationContext context) { - final String id = args.evaluated("id", context, String.class).orElse(""); - final IOState inputState = inputRegistry.getInputState(id); - return inputState.getStoppable(); - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(MessageInput.class) - .params(of(ParameterDescriptor.string("id"))) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java deleted file mode 100644 index a8670f930abf..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/LongCoercion.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; - -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.collect.ImmutableList.of; -import static com.google.common.primitives.Longs.tryParse; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; - -public class LongCoercion implements Function { - - public static final String NAME = "long"; - - private static final String VALUE = "value"; - private static final String DEFAULT = "default"; - - @Override - public Long evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); - final Long defaultValue = args.evaluated(DEFAULT, context, Long.class).orElse(0L); - - return firstNonNull(tryParse(evaluated.toString()), defaultValue); - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(Long.class) - .params(of( - object(VALUE), - param().optional().integer(DEFAULT).build() - )) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java deleted file mode 100644 index 3db64c382175..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/SetField.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; - -import java.util.Optional; - -import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.string; - -public class SetField implements Function { - - public static final String NAME = "set_field"; - public static final String FIELD = "field"; - public static final String VALUE = "value"; - - @Override - public Void evaluate(FunctionArgs args, EvaluationContext context) { - final Optional field = args.evaluated(FIELD, context, Object.class); - final Optional value = args.evaluated(VALUE, context, Object.class); - - context.currentMessage().addField(field.get().toString(), value.get()); - - return null; - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(Void.class) - .params(of(string(FIELD), - object(VALUE))) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java b/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java deleted file mode 100644 index ec44f73f7c3c..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/functions/StringCoercion.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.graylog.plugins.messageprocessor.functions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; - -import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; - -public class StringCoercion implements Function { - - public static final String NAME = "string"; - - private static final String VALUE = "value"; - private static final String DEFAULT = "default"; - - @Override - public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); - if (evaluated instanceof String) { - return (String) evaluated; - } else { - return args.evaluated(DEFAULT, context, String.class).orElse(""); - } - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(String.class) - .params(of( - object(VALUE), - param().optional().string(DEFAULT).build() - )) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java deleted file mode 100644 index a8d5a1cd2571..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/FunctionRegistry.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser; - -import org.graylog.plugins.messageprocessor.ast.functions.Function; - -import javax.inject.Inject; -import java.util.Map; - -public class FunctionRegistry { - - private final Map> functions; - - @Inject - public FunctionRegistry(Map> functions) { - this.functions = functions; - } - - - public Function resolve(String name) { - return functions.get(name); - } - - public Function resolveOrError(String name) { - final Function function = resolve(name); - if (function == null) { - return Function.ERROR_FUNCTION; - } - return function; - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java deleted file mode 100644 index 8f6d6104e4eb..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/ParseException.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser; - -import org.graylog.plugins.messageprocessor.parser.errors.ParseError; - -import java.util.Set; - -public class ParseException extends RuntimeException { - private final Set errors; - - public ParseException(Set errors) { - this.errors = errors; - } - - public Set getErrors() { - return errors; - } - - @Override - public String getMessage() { - StringBuilder sb = new StringBuilder("Errors:\n"); - for (ParseError parseError : getErrors()) { - sb.append(" ").append(parseError).append("\n"); - } - return sb.toString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java deleted file mode 100644 index 25831ac6cbb4..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleArgumentType.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; -import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class IncompatibleArgumentType extends ParseError { - private final FunctionExpression functionExpression; - private final ParameterDescriptor p; - private final Expression argExpr; - - public IncompatibleArgumentType(RuleLangParser.FunctionCallContext ctx, - FunctionExpression functionExpression, - ParameterDescriptor p, - Expression argExpr) { - super("incompatible_argument_type", ctx); - this.functionExpression = functionExpression; - this.p = p; - this.argExpr = argExpr; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Expected type " + p.type().getSimpleName() + - " for argument " + p.name() + - " but found " + argExpr.getType().getSimpleName() + - " in call to function " + functionExpression.getFunction().descriptor().name() - + positionString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java deleted file mode 100644 index 42ce4dbce311..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleType.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class IncompatibleType extends ParseError { - private final Class expected; - private final Class actual; - - public IncompatibleType(RuleLangParser.MessageRefContext ctx, Class expected, Class actual) { - super("incompatible_type", ctx); - this.expected = expected; - this.actual = actual; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + positionString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java deleted file mode 100644 index a92fb88e3f71..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/IncompatibleTypes.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class IncompatibleTypes extends ParseError { - private final RuleLangParser.ExpressionContext ctx; - private final BinaryExpression binaryExpr; - - public IncompatibleTypes(RuleLangParser.ExpressionContext ctx, BinaryExpression binaryExpr) { - super("incompatible_types", ctx); - this.ctx = ctx; - this.binaryExpr = binaryExpr; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); - } - - private String exprString(Expression e) { - return "(" + e.toString() + ") : " + e.getType().getSimpleName(); - } - - -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java deleted file mode 100644 index 5c30b2e48324..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/MissingRequiredParam.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class MissingRequiredParam extends ParseError { - private final Function function; - private final ParameterDescriptor param; - - public MissingRequiredParam(RuleLangParser.FunctionCallContext ctx, - Function function, - ParameterDescriptor param) { - super("missing_required_param", ctx); - this.function = function; - this.param = param; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Missing required parameter " + param.name() + - " of type " + param.type().getSimpleName() + - " in call to function " + function.descriptor().name() - + positionString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java deleted file mode 100644 index 769e17a1124c..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/OptionalParametersMustBeNamed.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class OptionalParametersMustBeNamed extends ParseError { - private final Function function; - - public OptionalParametersMustBeNamed(RuleLangParser.FunctionCallContext ctx, Function function) { - super("must_name_optional_params", ctx); - this.function = function; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Function " + function.descriptor().name() + " has optional parameters, must use named parameters to call" + positionString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java deleted file mode 100644 index 99967392e2a6..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredFunction.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class UndeclaredFunction extends ParseError { - private final RuleLangParser.FunctionCallContext ctx; - - public UndeclaredFunction(RuleLangParser.FunctionCallContext ctx) { - super("undeclared_function", ctx); - this.ctx = ctx; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Unknown function " + ctx.funcName.getText() + positionString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java deleted file mode 100644 index 3f513d6a04ca..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/UndeclaredVariable.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class UndeclaredVariable extends ParseError { - - @JsonIgnore - private final RuleLangParser.IdentifierContext ctx; - - public UndeclaredVariable(RuleLangParser.IdentifierContext ctx) { - super("undeclared_variable", ctx); - this.ctx = ctx; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Undeclared variable " + ctx.Identifier().getText() + positionString(); - } - -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java b/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java deleted file mode 100644 index e97c8cd08785..000000000000 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/WrongNumberOfArgs.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.graylog.plugins.messageprocessor.parser.errors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.parser.RuleLangParser; - -public class WrongNumberOfArgs extends ParseError { - private final Function function; - private final int argCount; - - public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, - Function function, - int argCount) { - super("wrong_number_of_arguments", ctx); - this.function = function; - this.argCount = argCount; - } - - @JsonProperty("reason") - @Override - public String toString() { - return "Expected " + function.descriptor().params().size() + - " arguments but found " + argCount + - " in call to function " + function.descriptor().name() - + positionString(); - } -} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java similarity index 56% rename from src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java index 1df0087201ab..9720d28210fa 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; import com.google.common.collect.Maps; import org.graylog2.plugin.Message; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java deleted file mode 100644 index f771d63f8744..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessor.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.graylog.plugins.pipelineprocessor; - -/** - * This is the plugin. Your class should implement one of the existing plugin - * interfaces. (i.e. AlarmCallback, MessageInput, MessageOutput) - */ -public class PipelineProcessor { -} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index c06db70bc0b8..a4bbfe6c4987 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor; import org.graylog2.plugin.PluginMetaData; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 416ddaacd929..f4629712467c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor; import org.graylog2.plugin.PluginConfigBean; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java index c445feb1fcff..6fd60cd2c545 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor; import org.graylog2.plugin.Plugin; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java new file mode 100644 index 000000000000..b228499e0845 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import org.graylog2.plugin.Plugin; +import org.graylog2.plugin.PluginMetaData; +import org.graylog2.plugin.PluginModule; + +import java.util.Collection; +import java.util.Collections; + +public class ProcessorPlugin implements Plugin { + @Override + public PluginMetaData metadata() { + return new ProcessorPluginMetaData(); + } + + @Override + public Collection modules () { + return Collections.singletonList(new ProcessorPluginModule()); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java new file mode 100644 index 000000000000..541c00a89f6c --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java @@ -0,0 +1,67 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.collect.Sets; +import org.graylog2.plugin.PluginMetaData; +import org.graylog2.plugin.ServerStatus; +import org.graylog2.plugin.Version; + +import java.net.URI; +import java.util.Set; + +public class ProcessorPluginMetaData implements PluginMetaData { + @Override + public String getUniqueId() { + return "org.graylog.plugins.pipelineprocessor.ProcessorPlugin"; + } + + @Override + public String getName() { + return "Pipeline Processor Plugin"; + } + + @Override + public String getAuthor() { + return "Graylog, Inc"; + } + + @Override + public URI getURL() { + return URI.create("https://www.graylog.org/"); + } + + @Override + public Version getVersion() { + return new Version(1, 0, 0, "SNAPSHOT"); + } + + @Override + public String getDescription() { + return "Pluggable pipeline processing framework"; + } + + @Override + public Version getRequiredVersion() { + return new Version(2, 0, 0); + } + + @Override + public Set getRequiredCapabilities() { + return Sets.newHashSet(ServerStatus.Capability.SERVER); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java similarity index 53% rename from src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java index cff35e294f03..fc6cff84b02f 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ProcessorPluginModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java @@ -1,18 +1,34 @@ -package org.graylog.plugins.messageprocessor; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; import com.google.inject.Binder; import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.functions.BooleanCoercion; -import org.graylog.plugins.messageprocessor.functions.DoubleCoercion; -import org.graylog.plugins.messageprocessor.functions.DropMessageFunction; -import org.graylog.plugins.messageprocessor.functions.HasField; -import org.graylog.plugins.messageprocessor.functions.InputFunction; -import org.graylog.plugins.messageprocessor.functions.LongCoercion; -import org.graylog.plugins.messageprocessor.functions.StringCoercion; -import org.graylog.plugins.messageprocessor.processors.NaiveRuleProcessor; -import org.graylog.plugins.messageprocessor.rest.MessageProcessorRuleResource; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; +import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; +import org.graylog.plugins.pipelineprocessor.functions.DropMessageFunction; +import org.graylog.plugins.pipelineprocessor.functions.HasField; +import org.graylog.plugins.pipelineprocessor.functions.InputFunction; +import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; +import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.processors.NaiveRuleProcessor; +import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; @@ -29,7 +45,7 @@ public Set getConfigBeans() { @Override protected void configure() { addMessageProcessor(NaiveRuleProcessor.class); - addRestResource(MessageProcessorRuleResource.class); + addRestResource(RuleResource.class); // built-in functions addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java new file mode 100644 index 000000000000..3205db6a6ed4 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java @@ -0,0 +1,41 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +public abstract class Pipeline { + + public abstract String name(); + public abstract List stages(); + + public static Builder builder() { + return new AutoValue_Pipeline.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Pipeline build(); + + public abstract Builder name(String name); + + public abstract Builder stages(List stages); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java new file mode 100644 index 000000000000..53b9117466e0 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java @@ -0,0 +1,48 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; + +import java.util.Collection; + +@AutoValue +public abstract class Rule { + + public abstract String name(); + + public abstract LogicalExpression when(); + + public abstract Collection then(); + + public static Builder builder() { + return new AutoValue_Rule.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder name(String name); + public abstract Builder when(LogicalExpression condition); + public abstract Builder then(Collection actions); + + public abstract Rule build(); + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java new file mode 100644 index 000000000000..dd92a5a3980a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +public abstract class Stage { + + public abstract int stage(); + public abstract boolean matchAll(); + public abstract List ruleReferences(); + + public static Builder builder() { + return new AutoValue_Stage.Builder(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Stage build(); + + public abstract Builder stage(int stageNumber); + + public abstract Builder matchAll(boolean mustMatchAll); + + public abstract Builder ruleReferences(List ruleRefs); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java new file mode 100644 index 000000000000..01037e68b99d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class AndExpression extends BinaryExpression implements LogicalExpression { + public AndExpression(LogicalExpression left, + LogicalExpression right) { + super(left, right); + } + + @Override + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + return ((LogicalExpression)left).evaluateBool(context) && ((LogicalExpression)right).evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public String toString() { + return left.toString() + " AND " + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java new file mode 100644 index 000000000000..2a45c146a766 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.base.Joiner; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.util.List; +import java.util.stream.Collectors; + +public class ArrayExpression implements Expression { + private final List elements; + + public ArrayExpression(List elements) { + this.elements = elements; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context) { + return elements.stream() + .map(expression -> expression.evaluate(context)) + .collect(Collectors.toList()); + } + + @Override + public Class getType() { + return List.class; + } + + @Override + public String toString() { + return "[" + Joiner.on(", ").join(elements) + "]"; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java new file mode 100644 index 000000000000..63c0702b6f65 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +public abstract class BinaryExpression extends UnaryExpression { + + protected final Expression left; + + public BinaryExpression(Expression left, Expression right) { + super(right); + this.left = left; + } + + @Override + public boolean isConstant() { + return left.isConstant() && right.isConstant(); + } + + public Expression left() { + return left; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java new file mode 100644 index 000000000000..cb7bf03b2ef2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class BooleanExpression extends ConstantExpression implements LogicalExpression { + private final boolean value; + + public BooleanExpression(boolean value) { + super(Boolean.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context) { + return value; + } + + + @Override + public boolean evaluateBool(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return Boolean.toString(value); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java new file mode 100644 index 000000000000..a744e9089271 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class BooleanValuedFunctionWrapper implements LogicalExpression { + private final Expression expr; + + public BooleanValuedFunctionWrapper(Expression expr) { + this.expr = expr; + if (!expr.getType().equals(Boolean.class)) { + throw new IllegalArgumentException("expr must be of boolean type"); + } + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + final Object value = expr.evaluate(context); + return value != null && (Boolean) value; + } + + @Override + public boolean isConstant() { + return expr.isConstant(); + } + + @Override + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public Class getType() { + return expr.getType(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java similarity index 69% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java index df6061082d3d..776762ef1509 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java @@ -1,6 +1,22 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; -import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class ComparisonExpression extends BinaryExpression implements LogicalExpression { private final String operator; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java new file mode 100644 index 000000000000..dd215610f97c --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java @@ -0,0 +1,37 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +public abstract class ConstantExpression implements Expression { + + private final Class type; + + protected ConstantExpression(Class type) { + this.type = type; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public Class getType() { + return type; + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java new file mode 100644 index 000000000000..ea476c7808d2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class DoubleExpression extends ConstantExpression implements NumericExpression { + private final double value; + + public DoubleExpression(double value) { + super(Double.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return Double.toString(value); + } + + @Override + public boolean isIntegral() { + return false; + } + + @Override + public boolean isFloatingPoint() { + return true; + } + + @Override + public long longValue() { + return (long) value; + } + + @Override + public double doubleValue() { + return value; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java similarity index 50% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java index 16b483f6cea4..7f147fffde17 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java @@ -1,6 +1,22 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; -import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class EqualityExpression extends BinaryExpression implements LogicalExpression { private final boolean checkEquality; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java new file mode 100644 index 000000000000..f49db4b233ca --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java @@ -0,0 +1,28 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface Expression { + + boolean isConstant(); + + Object evaluate(EvaluationContext context); + + Class getType(); +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java similarity index 62% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java index 36de1741e8f9..4702a9781443 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -1,7 +1,23 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; import org.apache.commons.beanutils.PropertyUtils; -import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java new file mode 100644 index 000000000000..a5026c57f66a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java @@ -0,0 +1,47 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class FieldRefExpression implements Expression { + private final String variableName; + + public FieldRefExpression(String variableName) { + this.variableName = variableName; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public Object evaluate(EvaluationContext context) { + return variableName; + } + + @Override + public Class getType() { + return String.class; + } + + @Override + public String toString() { + return variableName; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java similarity index 54% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java index eca6ff64935f..d261623b123a 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java @@ -1,10 +1,26 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; import com.google.common.base.Joiner; -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; public class FunctionExpression implements Expression { private final FunctionArgs args; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java new file mode 100644 index 000000000000..e6c606487b4d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java @@ -0,0 +1,24 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface LogicalExpression extends Expression { + + boolean evaluateBool(EvaluationContext context); +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java new file mode 100644 index 000000000000..0aa3f50ded7d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class LongExpression extends ConstantExpression implements NumericExpression { + private final long value; + + public LongExpression(long value) { + super(Long.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return Long.toString(value); + } + + @Override + public boolean isIntegral() { + return true; + } + + @Override + public boolean isFloatingPoint() { + return false; + } + + @Override + public long longValue() { + return value; + } + + @Override + public double doubleValue() { + return value; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapExpression.java similarity index 51% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapExpression.java index 0d55facb6f5c..ca0fe269e976 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/MapExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapExpression.java @@ -1,7 +1,23 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; import com.google.common.base.Joiner; -import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.jooq.lambda.Seq; import org.jooq.lambda.tuple.Tuple2; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java new file mode 100644 index 000000000000..77bb44b01f1a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java @@ -0,0 +1,52 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class MessageRefExpression implements Expression { + private final Expression fieldExpr; + + public MessageRefExpression(Expression fieldExpr) { + this.fieldExpr = fieldExpr; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluate(EvaluationContext context) { + final Object fieldName = fieldExpr.evaluate(context); + return context.currentMessage().getField(fieldName.toString()); + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return "$message." + fieldExpr.toString(); + } + + public Expression getFieldExpr() { + return fieldExpr; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java new file mode 100644 index 000000000000..3fb8b0d743dd --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java @@ -0,0 +1,45 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class NotExpression extends UnaryExpression implements LogicalExpression { + public NotExpression(LogicalExpression right) { + super(right); + } + + @Override + public Object evaluate(EvaluationContext context) { + return !evaluateBool(context); + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + return !((LogicalExpression)right).evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public String toString() { + return "NOT " + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java new file mode 100644 index 000000000000..9e42984c30ab --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java @@ -0,0 +1,28 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +public interface NumericExpression extends Expression { + + boolean isIntegral(); + + boolean isFloatingPoint(); + + long longValue(); + + double doubleValue(); +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java new file mode 100644 index 000000000000..1cbc95a2b128 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class OrExpression extends BinaryExpression implements LogicalExpression { + public OrExpression(LogicalExpression left, + LogicalExpression right) { + super(left, right); + } + + @Override + public Object evaluate(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + return ((LogicalExpression)left).evaluateBool(context) || ((LogicalExpression)right).evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public String toString() { + return left.toString() + " OR " + right.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java new file mode 100644 index 000000000000..1214c0aea20b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java @@ -0,0 +1,39 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class StringExpression extends ConstantExpression { + + private final String value; + + public StringExpression(String value) { + super(String.class); + this.value = value; + } + + @Override + public Object evaluate(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return '"' + value + '"'; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java new file mode 100644 index 000000000000..db1d35205994 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java @@ -0,0 +1,40 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +public abstract class UnaryExpression implements Expression { + + protected final Expression right; + + public UnaryExpression(Expression right) { + this.right = right; + } + + @Override + public boolean isConstant() { + return right.isConstant(); + } + + @Override + public Class getType() { + return right.getType(); + } + + public Expression right() { + return right; + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java similarity index 55% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java index f3d17d6eb447..27c32408df59 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/expressions/VarRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java @@ -1,6 +1,22 @@ -package org.graylog.plugins.messageprocessor.ast.expressions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; -import org.graylog.plugins.messageprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java new file mode 100644 index 000000000000..4ef0b79a22cc --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface Function { + + Function ERROR_FUNCTION = new Function() { + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("__unresolved_function") + .returnType(Void.class) + .params(ImmutableList.of()) + .build(); + } + }; + + T evaluate(FunctionArgs args, EvaluationContext context); + + FunctionDescriptor descriptor(); + +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java similarity index 53% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index 4aae4b0afdd7..1836414ea66e 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -1,7 +1,23 @@ -package org.graylog.plugins.messageprocessor.ast.functions; - -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java similarity index 51% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java index 570bb8a87250..419ae7196e1b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/FunctionDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.ast.functions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; diff --git a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java similarity index 68% rename from src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index b27feb667a9a..e8c766c31aeb 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.ast.functions; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; import com.google.auto.value.AutoValue; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java new file mode 100644 index 000000000000..74720ca4d39a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java @@ -0,0 +1,38 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.statements; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +public class FunctionStatement implements Statement { + private final Expression functionExpression; + + public FunctionStatement(Expression functionExpression) { + this.functionExpression = functionExpression; + } + + @Override + public Object evaluate(EvaluationContext context) { + return functionExpression.evaluate(context); + } + + @Override + public String toString() { + return functionExpression.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java new file mode 100644 index 000000000000..c47fa46b07fb --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java @@ -0,0 +1,25 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.statements; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface Statement { + + // TODO should this have a return value at all? + Object evaluate(EvaluationContext context); +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java new file mode 100644 index 000000000000..f88e4d57c98b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java @@ -0,0 +1,42 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.statements; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +public class VarAssignStatement implements Statement { + private final String name; + private final Expression expr; + + public VarAssignStatement(String name, Expression expr) { + this.name = name; + this.expr = expr; + } + + @Override + public Void evaluate(EvaluationContext context) { + final Object result = expr.evaluate(context); + context.define(name, expr.getType(), result); + return null; + } + + @Override + public String toString() { + return "let " + name + " = " + expr.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java new file mode 100644 index 000000000000..6efb68febe5b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java @@ -0,0 +1,77 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import com.google.common.collect.Sets; +import com.mongodb.MongoException; +import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoConnection; +import org.graylog2.database.NotFoundException; +import org.mongojack.DBCursor; +import org.mongojack.JacksonDBCollection; +import org.mongojack.WriteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.Collections; + +public class PipelineSourceService { + private static final Logger log = LoggerFactory.getLogger(PipelineSourceService.class); + + public static final String COLLECTION = "pipeline_processor_pipelines"; + + private final JacksonDBCollection dbCollection; + + @Inject + public PipelineSourceService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + dbCollection = JacksonDBCollection.wrap( + mongoConnection.getDatabase().getCollection(COLLECTION), + PipelineSource.class, + String.class, + mapper.get()); + } + + public PipelineSource save(PipelineSource pipeline) { + final WriteResult save = dbCollection.save(pipeline); + return save.getSavedObject(); + } + + public PipelineSource load(String id) throws NotFoundException { + final PipelineSource pipeline = dbCollection.findOneById(id); + if (pipeline == null) { + throw new NotFoundException("No pipeline with id " + id); + } + return pipeline; + } + + public Collection loadAll() { + try { + final DBCursor pipelineSources = dbCollection.find(); + return Sets.newHashSet(pipelineSources.iterator()); + } catch (MongoException e) { + log.error("Unable to load pipelines", e); + return Collections.emptySet(); + } + } + + public void delete(String id) { + dbCollection.removeById(id); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java similarity index 66% rename from src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java index b4529eb9dc51..9165f2fc9eff 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/db/RuleSourceService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java @@ -1,8 +1,24 @@ -package org.graylog.plugins.messageprocessor.db; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; import com.google.common.collect.Sets; import com.mongodb.MongoException; -import org.graylog.plugins.messageprocessor.rest.RuleSource; +import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; @@ -19,7 +35,7 @@ public class RuleSourceService { private static final Logger log = LoggerFactory.getLogger(RuleSourceService.class); - public static final String COLLECTION = "message_processor_rules"; + public static final String COLLECTION = "pipeline_processor_rules"; private final JacksonDBCollection dbCollection; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java b/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java new file mode 100644 index 000000000000..0cc7f80cc0b6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import com.google.common.collect.Sets; + +import java.util.Set; + +import static java.util.Collections.emptySet; + +@AutoValue +public abstract class PipelinesChangedEvent { + + @JsonProperty + public abstract Set deletedPipelineIds(); + + @JsonProperty + public abstract Set updatedPipelineIds(); + + public static Builder builder() { + return new AutoValue_PipelinesChangedEvent.Builder().deletedPipelineIds(emptySet()).updatedPipelineIds(emptySet()); + } + + public static PipelinesChangedEvent updatedPipelineId(String id) { + return builder().updatedPipelineId(id).build(); + } + + public static PipelinesChangedEvent deletedPipelineId(String id) { + return builder().deletedPipelineId(id).build(); + } + + @JsonCreator + public static PipelinesChangedEvent create(@JsonProperty("deleted_pipeline_ids") Set deletedIds, @JsonProperty("updated_pipeline_ids") Set updatedIds) { + return builder().deletedPipelineIds(deletedIds).updatedPipelineIds(updatedIds).build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder deletedPipelineIds(Set ids); + public Builder deletedPipelineId(String id) { + return deletedPipelineIds(Sets.newHashSet(id)); + } + public abstract Builder updatedPipelineIds(Set ids); + public Builder updatedPipelineId(String id) { + return updatedPipelineIds(Sets.newHashSet(id)); + } + public abstract PipelinesChangedEvent build(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java b/src/main/java/org/graylog/plugins/pipelineprocessor/events/RulesChangedEvent.java similarity index 66% rename from src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/events/RulesChangedEvent.java index 13b26c6d3569..82f049de39d1 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/events/RulesChangedEvent.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/events/RulesChangedEvent.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.events; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java new file mode 100644 index 000000000000..49f0f6939a97 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class BooleanCoercion implements Function { + public static final String NAME = "bool"; + + private static final String VALUE = "value"; + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse("false"); + return Boolean.parseBoolean(evaluated.toString()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(object(VALUE))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java new file mode 100644 index 000000000000..597ca4f25139 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.google.common.primitives.Doubles; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class DoubleCoercion implements Function { + + public static final String NAME = "double"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public Double evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); + return (Double) firstNonNull(Doubles.tryParse(evaluated.toString()), args.evaluated(DEFAULT, context, Double.class).orElse(0d)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Double.class) + .params(of( + object(VALUE), + param().optional().name(DEFAULT).type(Double.class).build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java new file mode 100644 index 000000000000..68a2bc4da905 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java @@ -0,0 +1,57 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Optional; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class DropMessageFunction implements Function { + + public static final String NAME = "drop_message"; + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Optional message; + if (args.isPresent("message")) { + message = args.evaluated("message", context, Message.class); + } else { + message = Optional.of(context.currentMessage()); + } + message.get().setFilterOut(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .pure(true) + .returnType(Void.class) + .params(ImmutableList.of( + param().type(Message.class).optional().name("message").build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java new file mode 100644 index 000000000000..4d4acb0244e4 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Optional; + +public class HasField implements Function { + + public static final String NAME = "has_field"; + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Optional field = args.evaluated("field", context, String.class); + return context.currentMessage().hasField(field.orElse(null)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(ImmutableList.of(ParameterDescriptor.string("field"))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java new file mode 100644 index 000000000000..77002649303a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.IOState; +import org.graylog2.plugin.inputs.MessageInput; +import org.graylog2.shared.inputs.InputRegistry; + +import javax.inject.Inject; + +import static com.google.common.collect.ImmutableList.of; + +public class InputFunction implements Function { + + public static final String NAME = "input"; + + private final InputRegistry inputRegistry; + + @Inject + public InputFunction(InputRegistry inputRegistry) { + this.inputRegistry = inputRegistry; + } + + @Override + public MessageInput evaluate(FunctionArgs args, EvaluationContext context) { + final String id = args.evaluated("id", context, String.class).orElse(""); + final IOState inputState = inputRegistry.getInputState(id); + return inputState.getStoppable(); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(MessageInput.class) + .params(of(ParameterDescriptor.string("id"))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java new file mode 100644 index 000000000000..4f5def52e3fc --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java @@ -0,0 +1,56 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.collect.ImmutableList.of; +import static com.google.common.primitives.Longs.tryParse; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class LongCoercion implements Function { + + public static final String NAME = "long"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public Long evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); + final Long defaultValue = args.evaluated(DEFAULT, context, Long.class).orElse(0L); + + return firstNonNull(tryParse(evaluated.toString()), defaultValue); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Long.class) + .params(of( + object(VALUE), + param().optional().integer(DEFAULT).build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java new file mode 100644 index 000000000000..dfb3f3e87692 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java @@ -0,0 +1,55 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; + +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class SetField implements Function { + + public static final String NAME = "set_field"; + public static final String FIELD = "field"; + public static final String VALUE = "value"; + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Optional field = args.evaluated(FIELD, context, Object.class); + final Optional value = args.evaluated(VALUE, context, Object.class); + + context.currentMessage().addField(field.get().toString(), value.get()); + + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of(string(FIELD), + object(VALUE))) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java new file mode 100644 index 000000000000..75b8e5801c9d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java @@ -0,0 +1,56 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class StringCoercion implements Function { + + public static final String NAME = "string"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); + if (evaluated instanceof String) { + return (String) evaluated; + } else { + return args.evaluated(DEFAULT, context, String.class).orElse(""); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + object(VALUE), + param().optional().string(DEFAULT).build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java new file mode 100644 index 000000000000..8f66aa95981f --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java @@ -0,0 +1,45 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; + +import javax.inject.Inject; +import java.util.Map; + +public class FunctionRegistry { + + private final Map> functions; + + @Inject + public FunctionRegistry(Map> functions) { + this.functions = functions; + } + + + public Function resolve(String name) { + return functions.get(name); + } + + public Function resolveOrError(String name) { + final Function function = resolve(name); + if (function == null) { + return Function.ERROR_FUNCTION; + } + return function; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java new file mode 100644 index 000000000000..fe034a1d784d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java @@ -0,0 +1,42 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; + +import java.util.Set; + +public class ParseException extends RuntimeException { + private final Set errors; + + public ParseException(Set errors) { + this.errors = errors; + } + + public Set getErrors() { + return errors; + } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder("Errors:\n"); + for (ParseError parseError : getErrors()) { + sb.append(" ").append(parseError).append("\n"); + } + return sb.toString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java similarity index 84% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParser.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index bda58e054daf..f0ab7304be70 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.parser; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -11,46 +27,45 @@ import org.antlr.v4.runtime.tree.ParseTreeProperty; import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.apache.mina.util.IdentityHashSet; -import org.graylog.plugins.messageprocessor.ast.Pipeline; -import org.graylog.plugins.messageprocessor.ast.Rule; -import org.graylog.plugins.messageprocessor.ast.Stage; -import org.graylog.plugins.messageprocessor.ast.expressions.AndExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.ArrayExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.BinaryExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.BooleanExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.BooleanValuedFunctionWrapper; -import org.graylog.plugins.messageprocessor.ast.expressions.ComparisonExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.DoubleExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.EqualityExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.Expression; -import org.graylog.plugins.messageprocessor.ast.expressions.FieldAccessExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.FieldRefExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.FunctionExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.LogicalExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.LongExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.MapExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.MessageRefExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.NotExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.OrExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.StringExpression; -import org.graylog.plugins.messageprocessor.ast.expressions.VarRefExpression; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.messageprocessor.ast.statements.FunctionStatement; -import org.graylog.plugins.messageprocessor.ast.statements.Statement; -import org.graylog.plugins.messageprocessor.ast.statements.VarAssignStatement; -import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; -import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleType; -import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleTypes; -import org.graylog.plugins.messageprocessor.parser.errors.MissingRequiredParam; -import org.graylog.plugins.messageprocessor.parser.errors.OptionalParametersMustBeNamed; -import org.graylog.plugins.messageprocessor.parser.errors.ParseError; -import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; -import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; -import org.graylog.plugins.messageprocessor.parser.errors.WrongNumberOfArgs; -import org.graylog2.plugin.Tools; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.statements.FunctionStatement; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleTypes; +import org.graylog.plugins.pipelineprocessor.parser.errors.MissingRequiredParam; +import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; +import org.graylog.plugins.pipelineprocessor.parser.errors.WrongNumberOfArgs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,6 +128,14 @@ public List parsePipelines(String pipeline) throws ParseException { throw new ParseException(parseContext.getErrors()); } + public static String unquote(String string, char quoteChar) { + if (string.charAt(0) == quoteChar && string.charAt(string.length() - 1) == quoteChar) { + return string.substring(1, string.length() - 1); + } + return string; + } + + private class AstBuilder extends RuleLangBaseListener { private final ParseContext parseContext; @@ -136,7 +159,7 @@ public AstBuilder(ParseContext parseContext) { public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ctx) { final Pipeline.Builder builder = Pipeline.builder(); - builder.name(Tools.unquote(ctx.name.getText(), '"')); + builder.name(unquote(ctx.name.getText(), '"')); List stages = Lists.newArrayList(); for (RuleLangParser.StageDeclarationContext stage : ctx.stageDeclaration()) { final Stage.Builder stageBuilder = Stage.builder(); @@ -148,7 +171,7 @@ public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ct stageBuilder.matchAll(isAllModifier); final List ruleRefs = stage.ruleRef().stream() - .map(ruleRefContext -> Tools.unquote(ruleRefContext.name.getText(), '"')) + .map(ruleRefContext -> unquote(ruleRefContext.name.getText(), '"')) .collect(toList()); stageBuilder.ruleReferences(ruleRefs); @@ -162,7 +185,7 @@ public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ct @Override public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { final Rule.Builder ruleBuilder = Rule.builder(); - ruleBuilder.name(Tools.unquote(ctx.name.getText(), '"')); + ruleBuilder.name(unquote(ctx.name.getText(), '"')); final Expression expr = exprs.get(ctx.condition); LogicalExpression condition; @@ -190,7 +213,7 @@ public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { @Override public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { - final String name = Tools.unquote(ctx.varName.getText(), '`'); + final String name = unquote(ctx.varName.getText(), '`'); final Expression expr = exprs.get(ctx.expression()); parseContext.defineVar(name, expr); definedVars.add(name); @@ -286,7 +309,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { final Map argMap = Maps.newHashMap(); for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { - final String argName = Tools.unquote(propAssignmentContext.Identifier().getText(), '`'); + final String argName = unquote(propAssignmentContext.Identifier().getText(), '`'); final Expression argValue = exprs.get(propAssignmentContext.expression()); argMap.put(argName, argValue); } @@ -385,7 +408,7 @@ public void exitChar(RuleLangParser.CharContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { - final String text = Tools.unquote(ctx.getText(), '\"'); + final String text = unquote(ctx.getText(), '\"'); final StringExpression expr = new StringExpression(text); log.info("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -415,7 +438,7 @@ public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { public void exitMapLiteralExpr(RuleLangParser.MapLiteralExprContext ctx) { final HashMap map = Maps.newHashMap(); for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { - final String key = Tools.unquote(propAssignmentContext.Identifier().getText(), '`'); + final String key = unquote(propAssignmentContext.Identifier().getText(), '`'); final Expression value = exprs.get(propAssignmentContext.expression()); map.put(key, value); } @@ -447,7 +470,7 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { // unquote identifier if necessary - final String identifierName = Tools.unquote(ctx.Identifier().getText(), '`'); + final String identifierName = unquote(ctx.Identifier().getText(), '`'); if (!idIsFieldAccess && !definedVars.contains(identifierName)) { parseContext.addError(new UndeclaredVariable(ctx)); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java new file mode 100644 index 000000000000..35c786593b3b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java @@ -0,0 +1,49 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleArgumentType extends ParseError { + private final FunctionExpression functionExpression; + private final ParameterDescriptor p; + private final Expression argExpr; + + public IncompatibleArgumentType(RuleLangParser.FunctionCallContext ctx, + FunctionExpression functionExpression, + ParameterDescriptor p, + Expression argExpr) { + super("incompatible_argument_type", ctx); + this.functionExpression = functionExpression; + this.p = p; + this.argExpr = argExpr; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + p.type().getSimpleName() + + " for argument " + p.name() + + " but found " + argExpr.getType().getSimpleName() + + " in call to function " + functionExpression.getFunction().descriptor().name() + + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java new file mode 100644 index 000000000000..980dff830c9d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java @@ -0,0 +1,37 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleType extends ParseError { + private final Class expected; + private final Class actual; + + public IncompatibleType(RuleLangParser.MessageRefContext ctx, Class expected, Class actual) { + super("incompatible_type", ctx); + this.expected = expected; + this.actual = actual; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java new file mode 100644 index 000000000000..757c7399d853 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java @@ -0,0 +1,45 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleTypes extends ParseError { + private final RuleLangParser.ExpressionContext ctx; + private final BinaryExpression binaryExpr; + + public IncompatibleTypes(RuleLangParser.ExpressionContext ctx, BinaryExpression binaryExpr) { + super("incompatible_types", ctx); + this.ctx = ctx; + this.binaryExpr = binaryExpr; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); + } + + private String exprString(Expression e) { + return "(" + e.toString() + ") : " + e.getType().getSimpleName(); + } + + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java new file mode 100644 index 000000000000..2ab56d9f6e5a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class MissingRequiredParam extends ParseError { + private final Function function; + private final ParameterDescriptor param; + + public MissingRequiredParam(RuleLangParser.FunctionCallContext ctx, + Function function, + ParameterDescriptor param) { + super("missing_required_param", ctx); + this.function = function; + this.param = param; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Missing required parameter " + param.name() + + " of type " + param.type().getSimpleName() + + " in call to function " + function.descriptor().name() + + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java new file mode 100644 index 000000000000..d3e9b9eaea71 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class OptionalParametersMustBeNamed extends ParseError { + private final Function function; + + public OptionalParametersMustBeNamed(RuleLangParser.FunctionCallContext ctx, Function function) { + super("must_name_optional_params", ctx); + this.function = function; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Function " + function.descriptor().name() + " has optional parameters, must use named parameters to call" + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/ParseError.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java similarity index 59% rename from src/main/java/org/graylog/plugins/messageprocessor/parser/errors/ParseError.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java index 3bae61442846..706b04859f0c 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/parser/errors/ParseError.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.parser.errors; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java new file mode 100644 index 000000000000..8872279b8037 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java @@ -0,0 +1,35 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class UndeclaredFunction extends ParseError { + private final RuleLangParser.FunctionCallContext ctx; + + public UndeclaredFunction(RuleLangParser.FunctionCallContext ctx) { + super("undeclared_function", ctx); + this.ctx = ctx; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Unknown function " + ctx.funcName.getText() + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java new file mode 100644 index 000000000000..7f75287699fc --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java @@ -0,0 +1,39 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class UndeclaredVariable extends ParseError { + + @JsonIgnore + private final RuleLangParser.IdentifierContext ctx; + + public UndeclaredVariable(RuleLangParser.IdentifierContext ctx) { + super("undeclared_variable", ctx); + this.ctx = ctx; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Undeclared variable " + ctx.Identifier().getText() + positionString(); + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java new file mode 100644 index 000000000000..4e1ef0e6b004 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class WrongNumberOfArgs extends ParseError { + private final Function function; + private final int argCount; + + public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, + Function function, + int argCount) { + super("wrong_number_of_arguments", ctx); + this.function = function; + this.argCount = argCount; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected " + function.descriptor().params().size() + + " arguments but found " + argCount + + " in call to function " + function.descriptor().name() + + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java similarity index 80% rename from src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java index e98560b36cec..f56ffb42d23b 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.processors; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; @@ -9,14 +25,14 @@ import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.Rule; -import org.graylog.plugins.messageprocessor.ast.statements.Statement; -import org.graylog.plugins.messageprocessor.db.RuleSourceService; -import org.graylog.plugins.messageprocessor.events.RulesChangedEvent; -import org.graylog.plugins.messageprocessor.parser.ParseException; -import org.graylog.plugins.messageprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.messageprocessor.rest.RuleSource; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java new file mode 100644 index 000000000000..db2be0539d93 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -0,0 +1,124 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.eventbus.EventBus; +import com.wordnik.swagger.annotations.Api; +import com.wordnik.swagger.annotations.ApiOperation; +import com.wordnik.swagger.annotations.ApiParam; +import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog2.database.NotFoundException; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.shared.rest.resources.RestResource; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Collection; + +@Api(value = "Pipelines", description = "Pipelines for the pipeline message processor") +@Path("/system/pipelines/pipeline") +public class PipelineResource extends RestResource implements PluginRestResource { + + private static final Logger log = LoggerFactory.getLogger(PipelineResource.class); + + private final PipelineSourceService pipelineSourceService; + private final PipelineRuleParser pipelineRuleParser; + private final EventBus clusterBus; + + @Inject + public PipelineResource(PipelineSourceService pipelineSourceService, + PipelineRuleParser pipelineRuleParser, + @ClusterEventBus EventBus clusterBus) { + this.pipelineSourceService = pipelineSourceService; + this.pipelineRuleParser = pipelineRuleParser; + this.clusterBus = clusterBus; + } + + @ApiOperation(value = "Create a processing pipeline from source", notes = "") + @Consumes(MediaType.TEXT_PLAIN) + @Produces(MediaType.APPLICATION_JSON) + @POST + public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull String pipelineSource) throws ParseException { + try { + pipelineRuleParser.parsePipelines(pipelineSource); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final PipelineSource newPipelineSource = PipelineSource.builder() + .source(pipelineSource) + .createdAt(DateTime.now()) + .modifiedAt(DateTime.now()) + .build(); + final PipelineSource save = pipelineSourceService.save(newPipelineSource); + clusterBus.post(PipelinesChangedEvent.updatedPipelineId(save.id())); + log.info("Created new pipeline {}", save); + return save; + } + + @ApiOperation(value = "Get all processing pipelines") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @GET + public Collection getAll() { + return pipelineSourceService.loadAll(); + } + + @ApiOperation(value = "Get a processing pipeline", notes = "It can take up to a second until the change is applied") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}") + @GET + public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + return pipelineSourceService.load(id); + } + + @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}") + @PUT + public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, + @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { + final PipelineSource pipelineSource = pipelineSourceService.load(id); + try { + pipelineRuleParser.parsePipelines(update.source()); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final PipelineSource toSave = pipelineSource.toBuilder() + .source(update.source()) + .modifiedAt(DateTime.now()) + .build(); + final PipelineSource savedPipeline = pipelineSourceService.save(toSave); + clusterBus.post(PipelinesChangedEvent.updatedPipelineId(savedPipeline.id())); + + return savedPipeline; + } + + @ApiOperation(value = "Delete a processing pipeline", notes = "It can take up to a second until the change is applied") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}") + @DELETE + public void delete(@ApiParam(name = "id") @PathParam("id") String id) { + pipelineSourceService.delete(id); + clusterBus.post(PipelinesChangedEvent.deletedPipelineId(id)); + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java new file mode 100644 index 000000000000..80213d850ff6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -0,0 +1,61 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.joda.time.DateTime; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineSource { + + @JsonProperty("_id") + @Nullable + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + public abstract DateTime createdAt(); + + @JsonProperty + public abstract DateTime modifiedAt(); + + public static Builder builder() { + return new AutoValue_PipelineSource.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static PipelineSource create(@JsonProperty("id") @ObjectId @Nullable String id, + @JsonProperty("source") String source, + @JsonProperty("created_at") DateTime createdAt, + @JsonProperty("modified_at") DateTime modifiedAt) { + return builder() + .id(id) + .source(source) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract PipelineSource build(); + + public abstract Builder id(String id); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + } +} diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java similarity index 72% rename from src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 6c9b3627704b..bf9d2349c102 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/MessageProcessorRuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -1,13 +1,29 @@ -package org.graylog.plugins.messageprocessor.rest; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; import com.google.common.eventbus.EventBus; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; -import org.graylog.plugins.messageprocessor.db.RuleSourceService; -import org.graylog.plugins.messageprocessor.events.RulesChangedEvent; -import org.graylog.plugins.messageprocessor.parser.ParseException; -import org.graylog.plugins.messageprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog2.database.NotFoundException; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.rest.PluginRestResource; @@ -31,20 +47,20 @@ import javax.ws.rs.core.Response; import java.util.Collection; -@Api(value = "MessageProcessing", description = "Message processing pipeline") -@Path("/messageprocessors") -public class MessageProcessorRuleResource extends RestResource implements PluginRestResource { +@Api(value = "Pipeline Rules", description = "Rules for the pipeline message processor") +@Path("/system/pipelines/rule") +public class RuleResource extends RestResource implements PluginRestResource { - private static final Logger log = LoggerFactory.getLogger(MessageProcessorRuleResource.class); + private static final Logger log = LoggerFactory.getLogger(RuleResource.class); private final RuleSourceService ruleSourceService; private final PipelineRuleParser pipelineRuleParser; private final EventBus clusterBus; @Inject - public MessageProcessorRuleResource(RuleSourceService ruleSourceService, - PipelineRuleParser pipelineRuleParser, - @ClusterEventBus EventBus clusterBus) { + public RuleResource(RuleSourceService ruleSourceService, + PipelineRuleParser pipelineRuleParser, + @ClusterEventBus EventBus clusterBus) { this.ruleSourceService = ruleSourceService; this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; @@ -54,7 +70,6 @@ public MessageProcessorRuleResource(RuleSourceService ruleSourceService, @ApiOperation(value = "Create a processing rule from source", notes = "") @Consumes(MediaType.TEXT_PLAIN) @Produces(MediaType.APPLICATION_JSON) - @Path("/rule") @POST public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull String ruleSource) throws ParseException { try { @@ -76,7 +91,6 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No @ApiOperation(value = "Get all processing rules") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("/rule") @GET public Collection getAll() { return ruleSourceService.loadAll(); @@ -85,7 +99,7 @@ public Collection getAll() { @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("/rule/{id}") + @Path("{id}") @GET public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { return ruleSourceService.load(id); @@ -94,7 +108,7 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("/rule/{id}") + @Path("{id}") @PUT public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { @@ -117,7 +131,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("/rule/{id}") + @Path("{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) { ruleSourceService.delete(id); diff --git a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java similarity index 67% rename from src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index 58a885860d12..e1b0ae243787 100644 --- a/src/main/java/org/graylog/plugins/messageprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -1,4 +1,20 @@ -package org.graylog.plugins.messageprocessor.rest; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonCreator; @@ -46,10 +62,6 @@ public static RuleSource create(@JsonProperty("_id") @ObjectId @Nullable String .build(); } - public RuleSource withId(String savedId) { - return toBuilder().id(savedId).build(); - } - @AutoValue.Builder public abstract static class Builder { public abstract RuleSource build(); diff --git a/src/test/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java similarity index 88% rename from src/test/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParserTest.java rename to src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index ca8fc89a47ed..fb94acff8a53 100644 --- a/src/test/java/org/graylog/plugins/messageprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -1,27 +1,43 @@ -package org.graylog.plugins.messageprocessor.parser; +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; import com.google.common.base.Charsets; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; -import org.graylog.plugins.messageprocessor.EvaluationContext; -import org.graylog.plugins.messageprocessor.ast.Pipeline; -import org.graylog.plugins.messageprocessor.ast.Rule; -import org.graylog.plugins.messageprocessor.ast.Stage; -import org.graylog.plugins.messageprocessor.ast.functions.Function; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.messageprocessor.ast.functions.FunctionDescriptor; -import org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.messageprocessor.ast.statements.Statement; -import org.graylog.plugins.messageprocessor.functions.HasField; -import org.graylog.plugins.messageprocessor.functions.LongCoercion; -import org.graylog.plugins.messageprocessor.functions.SetField; -import org.graylog.plugins.messageprocessor.functions.StringCoercion; -import org.graylog.plugins.messageprocessor.parser.errors.IncompatibleArgumentType; -import org.graylog.plugins.messageprocessor.parser.errors.OptionalParametersMustBeNamed; -import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredFunction; -import org.graylog.plugins.messageprocessor.parser.errors.UndeclaredVariable; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.functions.HasField; +import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; +import org.graylog.plugins.pipelineprocessor.functions.SetField; +import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; import org.graylog2.plugin.Message; import org.joda.time.DateTime; import org.junit.After; @@ -46,7 +62,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.messageprocessor.ast.functions.ParameterDescriptor.param; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/basicRule.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/booleanValuedFunctionAsCondition.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanValuedFunctionAsCondition.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/booleanValuedFunctionAsCondition.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanValuedFunctionAsCondition.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/declaredFunction.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/declaredFunction.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/declaredFunction.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/declaredFunction.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/inferVariableType.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/inferVariableType.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/inferVariableType.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/inferVariableType.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/invalidArgType.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgType.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/invalidArgType.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgType.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/mapArrayLiteral.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mapArrayLiteral.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/mapArrayLiteral.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mapArrayLiteral.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRef.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/messageRefQuotedField.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalArguments.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalArguments.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalArguments.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalArguments.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalParamsMustBeNamed.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalParamsMustBeNamed.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/optionalParamsMustBeNamed.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalParamsMustBeNamed.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/pipelineDeclaration.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/pipelineDeclaration.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/pipelineDeclaration.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/pipelineDeclaration.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/positionalArguments.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/positionalArguments.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/positionalArguments.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/positionalArguments.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/singleArgFunction.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/singleArgFunction.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/singleArgFunction.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/singleArgFunction.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/typedFieldAccess.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/typedFieldAccess.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredFunction.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredFunction.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredFunction.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredFunction.txt diff --git a/src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredIdentifier.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/messageprocessor/parser/undeclaredIdentifier.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredIdentifier.txt diff --git a/src/web/PipelinesActions.jsx b/src/web/PipelinesActions.jsx new file mode 100644 index 000000000000..9d0dab1f121e --- /dev/null +++ b/src/web/PipelinesActions.jsx @@ -0,0 +1,11 @@ +import Reflux from 'reflux'; + +const PipelinesActions = Reflux.createActions({ + 'delete': { asyncResult: true }, + 'list': { asyncResult: true }, + 'get': { asyncResult: true }, + 'save': { asyncResult: true }, + 'update': { asyncResult: true }, +}); + +export default PipelinesActions; \ No newline at end of file diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx new file mode 100644 index 000000000000..38507161e220 --- /dev/null +++ b/src/web/PipelinesPage.jsx @@ -0,0 +1,50 @@ +import React from 'react'; +import Reflux from 'reflux'; +import { Row, Col } from 'react-bootstrap'; +import PageHeader from 'components/common/PageHeader'; +import Spinner from 'components/common/Spinner'; + +import PipelinesActions from 'PipelinesActions'; +import PipelinesStore from 'PipelinesStore'; + +const PipelinesPage = React.createClass({ + mixins: [Reflux.connect(PipelinesStore)], + + getInitialState() { + return { + pipelines: undefined, + } + }, + + componentDidMount() { + this.loadData(); + }, + + loadData() { + PipelinesActions.list.triggerPromise().then((pipelines) => { + this.setState({pipelines: pipelines}); + }); + }, + + render() { + let content; + if (!this.state.pipelines) { + content = ; + } else { + content = this.state.pipelines.length; + } + return ( + + + Processing pipelines + + + + {content} + + + ); + }, +}); + +export default PipelinesPage; diff --git a/src/web/PipelinesStore.js b/src/web/PipelinesStore.js new file mode 100644 index 000000000000..36befe213f0e --- /dev/null +++ b/src/web/PipelinesStore.js @@ -0,0 +1,60 @@ +import Reflux from 'reflux'; + +import PipelinesActions from 'PipelinesActions'; + +import UserNotification from 'util/UserNotification'; +import URLUtils from 'util/URLUtils'; +import fetch from 'logic/rest/FetchProvider'; + +const urlPrefix = '/plugins/org.graylog.plugins.pipelineprocessor'; + +const PipelinesStore = Reflux.createStore({ + listenables: [PipelinesActions], + pipelines: undefined, + + init() { + this.list().then((pipelines) => { + this.pipelines = pipelines; + this.trigger({pipelines: pipelines}); + }); + }, + + getInitialState() { + return { + pipelines: this.pipelines, + } + }, + + list() { + const failCallback = (error) => { + UserNotification.error('Fetching pipelines failed with status: ' + error.message, + 'Could not retrieve processing pipelines'); + }; + + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline'); + const promise = fetch('GET', url).then((response) => { + return response.pipelines; + }, failCallback); + + PipelinesActions.list.promise(promise); + + return promise; + }, + + get(pipelineId) { + + }, + + save(pipelineSource) { + + }, + + update(pipelineId) { + + }, + delete(pipelineId) { + + }, +}); + +export default PipelinesStore; \ No newline at end of file diff --git a/src/web/index.jsx b/src/web/index.jsx index 1bf813e31222..8b5793ec9216 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -1,21 +1,13 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; +import PipelinesPage from './PipelinesPage'; PluginStore.register(new PluginManifest(packageJson, { - /* This is the place where you define which entities you are providing to the web interface. - Right now you can add routes and navigation elements to it. + routes: [ + {path: '/system/pipelines', component: PipelinesPage}, + ], - Examples: */ - - // Adding a route to /sample, rendering YourReactComponent when called: - - // routes: [ - // { path: '/sample', component: YourReactComponent }, - // ], - - // Adding an element to the top navigation pointing to /sample named "Sample": - - // navigation: [ - // { path: '/sample', description: 'Sample' }, - // ] + systemnavigation: [ + {path: '/system/pipelines', description: 'Pipelines'}, + ], })); From b5a470b635a480b3030af097ebeb57d4a9e05a60 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 21 Jan 2016 17:56:29 +0100 Subject: [PATCH 030/521] update to latest graylog-web-plugin version --- build.config.js | 6 ++++++ package.json | 2 +- webpack.config.js | 13 +++---------- 3 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 build.config.js diff --git a/build.config.js b/build.config.js new file mode 100644 index 000000000000..9a96101d263f --- /dev/null +++ b/build.config.js @@ -0,0 +1,6 @@ +const path = require('path'); + +module.exports = { + // Make sure that this is the correct path to the web interface part of the Graylog server repository. + web_src_path: path.resolve(__dirname, '../graylog2-server/graylog2-web-interface'), +}; \ No newline at end of file diff --git a/package.json b/package.json index 42a0201cd56d..d6c0f56be42b 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "babel-core": "^5.8.25", "babel-loader": "^5.3.2", "graylog-web-manifests": "2.0.0-SNAPSHOT-1", - "graylog-web-plugin": "~0.0.15", + "graylog-web-plugin": "~0.0.17", "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", "webpack": "^1.12.2" diff --git a/webpack.config.js b/webpack.config.js index 4ac151d85982..2daceb1b2ed9 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,14 +1,7 @@ const PluginWebpackConfig = require('graylog-web-plugin').PluginWebpackConfig; +const loadBuildConfig = require('graylog-web-plugin').loadBuildConfig; const path = require('path'); -const ROOT_PATH = path.resolve(__dirname); -const BUILD_PATH = path.resolve(ROOT_PATH, 'build'); -const ENTRY_PATH = path.resolve(ROOT_PATH, 'src/web/index.jsx'); -module.exports = new PluginWebpackConfig('org.graylog.plugins.pipelineprocessor.PipelineProcessor', { - root_path: ROOT_PATH, - build_path: BUILD_PATH, - entry_path: ENTRY_PATH -}, -{ +module.exports = new PluginWebpackConfig('${package}.${pluginClassName}', loadBuildConfig(path.resolve(__dirname, './build.config')), { // Here goes your additional webpack configuration. -}); +}); \ No newline at end of file From a78a9f443aeea07c292753f15cd3856991f942e6 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 21 Jan 2016 18:44:38 +0100 Subject: [PATCH 031/521] updated to latest graylog-web-plugin - initial dummy store and dummy page --- package.json | 1 + .../PipelineProcessorMetaData.java | 19 ++--- .../PipelineProcessorModule.java | 67 ++++++++++------- .../pipelineprocessor/ProcessorPlugin.java | 36 --------- .../ProcessorPluginMetaData.java | 67 ----------------- .../ProcessorPluginModule.java | 73 ------------------- .../rest/PipelineResource.java | 10 ++- .../pipelineprocessor/rest/RuleResource.java | 10 ++- src/web/PipelinesStore.js | 6 -- webpack.config.js | 9 ++- 10 files changed, 69 insertions(+), 229 deletions(-) delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java diff --git a/package.json b/package.json index d6c0f56be42b..fbaa5dd239e1 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "graylog-web-plugin": "~0.0.17", "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", + "ts-loader": "^0.8.0", "webpack": "^1.12.2" } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index a4bbfe6c4987..824b5e342572 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -16,12 +16,12 @@ */ package org.graylog.plugins.pipelineprocessor; +import com.google.common.collect.Sets; import org.graylog2.plugin.PluginMetaData; import org.graylog2.plugin.ServerStatus; import org.graylog2.plugin.Version; import java.net.URI; -import java.util.Collections; import java.util.Set; /** @@ -30,44 +30,41 @@ public class PipelineProcessorMetaData implements PluginMetaData { @Override public String getUniqueId() { - return "org.graylog.plugins.pipelineprocessor.PipelineProcessorPlugin"; + return "org.graylog.plugins.pipelineprocessor.ProcessorPlugin"; } @Override public String getName() { - return "PipelineProcessor"; + return "Pipeline Processor Plugin"; } @Override public String getAuthor() { - // TODO Insert author name - return "PipelineProcessor author"; + return "Graylog, Inc"; } @Override public URI getURL() { - // TODO Insert correct plugin website return URI.create("https://www.graylog.org/"); } @Override public Version getVersion() { - return new Version(1, 0, 0); + return new Version(1, 0, 0, "SNAPSHOT"); } @Override public String getDescription() { - // TODO Insert correct plugin description - return "Description of PipelineProcessor plugin"; + return "Pluggable pipeline processing framework"; } @Override public Version getRequiredVersion() { - return new Version(1, 2, 0); + return new Version(2, 0, 0); } @Override public Set getRequiredCapabilities() { - return Collections.emptySet(); + return Sets.newHashSet(ServerStatus.Capability.SERVER); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index f4629712467c..3c89cadec75e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -16,21 +16,28 @@ */ package org.graylog.plugins.pipelineprocessor; +import com.google.inject.Binder; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; +import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; +import org.graylog.plugins.pipelineprocessor.functions.DropMessageFunction; +import org.graylog.plugins.pipelineprocessor.functions.HasField; +import org.graylog.plugins.pipelineprocessor.functions.InputFunction; +import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; +import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.processors.NaiveRuleProcessor; +import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; +import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; import java.util.Collections; import java.util.Set; -/** - * Extend the PluginModule abstract class here to add you plugin to the system. - */ public class PipelineProcessorModule extends PluginModule { - /** - * Returns all configuration beans required by this plugin. - * - * Implementing this method is optional. The default method returns an empty {@link Set}. - */ + @Override public Set getConfigBeans() { return Collections.emptySet(); @@ -38,23 +45,31 @@ public Set getConfigBeans() { @Override protected void configure() { - /* - * Register your plugin types here. - * - * Examples: - * - * addMessageInput(Class); - * addMessageFilter(Class); - * addMessageOutput(Class); - * addPeriodical(Class); - * addAlarmCallback(Class); - * addInitializer(Class); - * addRestResource(Class); - * - * - * Add all configuration beans returned by getConfigBeans(): - * - * addConfigBeans(); - */ + addMessageProcessor(NaiveRuleProcessor.class); + addRestResource(RuleResource.class); + addRestResource(PipelineResource.class); + + // built-in functions + addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); + addMessageProcessorFunction(DoubleCoercion.NAME, DoubleCoercion.class); + addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); + addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); + + addMessageProcessorFunction(HasField.NAME, HasField.class); + addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); + addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); + } + + protected void addMessageProcessorFunction(String name, Class> functionClass) { + addMessageProcessorFunction(binder(), name, functionClass); + } + + public static MapBinder> processorFunctionBinder(Binder binder) { + return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral>() {}); + } + + public static void addMessageProcessorFunction(Binder binder, String name, Class> functionClass) { + processorFunctionBinder(binder).addBinding(name).to(functionClass); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java deleted file mode 100644 index b228499e0845..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPlugin.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor; - -import org.graylog2.plugin.Plugin; -import org.graylog2.plugin.PluginMetaData; -import org.graylog2.plugin.PluginModule; - -import java.util.Collection; -import java.util.Collections; - -public class ProcessorPlugin implements Plugin { - @Override - public PluginMetaData metadata() { - return new ProcessorPluginMetaData(); - } - - @Override - public Collection modules () { - return Collections.singletonList(new ProcessorPluginModule()); - } -} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java deleted file mode 100644 index 541c00a89f6c..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginMetaData.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor; - -import com.google.common.collect.Sets; -import org.graylog2.plugin.PluginMetaData; -import org.graylog2.plugin.ServerStatus; -import org.graylog2.plugin.Version; - -import java.net.URI; -import java.util.Set; - -public class ProcessorPluginMetaData implements PluginMetaData { - @Override - public String getUniqueId() { - return "org.graylog.plugins.pipelineprocessor.ProcessorPlugin"; - } - - @Override - public String getName() { - return "Pipeline Processor Plugin"; - } - - @Override - public String getAuthor() { - return "Graylog, Inc"; - } - - @Override - public URI getURL() { - return URI.create("https://www.graylog.org/"); - } - - @Override - public Version getVersion() { - return new Version(1, 0, 0, "SNAPSHOT"); - } - - @Override - public String getDescription() { - return "Pluggable pipeline processing framework"; - } - - @Override - public Version getRequiredVersion() { - return new Version(2, 0, 0); - } - - @Override - public Set getRequiredCapabilities() { - return Sets.newHashSet(ServerStatus.Capability.SERVER); - } -} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java deleted file mode 100644 index fc6cff84b02f..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ProcessorPluginModule.java +++ /dev/null @@ -1,73 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor; - -import com.google.inject.Binder; -import com.google.inject.TypeLiteral; -import com.google.inject.multibindings.MapBinder; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; -import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; -import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; -import org.graylog.plugins.pipelineprocessor.functions.DropMessageFunction; -import org.graylog.plugins.pipelineprocessor.functions.HasField; -import org.graylog.plugins.pipelineprocessor.functions.InputFunction; -import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; -import org.graylog.plugins.pipelineprocessor.processors.NaiveRuleProcessor; -import org.graylog.plugins.pipelineprocessor.rest.RuleResource; -import org.graylog2.plugin.PluginConfigBean; -import org.graylog2.plugin.PluginModule; - -import java.util.Collections; -import java.util.Set; - -public class ProcessorPluginModule extends PluginModule { - - @Override - public Set getConfigBeans() { - return Collections.emptySet(); - } - - @Override - protected void configure() { - addMessageProcessor(NaiveRuleProcessor.class); - addRestResource(RuleResource.class); - - // built-in functions - addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); - addMessageProcessorFunction(DoubleCoercion.NAME, DoubleCoercion.class); - addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); - addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); - - addMessageProcessorFunction(HasField.NAME, HasField.class); - addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); - addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); - } - - protected void addMessageProcessorFunction(String name, Class> functionClass) { - addMessageProcessorFunction(binder(), name, functionClass); - } - - public static MapBinder> processorFunctionBinder(Binder binder) { - return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral>() {}); - } - - public static void addMessageProcessorFunction(Binder binder, String name, Class> functionClass) { - processorFunctionBinder(binder).addBinding(name).to(functionClass); - - } -} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index db2be0539d93..66c5e7247479 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -32,7 +32,7 @@ import java.util.Collection; @Api(value = "Pipelines", description = "Pipelines for the pipeline message processor") -@Path("/system/pipelines/pipeline") +@Path("/system/pipelines") public class PipelineResource extends RestResource implements PluginRestResource { private static final Logger log = LoggerFactory.getLogger(PipelineResource.class); @@ -54,6 +54,7 @@ public PipelineResource(PipelineSourceService pipelineSourceService, @Consumes(MediaType.TEXT_PLAIN) @Produces(MediaType.APPLICATION_JSON) @POST + @Path("/pipeline") public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull String pipelineSource) throws ParseException { try { pipelineRuleParser.parsePipelines(pipelineSource); @@ -75,6 +76,7 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @GET + @Path("/pipeline") public Collection getAll() { return pipelineSourceService.loadAll(); } @@ -82,7 +84,7 @@ public Collection getAll() { @ApiOperation(value = "Get a processing pipeline", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("{id}") + @Path("/pipeline/{id}") @GET public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { return pipelineSourceService.load(id); @@ -91,7 +93,7 @@ public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) thr @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("{id}") + @Path("/pipeline/{id}") @PUT public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { @@ -114,7 +116,7 @@ public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiOperation(value = "Delete a processing pipeline", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("{id}") + @Path("/pipeline/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) { pipelineSourceService.delete(id); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index bf9d2349c102..0e44bbdbe5d5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -48,7 +48,7 @@ import java.util.Collection; @Api(value = "Pipeline Rules", description = "Rules for the pipeline message processor") -@Path("/system/pipelines/rule") +@Path("/system/pipelines") public class RuleResource extends RestResource implements PluginRestResource { private static final Logger log = LoggerFactory.getLogger(RuleResource.class); @@ -71,6 +71,7 @@ public RuleResource(RuleSourceService ruleSourceService, @Consumes(MediaType.TEXT_PLAIN) @Produces(MediaType.APPLICATION_JSON) @POST + @Path("/rule") public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull String ruleSource) throws ParseException { try { pipelineRuleParser.parseRule(ruleSource); @@ -92,6 +93,7 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @GET + @Path("/rule") public Collection getAll() { return ruleSourceService.loadAll(); } @@ -99,7 +101,7 @@ public Collection getAll() { @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("{id}") + @Path("/rule/{id}") @GET public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { return ruleSourceService.load(id); @@ -108,7 +110,7 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("{id}") + @Path("/rule/{id}") @PUT public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { @@ -131,7 +133,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - @Path("{id}") + @Path("/rule/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) { ruleSourceService.delete(id); diff --git a/src/web/PipelinesStore.js b/src/web/PipelinesStore.js index 36befe213f0e..917cd5b023bc 100644 --- a/src/web/PipelinesStore.js +++ b/src/web/PipelinesStore.js @@ -19,12 +19,6 @@ const PipelinesStore = Reflux.createStore({ }); }, - getInitialState() { - return { - pipelines: this.pipelines, - } - }, - list() { const failCallback = (error) => { UserNotification.error('Fetching pipelines failed with status: ' + error.message, diff --git a/webpack.config.js b/webpack.config.js index 2daceb1b2ed9..1b5effec7f8a 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -2,6 +2,11 @@ const PluginWebpackConfig = require('graylog-web-plugin').PluginWebpackConfig; const loadBuildConfig = require('graylog-web-plugin').loadBuildConfig; const path = require('path'); -module.exports = new PluginWebpackConfig('${package}.${pluginClassName}', loadBuildConfig(path.resolve(__dirname, './build.config')), { - // Here goes your additional webpack configuration. +module.exports = new PluginWebpackConfig('org.graylog.plugins.pipelineprocessor.PipelineProcessor', loadBuildConfig(path.resolve(__dirname, './build.config')), { + loaders: [ + { test: /\.ts$/, loader: 'babel-loader!ts-loader', exclude: /node_modules|\.node_cache/ }, + ], + resolve: { + extensions: ['.ts'], + } }); \ No newline at end of file From 50eade7336bd986369658662469610eefe15dad9 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 22 Jan 2016 13:10:16 +0100 Subject: [PATCH 032/521] Move frontend-maven-plugin into web-interface-build profile --- pom.xml | 92 +++++++++++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/pom.xml b/pom.xml index 7a636f931142..18a6ffeace62 100644 --- a/pom.xml +++ b/pom.xml @@ -223,45 +223,6 @@ com.mycila license-maven-plugin - - com.github.eirslett - frontend-maven-plugin - 0.0.26 - - - - install node and npm - - install-node-and-npm - - - v0.12.5 - 2.11.2 - - - - - npm install - - npm - - - - install - - - - - npm run build - - npm - - - run build - - - - org.apache.maven.plugins maven-shade-plugin @@ -363,4 +324,57 @@ + + + web-interface-build + + + !skip.web.build + + + + + + com.github.eirslett + frontend-maven-plugin + 0.0.26 + + + + install node and npm + + install-node-and-npm + + + v0.12.5 + 2.11.2 + + + + + npm install + + npm + + + + install + + + + + npm run build + + npm + + + run build + + + + + + + + From 46749159524bf4c970f2174d7966e0995b72d97f Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 22 Jan 2016 13:11:33 +0100 Subject: [PATCH 033/521] prevent loading pipelines too early - in plugins the store's init method is called early so we shouldn't use that --- src/web/PipelinesPage.jsx | 4 +--- src/web/PipelinesStore.js | 18 ++++-------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx index 38507161e220..9b02fa1c5f89 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/PipelinesPage.jsx @@ -21,9 +21,7 @@ const PipelinesPage = React.createClass({ }, loadData() { - PipelinesActions.list.triggerPromise().then((pipelines) => { - this.setState({pipelines: pipelines}); - }); + PipelinesActions.list(); }, render() { diff --git a/src/web/PipelinesStore.js b/src/web/PipelinesStore.js index 917cd5b023bc..eba3666b6848 100644 --- a/src/web/PipelinesStore.js +++ b/src/web/PipelinesStore.js @@ -12,13 +12,6 @@ const PipelinesStore = Reflux.createStore({ listenables: [PipelinesActions], pipelines: undefined, - init() { - this.list().then((pipelines) => { - this.pipelines = pipelines; - this.trigger({pipelines: pipelines}); - }); - }, - list() { const failCallback = (error) => { UserNotification.error('Fetching pipelines failed with status: ' + error.message, @@ -26,14 +19,11 @@ const PipelinesStore = Reflux.createStore({ }; const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline'); - const promise = fetch('GET', url).then((response) => { - return response.pipelines; + return fetch('GET', url).then((response) => { + this.pipelines = response; + this.trigger({pipelines: response}); }, failCallback); - - PipelinesActions.list.promise(promise); - - return promise; - }, + }, get(pipelineId) { From 65be9d5ecef875ce55d6f01ee2654790cabf9b1b Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 22 Jan 2016 14:20:56 +0100 Subject: [PATCH 034/521] load data in pipelines page and render subcomponents with props --- src/web/PipelinesComponent.jsx | 24 +++++++++++++++++++ src/web/PipelinesPage.jsx | 30 +++++++++++++++-------- src/web/RulesActions.jsx | 11 +++++++++ src/web/RulesComponent.jsx | 24 +++++++++++++++++++ src/web/RulesStore.js | 44 ++++++++++++++++++++++++++++++++++ 5 files changed, 123 insertions(+), 10 deletions(-) create mode 100644 src/web/PipelinesComponent.jsx create mode 100644 src/web/RulesActions.jsx create mode 100644 src/web/RulesComponent.jsx create mode 100644 src/web/RulesStore.js diff --git a/src/web/PipelinesComponent.jsx b/src/web/PipelinesComponent.jsx new file mode 100644 index 000000000000..a1e8e9071fd1 --- /dev/null +++ b/src/web/PipelinesComponent.jsx @@ -0,0 +1,24 @@ +import React, {PropTypes} from 'react'; +import Reflux from 'reflux'; +import { Row, Col } from 'react-bootstrap'; + +import PipelinesActions from 'PipelinesActions'; + +const PipelinesComponent = React.createClass({ + propTypes: { + pipelines: PropTypes.array.isRequired, + }, + + render() { + return ( + + +

Pipelines

+

{this.props.pipelines.length}

+ + + ); + }, +}); + +export default PipelinesComponent; \ No newline at end of file diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx index 9b02fa1c5f89..81d6e143a66e 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/PipelinesPage.jsx @@ -1,23 +1,34 @@ -import React from 'react'; +import React, {PropTypes} from 'react'; import Reflux from 'reflux'; + import { Row, Col } from 'react-bootstrap'; import PageHeader from 'components/common/PageHeader'; import Spinner from 'components/common/Spinner'; +import CurrentUserStore from 'stores/users/CurrentUserStore'; + import PipelinesActions from 'PipelinesActions'; import PipelinesStore from 'PipelinesStore'; +import PipelinesComponent from 'PipelinesComponent'; + +import RulesActions from 'RulesActions'; +import RulesStore from 'RulesStore'; +import RulesComponent from 'RulesComponent'; + const PipelinesPage = React.createClass({ - mixins: [Reflux.connect(PipelinesStore)], + mixins: [Reflux.connect(CurrentUserStore), Reflux.connect(PipelinesStore), Reflux.connect(RulesStore)], getInitialState() { return { pipelines: undefined, + rules: undefined, } }, componentDidMount() { - this.loadData(); + PipelinesActions.list(); + RulesActions.list(); }, loadData() { @@ -26,21 +37,20 @@ const PipelinesPage = React.createClass({ render() { let content; - if (!this.state.pipelines) { + if (!this.state.pipelines || !this.state.rules) { content = ; } else { - content = this.state.pipelines.length; + content = [ + , + + ]; } return ( Processing pipelines - - - {content} - - + {content} ); }, }); diff --git a/src/web/RulesActions.jsx b/src/web/RulesActions.jsx new file mode 100644 index 000000000000..6649379389c0 --- /dev/null +++ b/src/web/RulesActions.jsx @@ -0,0 +1,11 @@ +import Reflux from 'reflux'; + +const RulesActions = Reflux.createActions({ + 'delete': { asyncResult: true }, + 'list': { asyncResult: true }, + 'get': { asyncResult: true }, + 'save': { asyncResult: true }, + 'update': { asyncResult: true }, +}); + +export default RulesActions; \ No newline at end of file diff --git a/src/web/RulesComponent.jsx b/src/web/RulesComponent.jsx new file mode 100644 index 000000000000..f1cf08450046 --- /dev/null +++ b/src/web/RulesComponent.jsx @@ -0,0 +1,24 @@ +import React, {PropTypes} from 'react'; +import Reflux from 'reflux'; +import { Row, Col } from 'react-bootstrap'; + +import RulesActions from 'RulesActions'; + +const RulesComponent = React.createClass({ + propTypes: { + rules: PropTypes.array.isRequired, + }, + + render() { + return ( + + +

Rules

+

{this.props.rules.length}

+ + + ); + }, +}); + +export default RulesComponent; \ No newline at end of file diff --git a/src/web/RulesStore.js b/src/web/RulesStore.js new file mode 100644 index 000000000000..926eaf100214 --- /dev/null +++ b/src/web/RulesStore.js @@ -0,0 +1,44 @@ +import Reflux from 'reflux'; + +import RulesActions from 'RulesActions'; + +import UserNotification from 'util/UserNotification'; +import URLUtils from 'util/URLUtils'; +import fetch from 'logic/rest/FetchProvider'; + +const urlPrefix = '/plugins/org.graylog.plugins.pipelineprocessor'; + +const RulesStore = Reflux.createStore({ + listenables: [RulesActions], + rules: undefined, + + list() { + const failCallback = (error) => { + UserNotification.error('Fetching rules failed with status: ' + error.message, + 'Could not retrieve processing rules'); + }; + + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule'); + return fetch('GET', url).then((response) => { + this.rules = response; + this.trigger({rules: response}); + }, failCallback); + }, + + get(pipelineId) { + + }, + + save(pipelineSource) { + + }, + + update(pipelineId) { + + }, + delete(pipelineId) { + + }, +}); + +export default RulesStore; \ No newline at end of file From d7baecca5cf28600d10f30a52e0a535f8691c78b Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sat, 23 Jan 2016 13:54:19 +0100 Subject: [PATCH 035/521] rule edit/creation form - use minimal Ace Editor component for future support of highlighting/errors - rule is not saved to server yet --- package.json | 1 + src/web/Rule.jsx | 15 ++++ src/web/RuleForm.jsx | 140 +++++++++++++++++++++++++++++++++++++ src/web/RulesComponent.jsx | 37 +++++++++- 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 src/web/Rule.jsx create mode 100644 src/web/RuleForm.jsx diff --git a/package.json b/package.json index fbaa5dd239e1..306486fa3c82 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "license": "GPL-3.0", "dependencies": { "react": "0.14.x", + "react-ace": "^3.1.0", "react-bootstrap": "^0.28.1", "react-dom": "^0.14.5", "react-router": "~1.0.0", diff --git a/src/web/Rule.jsx b/src/web/Rule.jsx new file mode 100644 index 000000000000..0376128d40a4 --- /dev/null +++ b/src/web/Rule.jsx @@ -0,0 +1,15 @@ +import React, {PropTypes} from 'react'; + +const Rule = React.createClass({ + propTypes: { + rule: PropTypes.object.isRequired, + }, + + render() { + return
  • +

    {this.props.rule.name}

    +
    • ; + } +}); + +export default Rule; \ No newline at end of file diff --git a/src/web/RuleForm.jsx b/src/web/RuleForm.jsx new file mode 100644 index 000000000000..cc98ab5a1b4c --- /dev/null +++ b/src/web/RuleForm.jsx @@ -0,0 +1,140 @@ +import React, { PropTypes } from 'react'; + +import { Row, Col } from 'react-bootstrap'; +import { Input } from 'react-bootstrap'; + +import AceEditor from 'react-ace'; +import brace from 'brace'; + +import 'brace/mode/text'; +import 'brace/theme/chrome'; + +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; + +const RuleForm = React.createClass({ + propTypes: { + rule: PropTypes.object, + create: React.PropTypes.bool, + save: React.PropTypes.func.isRequired, + validateName: React.PropTypes.func.isRequired, + }, + + getDefaultProps() { + return { + rule: { + title: '', + description: '', + source: '', + } + } + }, + + getInitialState() { + return { + // when editing, take the rule that's been passed in + rule: this.props.rule, + error: false, + error_message: '', + } + }, + + openModal() { + this.refs.modal.open(); + }, + + _onSourceChange(value) { + const rule = this.state.rule; + rule.source = value; + this.setState({rule: rule}); + }, + + _onDescriptionChange(event) { + const rule = this.state.rule; + rule.description = event.target.value; + this.setState({rule: rule}); + }, + + _onTitleChange(event) { + const title = event.target.value; + + const rule = this.state.rule; + rule.title = title; + + if (!this.props.validateName(title)) { + this.setState({rule: rule, error: true, error_message: 'A rule with that title already exists!'}); + } else { + this.setState({rule: rule, error: false, error_message: ''}); + } + }, + + _getId(prefixIdName) { + return this.state.name !== undefined ? prefixIdName + this.state.name : prefixIdName; + }, + _closeModal() { + this.refs.modal.close(); + }, + _saved() { + this._closeModal(); + if (this.props.create) { + this.setState({rule: {}}); + } + }, + _save() { + if (!this.state.error) { + this.props.save(this.state.rule, this._saved); + } + }, + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Create rule'; + } else { + triggerButtonContent = Edit; + } + return ( + + + +
    + + + + +
    + +
    +
    +     +     + ); + }, +}); + +export default RuleForm; \ No newline at end of file diff --git a/src/web/RulesComponent.jsx b/src/web/RulesComponent.jsx index f1cf08450046..d050155d5858 100644 --- a/src/web/RulesComponent.jsx +++ b/src/web/RulesComponent.jsx @@ -1,20 +1,55 @@ import React, {PropTypes} from 'react'; import Reflux from 'reflux'; import { Row, Col } from 'react-bootstrap'; +import { Input, Alert } from 'react-bootstrap'; import RulesActions from 'RulesActions'; +import RuleForm from 'RuleForm'; const RulesComponent = React.createClass({ propTypes: { rules: PropTypes.array.isRequired, }, + _formatRule(rule) { + return ; + }, + + _sortByTitle(rule1, rule2) { + return rule1.title.localeCompare(rule2.title); + }, + + _save(rule, callback) { + console.log(rule); + callback(); + }, + + // TODO this should really validate the rule (as in parsing it server side) + _validateName(name) { + return true; + }, + render() { + let rules; + if (this.props.rules.length == 0) { + rules = + +   No rules configured. + + } else { + rules = this.props.rules.sort(this._sortByTitle).map(this._formatRule); + } + return (

    Rules

    -

    {this.props.rules.length}

    +
      + {rules} +
    +     ); From ae873132bfa96309c78adc5140df8979447d421d Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 25 Jan 2016 09:56:08 +0100 Subject: [PATCH 036/521] complete rule editing/deletion - creating rules is now done via json as well - properly returns id instead of _id - changes go into effect immediately, no simulate mode yet - load rule before deletion, to simplify 404 return value --- .../PipelineProcessorModule.java | 4 +- .../db/RuleSourceService.java | 5 +- .../pipelineprocessor/rest/RuleResource.java | 26 ++++------ .../pipelineprocessor/rest/RuleSource.java | 27 ++++++++-- src/web/Rule.jsx | 39 ++++++++++++++- src/web/RuleForm.jsx | 17 +++++-- src/web/RulesComponent.jsx | 17 ++++++- src/web/RulesStore.js | 49 ++++++++++++++++--- 8 files changed, 148 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 3c89cadec75e..bff5d5b4d456 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -26,6 +26,7 @@ import org.graylog.plugins.pipelineprocessor.functions.HasField; import org.graylog.plugins.pipelineprocessor.functions.InputFunction; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; +import org.graylog.plugins.pipelineprocessor.functions.SetField; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.processors.NaiveRuleProcessor; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; @@ -56,8 +57,9 @@ protected void configure() { addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); addMessageProcessorFunction(HasField.NAME, HasField.class); - addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); + addMessageProcessorFunction(SetField.NAME, SetField.class); addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); + addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java index 9165f2fc9eff..26f7d5033577 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java @@ -72,6 +72,9 @@ public Collection loadAll() { } public void delete(String id) { - dbCollection.removeById(id); + final WriteResult result = dbCollection.removeById(id); + if (result.getN() != 1) { + log.error("Unable to delete rule {}", id); + } } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 0e44bbdbe5d5..f8b0df5df386 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -49,6 +49,8 @@ @Api(value = "Pipeline Rules", description = "Rules for the pipeline message processor") @Path("/system/pipelines") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) public class RuleResource extends RestResource implements PluginRestResource { private static final Logger log = LoggerFactory.getLogger(RuleResource.class); @@ -68,18 +70,18 @@ public RuleResource(RuleSourceService ruleSourceService, @ApiOperation(value = "Create a processing rule from source", notes = "") - @Consumes(MediaType.TEXT_PLAIN) - @Produces(MediaType.APPLICATION_JSON) @POST @Path("/rule") - public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull String ruleSource) throws ParseException { + public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { try { - pipelineRuleParser.parseRule(ruleSource); + pipelineRuleParser.parseRule(ruleSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final RuleSource newRuleSource = RuleSource.builder() - .source(ruleSource) + .title(ruleSource.title()) + .description(ruleSource.description()) + .source(ruleSource.source()) .createdAt(DateTime.now()) .modifiedAt(DateTime.now()) .build(); @@ -90,17 +92,13 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No } @ApiOperation(value = "Get all processing rules") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) @GET @Path("/rule") public Collection getAll() { - return ruleSourceService.loadAll(); + return ruleSourceService.loadAll(); } @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) @Path("/rule/{id}") @GET public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { @@ -108,8 +106,6 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws } @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) @Path("/rule/{id}") @PUT public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @@ -131,14 +127,12 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, } @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) @Path("/rule/{id}") @DELETE - public void delete(@ApiParam(name = "id") @PathParam("id") String id) { + public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + ruleSourceService.load(id); ruleSourceService.delete(id); clusterBus.post(RulesChangedEvent.deletedRuleId(id)); } - } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index e1b0ae243787..2b32a833ad7d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import org.joda.time.DateTime; +import org.mongojack.Id; import org.mongojack.ObjectId; import javax.annotation.Nullable; @@ -29,18 +30,28 @@ @JsonAutoDetect public abstract class RuleSource { - @JsonProperty("_id") + @JsonProperty("id") @Nullable + @Id @ObjectId public abstract String id(); + @JsonProperty + public abstract String title(); + + @JsonProperty + @Nullable + public abstract String description(); + @JsonProperty public abstract String source(); @JsonProperty + @Nullable public abstract DateTime createdAt(); @JsonProperty + @Nullable public abstract DateTime modifiedAt(); public static Builder builder() { @@ -50,13 +61,17 @@ public static Builder builder() { public abstract Builder toBuilder(); @JsonCreator - public static RuleSource create(@JsonProperty("_id") @ObjectId @Nullable String id, + public static RuleSource create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, @JsonProperty("source") String source, - @JsonProperty("created_at") DateTime createdAt, - @JsonProperty("modified_at") DateTime modifiedAt) { + @JsonProperty("created_at") @Nullable DateTime createdAt, + @JsonProperty("modified_at") @Nullable DateTime modifiedAt) { return builder() .id(id) .source(source) + .title(title) + .description(description) .createdAt(createdAt) .modifiedAt(modifiedAt) .build(); @@ -68,6 +83,10 @@ public abstract static class Builder { public abstract Builder id(String id); + public abstract Builder title(String title); + + public abstract Builder description(String description); + public abstract Builder source(String source); public abstract Builder createdAt(DateTime createdAt); diff --git a/src/web/Rule.jsx b/src/web/Rule.jsx index 0376128d40a4..86f1cc9431b1 100644 --- a/src/web/Rule.jsx +++ b/src/web/Rule.jsx @@ -1,13 +1,50 @@ import React, {PropTypes} from 'react'; +import {Row, Col, Button} from 'react-bootstrap'; + +import AceEditor from 'react-ace'; +import brace from 'brace'; + +import 'brace/mode/text'; +import 'brace/theme/chrome'; + +import RuleForm from 'RuleForm'; const Rule = React.createClass({ propTypes: { rule: PropTypes.object.isRequired, }, + _delete() { + this.props.delete(this.props.rule); + }, + render() { return
  • -

    {this.props.rule.name}

    +

    {this.props.rule.title}

    + + {this.props.rule.description} + +
    + +
    + + + + + +
    • ; } }); diff --git a/src/web/RuleForm.jsx b/src/web/RuleForm.jsx index cc98ab5a1b4c..3f06a8a6603a 100644 --- a/src/web/RuleForm.jsx +++ b/src/web/RuleForm.jsx @@ -22,6 +22,7 @@ const RuleForm = React.createClass({ getDefaultProps() { return { rule: { + id: '', title: '', description: '', source: '', @@ -30,9 +31,15 @@ const RuleForm = React.createClass({ }, getInitialState() { + const rule = this.props.rule; return { // when editing, take the rule that's been passed in - rule: this.props.rule, + rule: { + id: rule.id, + title: rule.title, + description: rule.description, + source: rule.source + }, error: false, error_message: '', } @@ -103,7 +110,7 @@ const RuleForm = React.createClass({ submitButtonText="Save">
    Rule source
    ; + return ; }, _sortByTitle(rule1, rule2) { @@ -21,9 +27,18 @@ const RulesComponent = React.createClass({ _save(rule, callback) { console.log(rule); + if (rule.id) { + RulesActions.update(rule); + } else { + RulesActions.save(rule); + } callback(); }, + _delete(rule) { + RulesActions.delete(rule.id); + }, + // TODO this should really validate the rule (as in parsing it server side) _validateName(name) { return true; diff --git a/src/web/RulesStore.js b/src/web/RulesStore.js index 926eaf100214..29510aa66b40 100644 --- a/src/web/RulesStore.js +++ b/src/web/RulesStore.js @@ -25,19 +25,54 @@ const RulesStore = Reflux.createStore({ }, failCallback); }, - get(pipelineId) { + get(ruleId) { }, - save(pipelineSource) { - + save(ruleSource) { + const failCallback = (error) => { + UserNotification.error('Saving rule failed with status: ' + error.message, + 'Could not save processing rule'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule'); + const rule = { + title: ruleSource.title, + description: ruleSource.description, + source: ruleSource.source + }; + return fetch('POST', url, rule).then((response) => { + this.rules = response; + this.trigger({rules: response}); + }, failCallback); }, - update(pipelineId) { - + update(ruleSource) { + const failCallback = (error) => { + UserNotification.error('Updating rule failed with status: ' + error.message, + 'Could not update processing rule'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/' + ruleSource.id); + const rule = { + id: ruleSource.id, + title: ruleSource.title, + description: ruleSource.description, + source: ruleSource.source + }; + return fetch('PUT', url, rule).then((response) => { + this.rules = this.rules.map((e) => e.id === response.id ? response : e); + this.trigger({rules: this.rules}); + }, failCallback); }, - delete(pipelineId) { - + delete(ruleId) { + const failCallback = (error) => { + UserNotification.error('Updating rule failed with status: ' + error.message, + 'Could not update processing rule'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/' + ruleId); + return fetch('DELETE', url).then(() => { + this.rules = this.rules.filter((el) => el.id !== ruleId); + this.trigger({rules: this.rules}); + }, failCallback); }, }); From 24acc15256d3b5bf94c9b4918a7234152844c171 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 25 Jan 2016 10:49:27 +0100 Subject: [PATCH 037/521] fix warnings about deprecated Ace options --- src/web/Rule.jsx | 2 ++ src/web/RuleForm.jsx | 1 + 2 files changed, 3 insertions(+) diff --git a/src/web/Rule.jsx b/src/web/Rule.jsx index 86f1cc9431b1..fa2004b15ce0 100644 --- a/src/web/Rule.jsx +++ b/src/web/Rule.jsx @@ -24,6 +24,7 @@ const Rule = React.createClass({ {this.props.rule.description} +
    diff --git a/src/web/RuleForm.jsx b/src/web/RuleForm.jsx index 3f06a8a6603a..0301be5b1527 100644 --- a/src/web/RuleForm.jsx +++ b/src/web/RuleForm.jsx @@ -133,6 +133,7 @@ const RuleForm = React.createClass({ fontSize={11} height="14em" width="100%" + editorProps={{ $blockScrolling: "Infinity" }} value={this.state.rule.source} onChange={this._onSourceChange} /> From 672f4d73763e383794b86d02ee0296a36ff4621c Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 25 Jan 2016 13:04:21 +0100 Subject: [PATCH 038/521] fix initial loading of all rules during startup - unparseable rules will be replaced with a always-false dummy rule - rename drop message and input functions - input -> from_input now takes either id or name parameters and returns a boolean --- .../PipelineProcessorModule.java | 8 +- .../plugins/pipelineprocessor/ast/Rule.java | 5 ++ ...pMessageFunction.java => DropMessage.java} | 2 +- .../functions/FromInput.java | 81 +++++++++++++++++++ .../functions/InputFunction.java | 58 ------------- .../processors/NaiveRuleProcessor.java | 11 ++- 6 files changed, 99 insertions(+), 66 deletions(-) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{DropMessageFunction.java => DropMessage.java} (97%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index bff5d5b4d456..9a59959ee33a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -22,9 +22,9 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; -import org.graylog.plugins.pipelineprocessor.functions.DropMessageFunction; +import org.graylog.plugins.pipelineprocessor.functions.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.HasField; -import org.graylog.plugins.pipelineprocessor.functions.InputFunction; +import org.graylog.plugins.pipelineprocessor.functions.FromInput; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; import org.graylog.plugins.pipelineprocessor.functions.SetField; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; @@ -58,8 +58,8 @@ protected void configure() { addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(SetField.NAME, SetField.class); - addMessageProcessorFunction(DropMessageFunction.NAME, DropMessageFunction.class); - addMessageProcessorFunction(InputFunction.NAME, InputFunction.class); + addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); + addMessageProcessorFunction(FromInput.NAME, FromInput.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java index 53b9117466e0..1cef97208534 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java @@ -17,10 +17,12 @@ package org.graylog.plugins.pipelineprocessor.ast; import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import java.util.Collection; +import java.util.Collections; @AutoValue public abstract class Rule { @@ -35,6 +37,9 @@ public static Builder builder() { return new AutoValue_Rule.Builder(); } + public static Rule alwaysFalse(String name) { + return builder().name(name).when(new BooleanExpression(false)).then(Collections.emptyList()).build(); + } @AutoValue.Builder public abstract static class Builder { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessage.java similarity index 97% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessage.java index 68a2bc4da905..8b42570027f4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessageFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessage.java @@ -27,7 +27,7 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; -public class DropMessageFunction implements Function { +public class DropMessage implements Function { public static final String NAME = "drop_message"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java new file mode 100644 index 000000000000..bc53e86943a9 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -0,0 +1,81 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.IOState; +import org.graylog2.plugin.inputs.MessageInput; +import org.graylog2.shared.inputs.InputRegistry; + +import javax.inject.Inject; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class FromInput implements Function { + + public static final String NAME = "from_input"; + + private final InputRegistry inputRegistry; + + @Inject + public FromInput(InputRegistry inputRegistry) { + this.inputRegistry = inputRegistry; + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + String id = args.evaluated("id", context, String.class).orElse(""); + + MessageInput input = null; + if ("".equals(id)) { + final String name = args.evaluated("name", context, String.class).orElse(""); + for (IOState messageInputIOState : inputRegistry.getInputStates()) { + final MessageInput messageInput = messageInputIOState.getStoppable(); + if (messageInput.getTitle().equalsIgnoreCase(name)) { + input = messageInput; + break; + } + } + if ("".equals(name)) { + return null; + } + } else { + final IOState inputState = inputRegistry.getInputState(id); + if (inputState != null) { + input = inputState.getStoppable(); + } + + } + return input != null + && context.currentMessage().getSourceInputId().equals(input.getId()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + param().optional().string("id").build(), + param().optional().string("name").build())) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java deleted file mode 100644 index 77002649303a..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/InputFunction.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor.functions; - -import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; -import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; -import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; -import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; -import org.graylog2.plugin.IOState; -import org.graylog2.plugin.inputs.MessageInput; -import org.graylog2.shared.inputs.InputRegistry; - -import javax.inject.Inject; - -import static com.google.common.collect.ImmutableList.of; - -public class InputFunction implements Function { - - public static final String NAME = "input"; - - private final InputRegistry inputRegistry; - - @Inject - public InputFunction(InputRegistry inputRegistry) { - this.inputRegistry = inputRegistry; - } - - @Override - public MessageInput evaluate(FunctionArgs args, EvaluationContext context) { - final String id = args.evaluated("id", context, String.class).orElse(""); - final IOState inputState = inputRegistry.getInputState(id); - return inputState.getStoppable(); - } - - @Override - public FunctionDescriptor descriptor() { - return FunctionDescriptor.builder() - .name(NAME) - .returnType(MessageInput.class) - .params(of(ParameterDescriptor.string("id"))) - .build(); - } -} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java index f56ffb42d23b..26805f4145d2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java @@ -45,11 +45,12 @@ import javax.annotation.Nullable; import javax.inject.Inject; import javax.inject.Named; -import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; import static com.codahale.metrics.MetricRegistry.name; import static com.google.common.cache.CacheLoader.asyncReloading; @@ -75,7 +76,9 @@ public NaiveRuleProcessor(RuleSourceService ruleSourceService, .build(asyncReloading(new RuleLoader(ruleSourceService, pipelineRuleParser), scheduledExecutorService)); // prime the cache with all presently stored rules try { - ruleCache.getAll(Collections.emptyList()); + final List ruleIds = ruleSourceService.loadAll().stream().map(RuleSource::id).collect(Collectors.toList()); + log.info("Compiling {} processing rules", ruleIds.size()); + ruleCache.getAll(ruleIds); } catch (ExecutionException ignored) {} } @@ -147,6 +150,7 @@ public Map loadAll(Iterable keys) throws Excepti all.put(ruleSource.id(), pipelineRuleParser.parseRule(ruleSource.source())); } catch (ParseException e) { log.error("Unable to parse rule: " + e.getMessage()); + all.put(ruleSource.id(), Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id())); } } return all; @@ -159,7 +163,8 @@ public Rule load(@Nullable String ruleId) throws Exception { return pipelineRuleParser.parseRule(ruleSource.source()); } catch (ParseException e) { log.error("Unable to parse rule: " + e.getMessage()); - throw e; + // return dummy rule + return Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id()); } } } From 42511f602bd964c351a4976ff6a0938913405940 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Mon, 25 Jan 2016 14:47:03 +0100 Subject: [PATCH 039/521] Remove obsolete graylog2-shared dependency --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 18a6ffeace62..e4ab7642f9ed 100644 --- a/pom.xml +++ b/pom.xml @@ -105,12 +105,6 @@ - - org.graylog2 - graylog2-shared - ${graylog2.version} - provided - org.graylog2 graylog2-server From e5c0e055d14005a7fd0d57dfc4c3cc78a43bb85e Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Mon, 25 Jan 2016 14:56:42 +0100 Subject: [PATCH 040/521] Add missing brace dependency --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 306486fa3c82..e22ef6248356 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "author": "Kay Roepke ", "license": "GPL-3.0", "dependencies": { + "brace": "^0.7.0", "react": "0.14.x", "react-ace": "^3.1.0", "react-bootstrap": "^0.28.1", From 311fd476d8a37df2ef88aab8348e517ea53c7a78 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 25 Jan 2016 18:49:17 +0100 Subject: [PATCH 041/521] make FunctionArgs more robust against null/missing values - set_field now ignores empty or absent field names - improve error logging --- .../pipelineprocessor/ast/functions/FunctionArgs.java | 2 +- .../plugins/pipelineprocessor/functions/SetField.java | 5 +++-- .../pipelineprocessor/processors/NaiveRuleProcessor.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index 1836414ea66e..784adc443c8f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -48,7 +48,7 @@ public Optional evaluated(String name, EvaluationContext context, Class field = args.evaluated(FIELD, context, Object.class); final Optional value = args.evaluated(VALUE, context, Object.class); - context.currentMessage().addField(field.get().toString(), value.get()); - + if (!field.isPresent() && !field.get().toString().isEmpty()) { + context.currentMessage().addField(field.get().toString(), value.get()); + } return null; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java index 26805f4145d2..11928ba0193e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java @@ -102,7 +102,7 @@ public Messages process(Messages messages) { log.info("[✕] Message {} does not match condition", message.getId()); } } catch (Exception e) { - log.error("Unable to process message", e); + log.error("Unable to process message " + message.getId() + ", running rule " + rule.name(), e); } if (message.getFilterOut()) { log.info("[✝] Message {} was filtered out", message.getId()); From 09839388cd4994a198c3248aba062ff344908d90 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Tue, 26 Jan 2016 11:47:18 +0100 Subject: [PATCH 042/521] Move Google AutoValue dependency to provided scope --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index e4ab7642f9ed..46246be74266 100644 --- a/pom.xml +++ b/pom.xml @@ -114,6 +114,7 @@ com.google.auto.value auto-value + provided From 4727aef3aa1d5569f9cabd7c68d6609701582b7e Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 26 Jan 2016 12:00:03 +0100 Subject: [PATCH 043/521] rules do not have stages anymore --- .../org/graylog/plugins/pipelineprocessor/parser/basicRule.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt index 5d4362b2a9d3..f6019098a3fb 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt @@ -1,5 +1,4 @@ rule "something" -during stage 0 when double_valued_func() > 1.0d AND false == true then double_valued_func(); From 43bc971f1028403af9145139fd418f877e051861 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 26 Jan 2016 12:03:41 +0100 Subject: [PATCH 044/521] check for presence, not absence --- .../graylog/plugins/pipelineprocessor/functions/SetField.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java index d4deaab7857b..8a987ab72a19 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java @@ -38,7 +38,7 @@ public Void evaluate(FunctionArgs args, EvaluationContext context) { final Optional field = args.evaluated(FIELD, context, Object.class); final Optional value = args.evaluated(VALUE, context, Object.class); - if (!field.isPresent() && !field.get().toString().isEmpty()) { + if (field.isPresent() && !field.get().toString().isEmpty()) { context.currentMessage().addField(field.get().toString(), value.get()); } return null; From 6e1b1ef926c01332c801cdb12cb9d90f8b563e83 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 26 Jan 2016 12:11:37 +0100 Subject: [PATCH 045/521] my favorite build error: license headers --- .../pipelineprocessor/rest/PipelineResource.java | 16 ++++++++++++++++ .../pipelineprocessor/rest/PipelineSource.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index 66c5e7247479..81199e887792 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import com.google.common.eventbus.EventBus; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java index 80213d850ff6..eea79d10a513 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import com.fasterxml.jackson.annotation.JsonAutoDetect; From 86123572d8faef49a83ee663c1c323fb41bd4279 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Tue, 26 Jan 2016 12:16:24 +0100 Subject: [PATCH 046/521] Add "Pedantic" build profile with code quality checks --- config/checkstyle.xml | 293 ++++++++++++++++++++++++++++++++++++++++++ pom.xml | 133 ++++++++++++++++++- 2 files changed, 421 insertions(+), 5 deletions(-) create mode 100644 config/checkstyle.xml diff --git a/config/checkstyle.xml b/config/checkstyle.xml new file mode 100644 index 000000000000..ba67cd54a6e1 --- /dev/null +++ b/config/checkstyle.xml @@ -0,0 +1,293 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 46246be74266..35a9a2a4e00a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 3.0 + 3.1.0 org.graylog.plugins @@ -32,8 +32,13 @@ UTF-8 + UTF-8 1.8 1.8 + false + true + true + true 2.0.0-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 @@ -177,6 +182,16 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + + com.google.auto.value.processor.AutoValueProcessor + + + org.antlr antlr4-maven-plugin @@ -205,6 +220,26 @@ + + org.codehaus.mojo + findbugs-maven-plugin + 3.0.2 + + + org.apache.maven.plugins + maven-pmd-plugin + 3.5 + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.16 + + + org.codehaus.mojo + cobertura-maven-plugin + 2.7 + @@ -221,9 +256,10 @@ org.apache.maven.plugins maven-shade-plugin - 2.3 + 2.4.1 false + false @@ -254,7 +290,7 @@ jdeb org.vafer - 1.3 + 1.4 ${project.build.directory}/${project.artifactId}-${project.version}.deb @@ -272,11 +308,10 @@ - org.codehaus.mojo rpm-maven-plugin - 2.1.2 + 2.1.4 Application/Internet @@ -371,5 +406,93 @@ + + + edantic + + true + true + + + + + org.apache.maven.plugins + maven-compiler-plugin + + javac-with-errorprone + true + + + + org.codehaus.plexus + plexus-compiler-javac-errorprone + 2.5 + + + + com.google.errorprone + error_prone_core + 2.0.4 + + + + + org.codehaus.mojo + findbugs-maven-plugin + + Max + Default + true + false + + + + + check + + + + + + org.apache.maven.plugins + maven-pmd-plugin + + false + + + + + check + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + false + false + config/checkstyle.xml + + + + + check + + + + + + org.codehaus.mojo + cobertura-maven-plugin + + + + From 4a9cac4fe389502982a99e27a72854fafb2bb30c Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Tue, 26 Jan 2016 12:26:19 +0100 Subject: [PATCH 047/521] Remove Checkstyle from code analysis --- config/checkstyle.xml | 293 ------------------------------------------ pom.xml | 21 --- 2 files changed, 314 deletions(-) delete mode 100644 config/checkstyle.xml diff --git a/config/checkstyle.xml b/config/checkstyle.xml deleted file mode 100644 index ba67cd54a6e1..000000000000 --- a/config/checkstyle.xml +++ /dev/null @@ -1,293 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/pom.xml b/pom.xml index 35a9a2a4e00a..58f369ecbca6 100644 --- a/pom.xml +++ b/pom.xml @@ -230,11 +230,6 @@ maven-pmd-plugin 3.5 - - org.apache.maven.plugins - maven-checkstyle-plugin - 2.16 - org.codehaus.mojo cobertura-maven-plugin @@ -471,22 +466,6 @@ - - org.apache.maven.plugins - maven-checkstyle-plugin - - false - false - config/checkstyle.xml - - - - - check - - - - org.codehaus.mojo cobertura-maven-plugin From 689402f4a82cd42d928f9bcbb81309342f8d0047 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 27 Jan 2016 11:49:14 +0100 Subject: [PATCH 048/521] parse rule text before allowing to save it parsing requests start 500ms after typing stopped - use Ace's annotations to show parse errors - known issue: tooltip positioning is somewhat broken in Ace https://github.com/ajaxorg/ace/issues/2629 - not all parse errors are surfaced yet - formatting could be improved --- .../pipelineprocessor/rest/RuleResource.java | 18 ++++++ src/web/Rule.jsx | 3 +- src/web/RuleForm.jsx | 59 ++++++++++++++----- src/web/RulesActions.jsx | 1 + src/web/RulesComponent.jsx | 11 ++-- src/web/RulesStore.js | 23 ++++++++ 6 files changed, 95 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index f8b0df5df386..b95f4e2fc5a6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -91,6 +91,24 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No return save; } + @ApiOperation(value = "Parse a processing rule without saving it", notes = "") + @POST + @Path("/rule/parse") + public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { + try { + pipelineRuleParser.parseRule(ruleSource.source()); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + return RuleSource.builder() + .title(ruleSource.title()) + .description(ruleSource.description()) + .source(ruleSource.source()) + .createdAt(DateTime.now()) + .modifiedAt(DateTime.now()) + .build(); + } + @ApiOperation(value = "Get all processing rules") @GET @Path("/rule") diff --git a/src/web/Rule.jsx b/src/web/Rule.jsx index fa2004b15ce0..5d632dcc28a4 100644 --- a/src/web/Rule.jsx +++ b/src/web/Rule.jsx @@ -40,7 +40,8 @@ const Rule = React.createClass({ - + + + + ; + } +}); + +export default Pipeline; \ No newline at end of file diff --git a/src/web/PipelineForm.jsx b/src/web/PipelineForm.jsx new file mode 100644 index 000000000000..6ff910ab906d --- /dev/null +++ b/src/web/PipelineForm.jsx @@ -0,0 +1,179 @@ +import React, { PropTypes } from 'react'; + +import { Row, Col } from 'react-bootstrap'; +import { Input } from 'react-bootstrap'; + +import AceEditor from 'react-ace'; +import brace from 'brace'; + +import 'brace/mode/text'; +import 'brace/theme/chrome'; + +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; + +const PipelineForm = React.createClass({ + parseTimer: undefined, + propTypes: { + pipeline: PropTypes.object, + create: React.PropTypes.bool, + save: React.PropTypes.func.isRequired, + validatePipeline: React.PropTypes.func.isRequired, + }, + + getDefaultProps() { + return { + pipeline: { + id: '', + title: '', + description: '', + source: '', + }, + } + }, + + getInitialState() { + const pipeline = this.props.pipeline; + return { + // when editing, take the pipeline that's been passed in + pipeline: { + id: pipeline.id, + title: pipeline.title, + description: pipeline.description, + source: pipeline.source + }, + editor: undefined, + parseErrors: [], + } + }, + + componentWillUnmount() { + if (this.parseTimer !== undefined) { + clearTimeout(this.parseTimer); + this.parseTimer = undefined; + } + }, + + openModal() { + this.refs.modal.open(); + }, + + _updateEditor() { + const session = this.state.editor.session; + const annotations = this.state.parseErrors.map(e => { + return {row: e.line - 1, column: e.position_in_line - 1, text: e.reason, type: "error"} + }); + session.setAnnotations(annotations); + }, + + _setParseErrors(errors) { + this.setState({parseErrors: errors}, this._updateEditor); + }, + + _onSourceChange(value) { + // don't try to parse the previous value, gets reset below + if (this.parseTimer !== undefined) { + clearTimeout(this.parseTimer); + } + const pipeline = this.state.pipeline; + pipeline.source = value; + this.setState({pipeline: pipeline}); + + if (this.props.validatePipeline) { + // have the caller validate the pipeline after typing stopped for a while. usually this will mean send to server to parse + this.parseTimer = setTimeout(() => this.props.validatePipeline(pipeline, this._setParseErrors), 500); + } + }, + + _onDescriptionChange(event) { + const pipeline = this.state.pipeline; + pipeline.description = event.target.value; + this.setState({pipeline: pipeline}); + }, + + _onTitleChange(event) { + const pipeline = this.state.pipeline; + pipeline.title = event.target.value; + this.setState({pipeline: pipeline}); + }, + + _onLoad(editor) { + this.setState({editor: editor}); + }, + + _getId(prefixIdName) { + return this.state.name !== undefined ? prefixIdName + this.state.name : prefixIdName; + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _saved() { + this._closeModal(); + if (this.props.create) { + this.setState({pipeline: {}}); + } + }, + + _save() { + if (this.state.parseErrors.length === 0) { + this.props.save(this.state.pipeline, this._saved); + } + }, + + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Create pipeline'; + } else { + triggerButtonContent = Edit; + } + return ( + + + +
    + + + + +
    + +
    +
    +     +     + ); + }, +}); + +export default PipelineForm; \ No newline at end of file diff --git a/src/web/PipelinesActions.jsx b/src/web/PipelinesActions.jsx index 9d0dab1f121e..433843e23fd5 100644 --- a/src/web/PipelinesActions.jsx +++ b/src/web/PipelinesActions.jsx @@ -1,11 +1,12 @@ import Reflux from 'reflux'; -const PipelinesActions = Reflux.createActions({ +const RulesActions = Reflux.createActions({ 'delete': { asyncResult: true }, 'list': { asyncResult: true }, 'get': { asyncResult: true }, 'save': { asyncResult: true }, 'update': { asyncResult: true }, + 'parse' : { asyncResult: true }, }); -export default PipelinesActions; \ No newline at end of file +export default RulesActions; \ No newline at end of file diff --git a/src/web/PipelinesComponent.jsx b/src/web/PipelinesComponent.jsx index a1e8e9071fd1..91b7a0ebcd7c 100644 --- a/src/web/PipelinesComponent.jsx +++ b/src/web/PipelinesComponent.jsx @@ -1,20 +1,71 @@ import React, {PropTypes} from 'react'; import Reflux from 'reflux'; import { Row, Col } from 'react-bootstrap'; +import { Input, Alert } from 'react-bootstrap'; import PipelinesActions from 'PipelinesActions'; +import PipelineForm from 'PipelineForm'; +import Pipeline from 'Pipeline'; const PipelinesComponent = React.createClass({ propTypes: { pipelines: PropTypes.array.isRequired, }, + _formatPipeline(pipeline) { + return ; + }, + + _sortByTitle(pipeline1, pipeline2) { + return pipeline1.title.localeCompare(pipeline2.title); + }, + + _save(pipeline, callback) { + console.log(pipeline); + if (pipeline.id) { + PipelinesActions.update(pipeline); + } else { + PipelinesActions.save(pipeline); + } + callback(); + }, + + _delete(pipeline) { + PipelinesActions.delete(pipeline.id); + }, + + _validatePipeline(pipeline, setErrorsCb) { + PipelinesActions.parse(pipeline, setErrorsCb); + }, + render() { + let pipelines; + if (this.props.pipelines.length == 0) { + pipelines = + +   No pipelines configured. + + } else { + pipelines = this.props.pipelines.sort(this._sortByTitle).map(this._formatPipeline); + } + return (

    Pipelines

    -

    {this.props.pipelines.length}

    +
      + {pipelines} +
    +     ); diff --git a/src/web/PipelinesStore.js b/src/web/PipelinesStore.js index eba3666b6848..0b04e47632e4 100644 --- a/src/web/PipelinesStore.js +++ b/src/web/PipelinesStore.js @@ -30,15 +30,71 @@ const PipelinesStore = Reflux.createStore({ }, save(pipelineSource) { - + const failCallback = (error) => { + UserNotification.error('Saving pipeline failed with status: ' + error.message, + 'Could not save processing pipeline'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline'); + const pipeline = { + title: pipelineSource.title, + description: pipelineSource.description, + source: pipelineSource.source + }; + return fetch('POST', url, pipeline).then((response) => { + this.pipelines = response; + this.trigger({pipelines: response}); + }, failCallback); }, - update(pipelineId) { - + update(pipelineSource) { + const failCallback = (error) => { + UserNotification.error('Updating pipeline failed with status: ' + error.message, + 'Could not update processing pipeline'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline/' + pipelineSource.id); + const pipeline = { + id: pipelineSource.id, + title: pipelineSource.title, + description: pipelineSource.description, + source: pipelineSource.source + }; + return fetch('PUT', url, pipeline).then((response) => { + this.pipelines = this.pipelines.map((e) => e.id === response.id ? response : e); + this.trigger({pipelines: this.pipelines}); + }, failCallback); }, delete(pipelineId) { - + const failCallback = (error) => { + UserNotification.error('Updating pipeline failed with status: ' + error.message, + 'Could not update processing pipeline'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline/' + pipelineId); + return fetch('DELETE', url).then(() => { + this.pipelines = this.pipelines.filter((el) => el.id !== pipelineId); + this.trigger({pipelines: this.pipelines}); + }, failCallback); }, + parse(pipelineSource, callback) { + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline/parse'); + const pipeline = { + title: pipelineSource.title, + description: pipelineSource.description, + source: pipelineSource.source + }; + return fetch('POST', url, pipeline).then( + (response) => { + // call to clear the errors, the parsing was successful + callback([]); + }, + (error) => { + // a Bad Request indicates a parse error, set all the returned errors in the editor + const response = error.additional.res; + if (response.status === 400) { + callback(response.body); + } + } + ); + } }); export default PipelinesStore; \ No newline at end of file From 3c667cd04ee93709a0568099195b28cf5f678f5c Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Tue, 2 Feb 2016 16:28:20 +0100 Subject: [PATCH 053/521] Update to graylog-web-plugin 0.0.18 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e22ef6248356..61451694cf30 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "babel-core": "^5.8.25", "babel-loader": "^5.3.2", "graylog-web-manifests": "2.0.0-SNAPSHOT-1", - "graylog-web-plugin": "~0.0.17", + "graylog-web-plugin": "~0.0.18", "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", "ts-loader": "^0.8.0", From cc64c94d047f776e77fc91774886856adae712f3 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Thu, 4 Feb 2016 17:06:37 +0100 Subject: [PATCH 054/521] Bump graylog-server dependency to 2.0.0-alpha.2-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 58f369ecbca6..bd164b3f9cc6 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-SNAPSHOT + 2.0.0-alpha.2-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From dd69b0de8b0f3184786d5d6df2d049e3ffcb9484 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sat, 30 Jan 2016 12:06:16 +0100 Subject: [PATCH 055/521] initial rough interpreter for rules and pipelines [wip] currently needs message-changes branch of graylog2-server --- .../PipelineProcessorModule.java | 7 +- .../pipelineprocessor/ast/Pipeline.java | 13 +- .../plugins/pipelineprocessor/ast/Stage.java | 18 +- .../parser/PipelineRuleParser.java | 5 +- .../parser/errors/SyntaxError.java | 16 + .../processors/PipelineInterpreter.java | 399 ++++++++++++++++++ .../processors/StageIterator.java | 63 +++ .../pipelineprocessor/rest/RuleResource.java | 5 + .../parser/PipelineRuleParserTest.java | 4 +- .../processors/StageIteratorTest.java | 164 +++++++ 10 files changed, 684 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java create mode 100644 src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 9a59959ee33a..c3d03a51a5b4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -23,12 +23,12 @@ import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; import org.graylog.plugins.pipelineprocessor.functions.DropMessage; -import org.graylog.plugins.pipelineprocessor.functions.HasField; import org.graylog.plugins.pipelineprocessor.functions.FromInput; +import org.graylog.plugins.pipelineprocessor.functions.HasField; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; import org.graylog.plugins.pipelineprocessor.functions.SetField; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; -import org.graylog.plugins.pipelineprocessor.processors.NaiveRuleProcessor; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; @@ -46,7 +46,8 @@ public Set getConfigBeans() { @Override protected void configure() { - addMessageProcessor(NaiveRuleProcessor.class); + //addMessageProcessor(NaiveRuleProcessor.class); + addMessageProcessor(PipelineInterpreter.class); addRestResource(RuleResource.class); addRestResource(PipelineResource.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java index 3205db6a6ed4..00b8d13a31d9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java @@ -17,25 +17,32 @@ package org.graylog.plugins.pipelineprocessor.ast; import com.google.auto.value.AutoValue; +import com.google.common.collect.Sets; -import java.util.List; +import java.util.SortedSet; @AutoValue public abstract class Pipeline { public abstract String name(); - public abstract List stages(); + public abstract SortedSet stages(); public static Builder builder() { return new AutoValue_Pipeline.Builder(); } + public static Pipeline empty(String name) { + return builder().name(name).stages(Sets.newTreeSet()).build(); + } + + public abstract Builder toBuilder(); + @AutoValue.Builder public abstract static class Builder { public abstract Pipeline build(); public abstract Builder name(String name); - public abstract Builder stages(List stages); + public abstract Builder stages(SortedSet stages); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java index dd92a5a3980a..b063b211713d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java @@ -21,16 +21,32 @@ import java.util.List; @AutoValue -public abstract class Stage { +public abstract class Stage implements Comparable { + private List rules; public abstract int stage(); public abstract boolean matchAll(); public abstract List ruleReferences(); + public List getRules() { + return rules; + } + + public void setRules(List rules) { + this.rules = rules; + } + public static Builder builder() { return new AutoValue_Stage.Builder(); } + public abstract Builder toBuilder(); + + @Override + public int compareTo(@SuppressWarnings("NullableProblems") Stage other) { + return Integer.compare(stage(), other.stage()); + } + @AutoValue.Builder public abstract static class Builder { public abstract Stage build(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 71a68251ffcf..50c67c58eac4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -76,10 +76,12 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import static java.util.stream.Collectors.toList; @@ -727,7 +729,8 @@ public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ct final Pipeline.Builder builder = Pipeline.builder(); builder.name(unquote(ctx.name.getText(), '"')); - List stages = Lists.newArrayList(); + SortedSet stages = Sets.newTreeSet(Comparator.comparingInt(Stage::stage)); + for (RuleLangParser.StageDeclarationContext stage : ctx.stageDeclaration()) { final Stage.Builder stageBuilder = Stage.builder(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java index 32d2333c5776..b74f865fa979 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java new file mode 100644 index 000000000000..5765c140fcc4 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -0,0 +1,399 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; +import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; +import org.graylog.plugins.pipelineprocessor.rest.RuleSource; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.MessageCollection; +import org.graylog2.plugin.Messages; +import org.graylog2.plugin.messageprocessors.MessageProcessor; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.shared.buffers.processors.ProcessBufferProcessor; +import org.graylog2.shared.journal.Journal; +import org.jooq.lambda.Seq; +import org.jooq.lambda.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.inject.Named; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static com.codahale.metrics.MetricRegistry.name; +import static com.google.common.cache.CacheLoader.asyncReloading; +import static org.jooq.lambda.tuple.Tuple.tuple; + +public class PipelineInterpreter implements MessageProcessor { + private static final Logger log = LoggerFactory.getLogger(PipelineInterpreter.class); + + private final PipelineSourceService pipelineSourceService; + private final PipelineRuleParser pipelineRuleParser; + private final Journal journal; + private final Meter filteredOutMessages; + private final LoadingCache ruleCache; + private final ListeningScheduledExecutorService scheduledExecutorService; + + private final AtomicReference> currentPipelines = new AtomicReference<>(); + private HashMultimap streamPipelineAssignments = HashMultimap.create(); + + @Inject + public PipelineInterpreter(RuleSourceService ruleSourceService, + PipelineSourceService pipelineSourceService, + PipelineRuleParser pipelineRuleParser, + Journal journal, + MetricRegistry metricRegistry, + @Named("daemonScheduler") ScheduledExecutorService scheduledExecutorService, + @ClusterEventBus EventBus clusterBus) { + this.pipelineSourceService = pipelineSourceService; + this.pipelineRuleParser = pipelineRuleParser; + this.journal = journal; + this.scheduledExecutorService = MoreExecutors.listeningDecorator(scheduledExecutorService); + this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); + clusterBus.register(this); + ruleCache = CacheBuilder.newBuilder() + .build(asyncReloading(new RuleLoader(ruleSourceService, pipelineRuleParser), scheduledExecutorService)); + + // prime the cache with all presently stored rules + try { + final List ruleIds = ruleSourceService.loadAll().stream().map(RuleSource::id).collect(Collectors.toList()); + log.info("Compiling {} processing rules", ruleIds.size()); + ruleCache.getAll(ruleIds); + } catch (ExecutionException ignored) { + } + } + + /** + * @param messages messages to process + * @return messages to pass on to the next stage + */ + @Override + public Messages process(Messages messages) { + // message id + stream id + final Set> processingBlacklist = Sets.newHashSet(); + + final List fullyProcessed = Lists.newArrayList(); + List toProcess = Lists.newArrayList(messages); + + while (!toProcess.isEmpty()) { + final MessageCollection currentSet = new MessageCollection(toProcess); + // we'll add them back below + toProcess.clear(); + + for (Message message : currentSet) { + final String msgId = message.getId(); + + // 1. for each message, determine which pipelines are supposed to be executed, based on their streams + // null is the default stream, the other streams are identified by their id + final Set pipelinesToRun; + + // this makes a copy of the list! + final Set initialStreamIds = Sets.newHashSet(message.getStreamIds()); + + if (initialStreamIds.isEmpty()) { + if (processingBlacklist.contains(tuple(msgId, (String) null))) { + // already processed default pipeline for this message + pipelinesToRun = Collections.emptySet(); + log.info("[{}] already processed default stream, skipping", msgId); + } else { + // get the default stream pipeline assignments for this message + pipelinesToRun = streamPipelineAssignments.get(null); + log.info("[{}] running default stream pipelines: [{}]", + msgId, + pipelinesToRun.stream().map(Pipeline::name).toArray()); + } + } else { + pipelinesToRun = initialStreamIds.stream() + // 2. if a message-stream combination has already been processed (is in the set), skip that execution + .filter(streamId -> !processingBlacklist.contains(tuple(msgId, streamId))) + .flatMap(streamId -> streamPipelineAssignments.get(streamId).stream()) + .collect(Collectors.toSet()); + log.info("[{}] running pipelines {}", msgId, pipelinesToRun); + } + + final StageIterator stages = new StageIterator(pipelinesToRun); + final Set pipelinesToProceedWith = Sets.newHashSet(); + + // iterate through all stages for all matching pipelines, per "stage slice" instead of per pipeline. + // pipeline execution ordering is not guaranteed + while (stages.hasNext()) { + final Set> stageSet = stages.next(); + for (Tuple2 pair : stageSet) { + final Stage stage = pair.v1(); + final Pipeline pipeline = pair.v2(); + if (!pipelinesToProceedWith.isEmpty() && + !pipelinesToProceedWith.contains(pipeline)) { + log.info("[{}] previous stage result prevents further processing of pipeline `{}`", + msgId, + pipeline.name()); + continue; + } + log.info("[{}] evaluating rule conditions in stage {}: match {}", + msgId, + stage.stage(), + stage.matchAll() ? "all" : "either"); + + // TODO the message should be decorated to allow layering changes and isolate stages + final EvaluationContext context = new EvaluationContext(message); + + // 3. iterate over all the stages in these pipelines and execute them in order + final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); + for (Rule rule : stage.getRules()) { + if (rule.when().evaluateBool(context)) { + log.info("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); + rulesToRun.add(rule); + } else { + log.info("[{}] rule `{}` does not match", msgId, rule.name()); + } + } + for (Rule rule : rulesToRun) { + log.info("[{}] rule `{}` matched running actions", msgId, rule.name()); + for (Statement statement : rule.then()) { + statement.evaluate(context); + } + } + // stage needed to match all rule conditions to enable the next stage, + // record that it is ok to proceed with this pipeline + // OR + // any rule could match, but at least one had to, + // record that it is ok to proceed with the pipeline + if ((stage.matchAll() && (rulesToRun.size() == stage.getRules().size())) + || (rulesToRun.size() > 0)) { + log.info("[{}] stage for pipeline `{}` required match: {}, ok to proceed with next stage", + msgId, pipeline.name(), stage.matchAll() ? "all" : "either"); + pipelinesToProceedWith.add(pipeline); + } + + // 4. after each complete stage run, merge the processing changes, stages are isolated from each other + // TODO message changes become visible immediately for now + + } + + } + boolean addedStreams = false; + // 5. add each message-stream combination to the blacklist set + for (Stream stream : message.getStreams()) { + processingBlacklist.add(tuple(msgId, stream.getId())); + if (!initialStreamIds.remove(stream.getId())) { + addedStreams = true; + } + } + if (message.getFilterOut()) { + log.debug("[{}] marked message to be discarded. Dropping message.", + msgId); + filteredOutMessages.mark(); + journal.markJournalOffsetCommitted(message.getJournalOffset()); + } + // 6. go to 1 and iterate over all messages again until no more streams are being assigned + if (!addedStreams) { + log.info("[{}] no new streams matches, not running again", msgId); + fullyProcessed.add(message); + } else { + // process again, we've added a stream + log.info("[{}] new streams assigned, running again for those streams", msgId); + toProcess.add(message); + } + } + } + // 7. return the processed messages + return new MessageCollection(fullyProcessed); + } + + @Subscribe + public void handleRuleChanges(RulesChangedEvent event) { + event.deletedRuleIds().forEach(id -> { + ruleCache.invalidate(id); + log.info("Invalidated rule {}", id); + }); + event.updatedRuleIds().forEach(id -> { + ruleCache.refresh(id); + log.info("Refreshing rule {}", id); + }); + + triggerPipelineUpdate(); + + } + + @Subscribe + public void handlePipelineChanges(PipelinesChangedEvent event) { + event.deletedPipelineIds().forEach(id -> { + log.info("Invalidated pipeline {}", id); + }); + event.updatedPipelineIds().forEach(id -> { + log.info("Refreshing pipeline {}", id); + }); + + triggerPipelineUpdate(); + } + + private void triggerPipelineUpdate() { + Futures.addCallback( + scheduledExecutorService.schedule(new PipelineResolver(), 250, TimeUnit.MILLISECONDS), + new FutureCallback>() { + @Override + public void onSuccess(@Nullable ImmutableSet result) { + // TODO how do we deal with concurrent updates? canceling earlier attempts? + currentPipelines.set(result); + if (result != null) { + streamPipelineAssignments.putAll(null, result); + } + } + + @Override + public void onFailure(Throwable t) { + // do not touch the existing pipeline configuration + log.error("Unable to update pipeline processor", t); + } + }); + } + + private static class RuleLoader extends CacheLoader { + private final RuleSourceService ruleSourceService; + private final PipelineRuleParser pipelineRuleParser; + + public RuleLoader(RuleSourceService ruleSourceService, PipelineRuleParser pipelineRuleParser) { + this.ruleSourceService = ruleSourceService; + this.pipelineRuleParser = pipelineRuleParser; + } + + @Override + public Map loadAll(Iterable keys) throws Exception { + final Map all = Maps.newHashMap(); + final HashSet keysToLoad = Sets.newHashSet(keys); + for (RuleSource ruleSource : ruleSourceService.loadAll()) { + if (!keysToLoad.isEmpty()) { + if (!keysToLoad.contains(ruleSource.id())) { + continue; + } + } + try { + all.put(ruleSource.id(), pipelineRuleParser.parseRule(ruleSource.source())); + } catch (ParseException e) { + log.error("Unable to parse rule: " + e.getMessage()); + all.put(ruleSource.id(), Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id())); + } + } + return all; + } + + @Override + public Rule load(@Nullable String ruleId) throws Exception { + final RuleSource ruleSource = ruleSourceService.load(ruleId); + try { + return pipelineRuleParser.parseRule(ruleSource.source()); + } catch (ParseException e) { + log.error("Unable to parse rule: " + e.getMessage()); + // return dummy rule + return Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id()); + } + } + } + + private class PipelineResolver implements Callable> { + private final Logger log = LoggerFactory.getLogger(PipelineResolver.class); + + + @Override + public ImmutableSet call() throws Exception { + log.info("Updating pipeline processor after rule/pipeline update"); + + final Collection allPipelineSources = pipelineSourceService.loadAll(); + log.info("Found {} pipelines to resolve", allPipelineSources.size()); + + // compile all pipelines + Set pipelines = Sets.newHashSetWithExpectedSize(allPipelineSources.size()); + for (PipelineSource source : allPipelineSources) { + try { + final Pipeline pipeline = pipelineRuleParser.parsePipeline(source.source()); + pipelines.add(pipeline); + log.info("Parsed pipeline {} with {} stages", pipeline.name(), pipeline.stages().size()); + } catch (ParseException e) { + log.warn("Unable to compile pipeline {}: {}", source.title(), e); + } + } + + // change the rule cache to be able to quickly look up rules by name + final Map nameRuleMap = + Seq.toMap(Seq.seq(ruleCache.asMap()) + .map(entry -> tuple(entry.v2().name(), entry.v2()))); + + // resolve all rules + pipelines.stream() + .flatMap(pipeline -> { + log.info("Resolving pipeline {}", pipeline.name()); + return pipeline.stages().stream(); + }) + .forEach(stage -> { + final List resolvedRules = stage.ruleReferences().stream(). + map(ref -> { + Rule rule = nameRuleMap.get(ref); + if (rule == null) { + rule = Rule.alwaysFalse("Unresolved rule " + ref); + } + log.info("Resolved rule `{}` to {}", ref, rule); + return rule; + }) + .collect(Collectors.toList()); + stage.setRules(resolvedRules); + }); + + return ImmutableSet.copyOf(pipelines); + } + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java new file mode 100644 index 000000000000..b8e2fdd5ab3b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java @@ -0,0 +1,63 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.PeekingIterator; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.jooq.lambda.tuple.Tuple; +import org.jooq.lambda.tuple.Tuple2; + +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.collect.Iterators.peekingIterator; + +public class StageIterator extends AbstractIterator>> { + + private final ImmutableList, Pipeline>> stageIterators; + + public StageIterator(Set pipelines) { + stageIterators = ImmutableList.copyOf(pipelines.stream() + .map(p -> Tuple.tuple(peekingIterator(p.stages().iterator()), p)) + .iterator()); + } + + @Override + protected Set> computeNext() { + final OptionalInt min = stageIterators.stream() + .filter(pair -> pair.v1().hasNext()) // only iterators that have remaining elements + .mapToInt(pair -> pair.v1().peek().stage()) // get the stage of each remaining element + .min(); // we want the minimum stage number of them all + + if (!min.isPresent()) { + return endOfData(); + + } + final int currStage = min.getAsInt(); + + return stageIterators.stream() + .filter(pair -> pair.v1().hasNext()) // only iterators that have remaining elements + .filter(pair -> pair.v1().peek().stage() == currStage) // only elements for the current stage + .map(pair -> Tuple.tuple(pair.v1().next(), pair.v2())) + .collect(Collectors.toSet()); + + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index b95f4e2fc5a6..d82f9eab1cce 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -86,6 +86,7 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No .modifiedAt(DateTime.now()) .build(); final RuleSource save = ruleSourceService.save(newRuleSource); + // TODO determine which pipelines could change because of this new rule (there could be pipelines referring to a previously unresolved rule) clusterBus.post(RulesChangedEvent.updatedRuleId(save.id())); log.info("Created new rule {}", save); return save; @@ -139,6 +140,8 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, .modifiedAt(DateTime.now()) .build(); final RuleSource savedRule = ruleSourceService.save(toSave); + + // TODO determine which pipelines could change because of this updated rule clusterBus.post(RulesChangedEvent.updatedRuleId(savedRule.id())); return savedRule; @@ -150,6 +153,8 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { ruleSourceService.load(id); ruleSourceService.delete(id); + + // TODO determine which pipelines could change because of this deleted rule, causing them to recompile clusterBus.post(RulesChangedEvent.deletedRuleId(id)); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index fb94acff8a53..689d9c0afb2f 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -433,8 +433,8 @@ public void pipelineDeclaration() throws Exception { final Pipeline pipeline = Iterables.getOnlyElement(pipelines); assertEquals("cisco", pipeline.name()); assertEquals(2, pipeline.stages().size()); - final Stage stage1 = pipeline.stages().get(0); - final Stage stage2 = pipeline.stages().get(1); + final Stage stage1 = pipeline.stages().first(); + final Stage stage2 = pipeline.stages().last(); assertEquals(true, stage1.matchAll()); assertEquals(1, stage1.stage()); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java new file mode 100644 index 000000000000..60796b9d50c5 --- /dev/null +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java @@ -0,0 +1,164 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.jooq.lambda.Seq; +import org.jooq.lambda.tuple.Tuple2; +import org.junit.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import static com.google.common.collect.ImmutableSortedSet.of; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class StageIteratorTest { + + @Test + public void singleEmptyPipeline() { + final ImmutableSet empty = ImmutableSet.of(Pipeline.empty("empty")); + final StageIterator iterator = new StageIterator(empty); + + assertFalse(iterator.hasNext()); + } + + @Test + public void singlePipelineNoStage() { + + final ImmutableSet input = + ImmutableSet.of(Pipeline.builder() + .name("hallo") + .stages(of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build())) + .build()); + final StageIterator iterator = new StageIterator(input); + assertTrue(iterator.hasNext()); + final Set> nextStages = iterator.next(); + assertEquals(1, nextStages.size()); + + final Tuple2 stage = Iterables.getOnlyElement(nextStages); + assertEquals(0, stage.v1.ruleReferences().size()); + } + + @Test + public void singlePipelineTwoStages() { + final ImmutableSet input = + ImmutableSet.of(Pipeline.builder() + .name("hallo") + .stages(of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(10) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build() + )).build()); + final StageIterator iterator = new StageIterator(input); + //noinspection unchecked + final Set>[] stages = Iterators.toArray(iterator, Set.class); + + assertEquals(2, stages.length); + assertEquals(1, stages[0].size()); + assertEquals("last set of stages are on stage 0", 0, Iterables.getOnlyElement(stages[0]).v1.stage()); + assertEquals(1, stages[1].size()); + assertEquals("last set of stages are on stage 1", 10, Iterables.getOnlyElement(stages[1]).v1.stage()); + } + + + @Test + public void multiplePipelines() { + final ImmutableSortedSet stages1 = + of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(10) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build() + ); + final ImmutableSortedSet stages2 = + of(Stage.builder() + .stage(-1) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(4) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(11) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build() + ); + final ImmutableSortedSet stages3 = + of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build()); + + final ImmutableSet input = + ImmutableSet.of(Pipeline.builder() + .name("p1") + .stages(stages1).build(), + Pipeline.builder() + .name("p2") + .stages(stages2).build() + ,Pipeline.builder() + .name("p3") + .stages(stages3).build() + ); + final StageIterator iterator = new StageIterator(input); + + final List>> stageSets = Lists.newArrayList(iterator); + + assertEquals("5 different stages to execute", 5, stageSets.size()); + + for (Set> stageSet : stageSets) { + assertEquals("Each stage set should only contain stages with the same number", + 1, + Seq.seq(stageSet).map(Tuple2::v1).groupBy(Stage::stage).keySet().size()); + } + assertArrayEquals("Stages must be sorted numerically", + new int[] {-1, 0, 4, 10, 11}, + stageSets.stream().flatMap(Collection::stream).map(Tuple2::v1).mapToInt(Stage::stage).distinct().toArray()); + } +} \ No newline at end of file From 0c4d47c12946c89a9bc01ca5794fe1a03ceb7cd8 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 3 Feb 2016 10:07:43 +0100 Subject: [PATCH 056/521] working pipeline intepreter The interpreter depends on a few data structures: - rules collection and fully loaded rule set in memory (cached + updated via event bus) - pipeline collection and loaded set in memory (cached + updated via event bus) - pipeline -> stream assignment (cached + updated via event bus) Pipelines are not being run until they are attached to a stream. It uses a "stream id" of 'default' to identify incoming messages which do not have a stream assigned yet. The pipeline updater resolves all functions, rules and stages inside pipelines, this is done by name and not internal ID. Execution is as follows: For each incoming message the interpreter determines which pipelines can possibly run, based on the streams a message is assigned to. For each of the pipelines their stages will be executed, in ascending stage number order. Stages with the same order number are being run after each other. For each of the stages their rules' conditions are evaluated and gate which rules' actions are being run. The match clause determines whether the subsequent stages for that particular pipeline will be run. Each message is being run through a pipeline once per stream. After all pipelines have been run, the interpreter determines whether additional pipelines need to be run as well, based on the new stream assignments. The loop stops when no more new streams have been assigned to a message. Note: This implementation does not yet guarantee that stages are isolated from each other while they run. --- pom.xml | 12 +- .../pipelineprocessor/EvaluationContext.java | 17 +++ .../PipelineProcessorModule.java | 15 ++- .../pipelineprocessor/ast/Pipeline.java | 9 ++ .../db/PipelineStreamAssignmentService.java | 84 +++++++++++++ .../functions/messages/CreateMessage.java | 70 +++++++++++ .../functions/{ => messages}/DropMessage.java | 5 +- .../functions/{ => messages}/HasField.java | 2 +- .../functions/messages/RouteToStream.java | 88 ++++++++++++++ .../functions/{ => messages}/SetField.java | 2 +- .../parser/PipelineRuleParser.java | 8 +- .../processors/PipelineInterpreter.java | 79 +++++++++---- .../rest/PipelineResource.java | 8 +- .../rest/PipelineStreamAssignment.java | 72 ++++++++++++ .../rest/PipelineStreamResource.java | 94 +++++++++++++++ .../pipelineprocessor/rest/RuleResource.java | 2 +- .../parser/PipelineRuleParserTest.java | 4 +- .../processors/PipelineInterpreterTest.java | 111 ++++++++++++++++++ 18 files changed, 642 insertions(+), 40 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{ => messages}/DropMessage.java (93%) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{ => messages}/HasField.java (96%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{ => messages}/SetField.java (97%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java create mode 100644 src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java diff --git a/pom.xml b/pom.xml index bd164b3f9cc6..c2a9fa8da2a0 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,12 @@ junit 4.12 + + org.mockito + mockito-core + 2.0.40-beta + test + org.jooq jool @@ -135,7 +141,11 @@ junit test - + + org.mockito + mockito-core + test + org.apache.logging.log4j log4j-api diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java index 9720d28210fa..65d12470a108 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java @@ -16,15 +16,20 @@ */ package org.graylog.plugins.pipelineprocessor; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.graylog2.plugin.Message; +import org.graylog2.plugin.MessageCollection; +import org.graylog2.plugin.Messages; +import java.util.List; import java.util.Map; public class EvaluationContext { private final Message message; private Map ruleVars; + private List createdMessages = Lists.newArrayList(); public EvaluationContext(Message message) { this.message = message; @@ -43,6 +48,18 @@ public TypedValue get(String identifier) { return ruleVars.get(identifier); } + public Messages createdMessages() { + return new MessageCollection(createdMessages); + } + + public void addCreatedMessage(Message newMessage) { + createdMessages.add(newMessage); + } + + public void clearCreatedMessages() { + createdMessages.clear(); + } + public class TypedValue { private final Class type; private final Object value; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index c3d03a51a5b4..b87a4940529b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -22,14 +22,17 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; -import org.graylog.plugins.pipelineprocessor.functions.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.FromInput; -import org.graylog.plugins.pipelineprocessor.functions.HasField; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.SetField; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamResource; import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; @@ -50,6 +53,7 @@ protected void configure() { addMessageProcessor(PipelineInterpreter.class); addRestResource(RuleResource.class); addRestResource(PipelineResource.class); + addRestResource(PipelineStreamResource.class); // built-in functions addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); @@ -57,9 +61,14 @@ protected void configure() { addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); + // message related functions addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(SetField.NAME, SetField.class); addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); + addMessageProcessorFunction(CreateMessage.NAME, CreateMessage.class); + addMessageProcessorFunction(RouteToStream.NAME, RouteToStream.class); + + // input related functions addMessageProcessorFunction(FromInput.NAME, FromInput.class); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java index 00b8d13a31d9..08c620d3ec5b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java @@ -19,11 +19,14 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.Sets; +import javax.annotation.Nullable; import java.util.SortedSet; @AutoValue public abstract class Pipeline { + @Nullable + public abstract String id(); public abstract String name(); public abstract SortedSet stages(); @@ -37,10 +40,16 @@ public static Pipeline empty(String name) { public abstract Builder toBuilder(); + public Pipeline withId(String id) { + return toBuilder().id(id).build(); + } + @AutoValue.Builder public abstract static class Builder { public abstract Pipeline build(); + public abstract Builder id(String id); + public abstract Builder name(String name); public abstract Builder stages(SortedSet stages); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java new file mode 100644 index 000000000000..9ec9325c9518 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java @@ -0,0 +1,84 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoException; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoConnection; +import org.graylog2.database.NotFoundException; +import org.mongojack.DBCursor; +import org.mongojack.DBQuery; +import org.mongojack.JacksonDBCollection; +import org.mongojack.WriteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collections; +import java.util.Set; + +public class PipelineStreamAssignmentService { + private static final Logger log = LoggerFactory.getLogger(PipelineStreamAssignmentService.class); + + public static final String COLLECTION = "pipeline_processor_pipelines_streams"; + + private final JacksonDBCollection dbCollection; + + @Inject + public PipelineStreamAssignmentService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + dbCollection = JacksonDBCollection.wrap( + mongoConnection.getDatabase().getCollection(COLLECTION), + PipelineStreamAssignment.class, + String.class, + mapper.get()); + dbCollection.createIndex(new BasicDBObject("stream_id", 1)); + } + + + public PipelineStreamAssignment save(PipelineStreamAssignment assignment) { + PipelineStreamAssignment existingAssignment = dbCollection.findOne(DBQuery.is("stream_id", assignment.streamId())); + if (existingAssignment == null) { + existingAssignment = PipelineStreamAssignment.create(null, assignment.streamId(), Collections.emptySet()); + } + + final PipelineStreamAssignment toSave = existingAssignment.toBuilder() + .pipelineIds(assignment.pipelineIds()).build(); + final WriteResult save = dbCollection.save(toSave); + return save.getSavedObject(); + } + + public PipelineStreamAssignment load(String streamId) throws NotFoundException { + final PipelineStreamAssignment oneById = dbCollection.findOne(DBQuery.is("stream_id", streamId)); + if (oneById == null) { + throw new NotFoundException("No pipeline assignments with for stream " + streamId); + } + return oneById; + } + + public Set loadAll() { + try { + final DBCursor assignments = dbCollection.find(); + return Sets.newHashSet(assignments.iterator()); + } catch (MongoException e) { + log.error("Unable to load pipelines", e); + return Collections.emptySet(); + } + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java new file mode 100644 index 000000000000..a70741daadbf --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java @@ -0,0 +1,70 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Tools; +import org.joda.time.DateTime; + +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class CreateMessage implements Function { + + public static final String NAME = "create_message"; + + private static final String MESSAGE_ARG = "message"; + private static final String SOURCE_ARG = "source"; + private static final String TIMESTAMP_ARG = "timestamp"; + + @Override + public Message evaluate(FunctionArgs args, EvaluationContext context) { + final Optional optMessage = args.evaluated(MESSAGE_ARG, context, String.class); + final String message = optMessage.isPresent() ? optMessage.get() : context.currentMessage().getMessage(); + + final Optional optSource = args.evaluated(SOURCE_ARG, context, String.class); + final String source = optSource.isPresent() ? optSource.get() : context.currentMessage().getSource(); + + final Optional optTimestamp = args.evaluated(TIMESTAMP_ARG, context, DateTime.class); + final DateTime timestamp = optTimestamp.isPresent() ? optTimestamp.get() : Tools.nowUTC(); + + final Message newMessage = new Message(message, source, timestamp); + + // register in context so the processor can inject it later on + context.addCreatedMessage(newMessage); + return newMessage; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Message.class) + .params(of( + param().string(MESSAGE_ARG).optional().build(), + param().string(SOURCE_ARG).optional().build(), + param().name(TIMESTAMP_ARG).type(DateTime.class).optional().build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java similarity index 93% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessage.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java index 8b42570027f4..28af07efa4a3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DropMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.messages; import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; @@ -30,6 +30,7 @@ public class DropMessage implements Function { public static final String NAME = "drop_message"; + public static final String MESSAGE_ARG = "message"; @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { @@ -50,7 +51,7 @@ public FunctionDescriptor descriptor() { .pure(true) .returnType(Void.class) .params(ImmutableList.of( - param().type(Message.class).optional().name("message").build() + param().type(Message.class).optional().name(MESSAGE_ARG).build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java similarity index 96% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java index 4d4acb0244e4..c6a3828cb810 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/HasField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.messages; import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java new file mode 100644 index 000000000000..640f165e108b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -0,0 +1,88 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.inject.Inject; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.database.NotFoundException; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.streams.StreamService; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; + +public class RouteToStream implements Function { + + public static final String NAME = "route_to_stream"; + private static final String ID_ARG = "id"; + private static final String NAME_ARG = "name"; + private final StreamService streamService; + + @Inject + public RouteToStream(StreamService streamService) { + this.streamService = streamService; + streamService.loadAllEnabled(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + String id = args.evaluated(ID_ARG, context, String.class).orElse(""); + + final Stream stream; + if ("".equals(id)) { + final String name = args.evaluated(NAME_ARG, context, String.class).orElse(""); + if ("".equals(name)) { + return null; + } + // TODO efficiency + final ImmutableMap stringStreamImmutableMap = Maps.uniqueIndex(streamService.loadAll(), + Stream::getTitle); + stream = stringStreamImmutableMap.get(name); + if (stream == null) { + // TODO signal error somehow + return null; + } + } else { + try { + stream = streamService.load(id); + } catch (NotFoundException e) { + return null; + } + } + // TODO needs message stack in context to pick message + if (!stream.isPaused()) { + context.currentMessage().addStream(stream); + } + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of( + param().optional().string(NAME_ARG).build(), + param().optional().string(ID_ARG).build())) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java similarity index 97% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index 8a987ab72a19..ff40e0e964e4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.messages; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 50c67c58eac4..5cf24552f554 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -72,6 +72,7 @@ import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; import org.graylog.plugins.pipelineprocessor.parser.errors.WrongNumberOfArgs; +import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,11 +153,11 @@ public List parsePipelines(String pipelines) throws ParseException { throw new ParseException(parseContext.getErrors()); } - public Pipeline parsePipeline(String pipeline) { + public Pipeline parsePipeline(PipelineSource pipelineSource) { final ParseContext parseContext = new ParseContext(); final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); - final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(pipeline)); + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(pipelineSource.source())); lexer.removeErrorListeners(); lexer.addErrorListener(errorListener); @@ -170,7 +171,8 @@ public Pipeline parsePipeline(String pipeline) { WALKER.walk(new PipelineAstBuilder(parseContext), pipelineContext); if (parseContext.getErrors().isEmpty()) { - return parseContext.pipelines.get(0); + final Pipeline pipeline = parseContext.pipelines.get(0); + return pipeline.withId(pipelineSource.id()); } throw new ParseException(parseContext.getErrors()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 5765c140fcc4..a64215f483f1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -22,7 +22,10 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -38,12 +41,14 @@ import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; @@ -63,7 +68,6 @@ import javax.inject.Named; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -89,12 +93,13 @@ public class PipelineInterpreter implements MessageProcessor { private final LoadingCache ruleCache; private final ListeningScheduledExecutorService scheduledExecutorService; - private final AtomicReference> currentPipelines = new AtomicReference<>(); - private HashMultimap streamPipelineAssignments = HashMultimap.create(); + private final AtomicReference> currentPipelines = new AtomicReference<>(); + private final AtomicReference> streamPipelineAssignments = new AtomicReference<>(ImmutableSetMultimap.of()); @Inject public PipelineInterpreter(RuleSourceService ruleSourceService, PipelineSourceService pipelineSourceService, + PipelineStreamAssignmentService pipelineStreamAssignmentService, PipelineRuleParser pipelineRuleParser, Journal journal, MetricRegistry metricRegistry, @@ -113,7 +118,9 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, try { final List ruleIds = ruleSourceService.loadAll().stream().map(RuleSource::id).collect(Collectors.toList()); log.info("Compiling {} processing rules", ruleIds.size()); + // TODO this sucks, because it is completely async and we can't use listenable futures to trigger the pipeline updates ruleCache.getAll(ruleIds); + triggerPipelineUpdate(); } catch (ExecutionException ignored) { } } @@ -140,30 +147,35 @@ public Messages process(Messages messages) { // 1. for each message, determine which pipelines are supposed to be executed, based on their streams // null is the default stream, the other streams are identified by their id - final Set pipelinesToRun; + final ImmutableSet pipelinesToRun; // this makes a copy of the list! - final Set initialStreamIds = Sets.newHashSet(message.getStreamIds()); + final Set initialStreamIds = message.getStreams().stream().map(Stream::getId).collect(Collectors.toSet()); + + final ImmutableSetMultimap streamAssignment = streamPipelineAssignments.get(); if (initialStreamIds.isEmpty()) { - if (processingBlacklist.contains(tuple(msgId, (String) null))) { + if (processingBlacklist.contains(tuple(msgId, "default"))) { // already processed default pipeline for this message - pipelinesToRun = Collections.emptySet(); + pipelinesToRun = ImmutableSet.of(); log.info("[{}] already processed default stream, skipping", msgId); } else { // get the default stream pipeline assignments for this message - pipelinesToRun = streamPipelineAssignments.get(null); + pipelinesToRun = streamAssignment.get("default"); log.info("[{}] running default stream pipelines: [{}]", msgId, pipelinesToRun.stream().map(Pipeline::name).toArray()); } } else { - pipelinesToRun = initialStreamIds.stream() - // 2. if a message-stream combination has already been processed (is in the set), skip that execution + // 2. if a message-stream combination has already been processed (is in the set), skip that execution + final Set streamsIds = initialStreamIds.stream() .filter(streamId -> !processingBlacklist.contains(tuple(msgId, streamId))) - .flatMap(streamId -> streamPipelineAssignments.get(streamId).stream()) + .filter(streamAssignment::containsKey) .collect(Collectors.toSet()); - log.info("[{}] running pipelines {}", msgId, pipelinesToRun); + pipelinesToRun = ImmutableSet.copyOf(streamsIds.stream() + .flatMap(streamId -> streamAssignment.get(streamId).stream()) + .collect(Collectors.toSet())); + log.info("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); } final StageIterator stages = new StageIterator(pipelinesToRun); @@ -222,15 +234,21 @@ public Messages process(Messages messages) { // 4. after each complete stage run, merge the processing changes, stages are isolated from each other // TODO message changes become visible immediately for now + // 4a. also add all new messages from the context to the toProcess work list + Iterables.addAll(toProcess, context.createdMessages()); + context.clearCreatedMessages(); } } boolean addedStreams = false; // 5. add each message-stream combination to the blacklist set for (Stream stream : message.getStreams()) { - processingBlacklist.add(tuple(msgId, stream.getId())); if (!initialStreamIds.remove(stream.getId())) { addedStreams = true; + } else { + // only add pre-existing streams to blacklist, this has the effect of only adding already processed streams, + // not newly added ones. + processingBlacklist.add(tuple(msgId, stream.getId())); } } if (message.getFilterOut()) { @@ -240,8 +258,8 @@ public Messages process(Messages messages) { journal.markJournalOffsetCommitted(message.getJournalOffset()); } // 6. go to 1 and iterate over all messages again until no more streams are being assigned - if (!addedStreams) { - log.info("[{}] no new streams matches, not running again", msgId); + if (!addedStreams || message.getFilterOut()) { + log.info("[{}] no new streams matches or dropped message, not running again", msgId); fullyProcessed.add(message); } else { // process again, we've added a stream @@ -266,7 +284,6 @@ public void handleRuleChanges(RulesChangedEvent event) { }); triggerPipelineUpdate(); - } @Subscribe @@ -281,19 +298,37 @@ public void handlePipelineChanges(PipelinesChangedEvent event) { triggerPipelineUpdate(); } + @Subscribe + public void handlePipelineAssignmentChanges(PipelineStreamAssignment assignment) { + // rebuild the stream -> pipelines multimap + // default stream is represented as "default" in the map + final String streamId = assignment.streamId(); + final Set pipelineIds = assignment.pipelineIds(); + + final ImmutableSetMultimap multimap = streamPipelineAssignments.get(); + final ImmutableMap pipelines = currentPipelines.get(); + + // set the new per-stream mapping + final HashMultimap newMap = HashMultimap.create(multimap); + newMap.removeAll(streamId); + pipelineIds.stream().map(pipelines::get).forEach(pipeline -> newMap.put(streamId, pipeline)); + + streamPipelineAssignments.set(ImmutableSetMultimap.copyOf(newMap)); + } + private void triggerPipelineUpdate() { Futures.addCallback( - scheduledExecutorService.schedule(new PipelineResolver(), 250, TimeUnit.MILLISECONDS), + scheduledExecutorService.schedule(new PipelineResolver(), 500, TimeUnit.MILLISECONDS), new FutureCallback>() { @Override public void onSuccess(@Nullable ImmutableSet result) { // TODO how do we deal with concurrent updates? canceling earlier attempts? - currentPipelines.set(result); - if (result != null) { - streamPipelineAssignments.putAll(null, result); + if (result == null) { + currentPipelines.set(ImmutableMap.of()); + } else { + currentPipelines.set(Maps.uniqueIndex(result, Pipeline::id)); } } - @Override public void onFailure(Throwable t) { // do not touch the existing pipeline configuration @@ -359,7 +394,7 @@ public ImmutableSet call() throws Exception { Set pipelines = Sets.newHashSetWithExpectedSize(allPipelineSources.size()); for (PipelineSource source : allPipelineSources) { try { - final Pipeline pipeline = pipelineRuleParser.parsePipeline(source.source()); + final Pipeline pipeline = pipelineRuleParser.parsePipeline(source); pipelines.add(pipeline); log.info("Parsed pipeline {} with {} stages", pipeline.name(), pipeline.stages().size()); } catch (ParseException e) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index a221cafb608a..a1fab01812e8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -47,7 +47,7 @@ import javax.ws.rs.core.Response; import java.util.Collection; -@Api(value = "Pipeline Pipelines", description = "Pipelines for the pipeline message processor") +@Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") @Path("/system/pipelines") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @@ -74,7 +74,7 @@ public PipelineResource(PipelineSourceService pipelineSourceService, @Path("/pipeline") public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { try { - pipelineRuleParser.parsePipeline(pipelineSource.source()); + pipelineRuleParser.parsePipeline(pipelineSource); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -96,7 +96,7 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t @Path("/pipeline/parse") public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { try { - pipelineRuleParser.parsePipeline(pipelineSource.source()); + pipelineRuleParser.parsePipeline(pipelineSource); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -130,7 +130,7 @@ public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { final PipelineSource pipelineSource = pipelineSourceService.load(id); try { - pipelineRuleParser.parsePipeline(update.source()); + pipelineRuleParser.parsePipeline(update); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java new file mode 100644 index 000000000000..dd62c9ba59c6 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java @@ -0,0 +1,72 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; +import java.util.Set; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineStreamAssignment { + + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String streamId(); + + @JsonProperty + public abstract Set pipelineIds(); + + @JsonCreator + public static PipelineStreamAssignment create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + @JsonProperty("stream_id") String streamId, + @JsonProperty("pipeline_ids") Set pipelineIds) { + return builder() + .id(id) + .streamId(streamId) + .pipelineIds(pipelineIds) + .build(); + } + + public static Builder builder() { + return new AutoValue_PipelineStreamAssignment.Builder(); + } + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + public abstract PipelineStreamAssignment build(); + + public abstract Builder id(String id); + + public abstract Builder streamId(String streamId); + + public abstract Builder pipelineIds(Set pipelineIds); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java new file mode 100644 index 000000000000..abfa6c112163 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java @@ -0,0 +1,94 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.eventbus.EventBus; +import com.wordnik.swagger.annotations.Api; +import com.wordnik.swagger.annotations.ApiOperation; +import com.wordnik.swagger.annotations.ApiParam; +import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog2.database.NotFoundException; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.streams.StreamService; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.Set; + +@Api(value = "Pipeline/Streams", description = "Stream assignment of processing pipelines") +@Path("/system/pipelines/streams") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +public class PipelineStreamResource extends RestResource implements PluginRestResource { + + private final PipelineStreamAssignmentService assignmentService; + private final PipelineSourceService pipelineSourceService; + private final StreamService streamService; + private final EventBus clusterBus; + + @Inject + public PipelineStreamResource(PipelineStreamAssignmentService assignmentService, + PipelineSourceService pipelineSourceService, + StreamService streamService, + @ClusterEventBus EventBus clusterBus) { + this.assignmentService = assignmentService; + this.pipelineSourceService = pipelineSourceService; + this.streamService = streamService; + this.clusterBus = clusterBus; + } + + @ApiOperation(value = "Attach a processing pipeline to a stream", notes = "") + @POST + public PipelineStreamAssignment assignPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineStreamAssignment assignment) throws NotFoundException { + final String streamId = assignment.streamId(); + // the default stream doesn't exist as an entity + if (!streamId.equalsIgnoreCase("default")) { + streamService.load(streamId); + } + // verify the pipelines exist + for (String s : assignment.pipelineIds()) { + pipelineSourceService.load(s); + } + final PipelineStreamAssignment save = assignmentService.save(assignment); + clusterBus.post(save); + return save; + } + + @ApiOperation("Get pipeline attachments for the given stream") + @GET + @Path("/{streamId}") + public PipelineStreamAssignment getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { + return assignmentService.load(streamId); + } + + @ApiOperation("Get all pipeline attachments") + @GET + public Set getAllAttachments() throws NotFoundException { + return assignmentService.loadAll(); + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index d82f9eab1cce..32f64cf82cce 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -47,7 +47,7 @@ import javax.ws.rs.core.Response; import java.util.Collection; -@Api(value = "Pipeline Rules", description = "Rules for the pipeline message processor") +@Api(value = "Pipeline/Rules", description = "Rules for the pipeline message processor") @Path("/system/pipelines") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index 689d9c0afb2f..e600f98bfec0 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -30,9 +30,9 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; -import org.graylog.plugins.pipelineprocessor.functions.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.SetField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java new file mode 100644 index 000000000000..e80e043873e8 --- /dev/null +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -0,0 +1,111 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import com.google.common.util.concurrent.Uninterruptibles; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; +import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog.plugins.pipelineprocessor.rest.RuleSource; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Messages; +import org.graylog2.plugin.Tools; +import org.graylog2.shared.journal.Journal; +import org.junit.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static com.google.common.collect.Sets.newHashSet; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PipelineInterpreterTest { + + @Test + public void testCreateMessage() { + final RuleSourceService ruleSourceService = mock(RuleSourceService.class); + when(ruleSourceService.loadAll()).thenReturn(Collections.singleton( + RuleSource.create("abc", + "title", + "description", + "rule \"creates message\"\n" + + "when string(message.`message`) == \"original message\"\n" + + "then\n" + + " create_message(\"derived message\");\n" + + "end", + Tools.nowUTC(), + null) + )); + + final PipelineSourceService pipelineSourceService = mock(PipelineSourceService.class); + when(pipelineSourceService.loadAll()).thenReturn(Collections.singleton( + PipelineSource.create("cde", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"creates message\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final Map> functions = Maps.newHashMap(); + functions.put(CreateMessage.NAME, new CreateMessage()); + functions.put(StringCoercion.NAME, new StringCoercion()); + + final FunctionRegistry functionRegistry = new FunctionRegistry(functions); + final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry); + + final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); + final PipelineStreamAssignment pipelineStreamAssignment = PipelineStreamAssignment.create(null, + "default", + newHashSet("cde")); + when(pipelineStreamAssignmentService.loadAll()).thenReturn( + newHashSet(pipelineStreamAssignment)) + ; + + final PipelineInterpreter interpreter = new PipelineInterpreter( + ruleSourceService, pipelineSourceService, pipelineStreamAssignmentService, parser, mock(Journal.class), mock(MetricRegistry.class), + Executors.newSingleThreadScheduledExecutor(), mock(EventBus.class) + ); + interpreter.handlePipelineChanges(PipelinesChangedEvent.create(Collections.emptySet(),Collections.emptySet())); + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + interpreter.handlePipelineAssignmentChanges(pipelineStreamAssignment); + + Message msg = new Message("original message", "test", Tools.nowUTC()); + final Messages processed = interpreter.process(msg); + + final Message[] messages = Iterables.toArray(processed, Message.class); + assertEquals(2, messages.length); + } + +} \ No newline at end of file From c616673cd756c9c69340a980e00efc639cdbfaba Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 3 Feb 2016 10:10:00 +0100 Subject: [PATCH 057/521] initialize the pipeline stream assignements during start --- .../pipelineprocessor/processors/PipelineInterpreter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index a64215f483f1..68d5263743b3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -123,6 +123,10 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, triggerPipelineUpdate(); } catch (ExecutionException ignored) { } + + // initialize all assignments + pipelineStreamAssignmentService.loadAll().forEach(this::handlePipelineAssignmentChanges); + } /** From 409c2d05b75c9dd2a56f40b898e061bcb9d9657a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 3 Feb 2016 10:20:10 +0100 Subject: [PATCH 058/521] make id() consistent with other uses are the codebase --- .../pipelineprocessor/rest/PipelineStreamAssignment.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java index dd62c9ba59c6..8b74bee4c21e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java @@ -34,7 +34,7 @@ public abstract class PipelineStreamAssignment { @Nullable @Id @ObjectId - public abstract String id(); + public abstract String _id(); @JsonProperty public abstract String streamId(); @@ -43,11 +43,11 @@ public abstract class PipelineStreamAssignment { public abstract Set pipelineIds(); @JsonCreator - public static PipelineStreamAssignment create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + public static PipelineStreamAssignment create(@Id @ObjectId @JsonProperty("id") @Nullable String _id, @JsonProperty("stream_id") String streamId, @JsonProperty("pipeline_ids") Set pipelineIds) { return builder() - .id(id) + ._id(_id) .streamId(streamId) .pipelineIds(pipelineIds) .build(); @@ -63,7 +63,7 @@ public static Builder builder() { public abstract static class Builder { public abstract PipelineStreamAssignment build(); - public abstract Builder id(String id); + public abstract Builder _id(String _id); public abstract Builder streamId(String streamId); From a69dfe480333c60e013ba21ac632a671f470d818 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 3 Feb 2016 13:12:43 +0100 Subject: [PATCH 059/521] restructure test case in preparation refactoring rule, pipeline, assignment loading/refreshing get rid of NPEs in pipeline interpreter --- .../processors/PipelineInterpreter.java | 8 +++- .../processors/PipelineInterpreterTest.java | 38 ++++++++++--------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 68d5263743b3..d177cbcf4cd9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -71,6 +71,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -93,7 +94,7 @@ public class PipelineInterpreter implements MessageProcessor { private final LoadingCache ruleCache; private final ListeningScheduledExecutorService scheduledExecutorService; - private final AtomicReference> currentPipelines = new AtomicReference<>(); + private final AtomicReference> currentPipelines = new AtomicReference<>(ImmutableMap.of()); private final AtomicReference> streamPipelineAssignments = new AtomicReference<>(ImmutableSetMultimap.of()); @Inject @@ -315,7 +316,10 @@ public void handlePipelineAssignmentChanges(PipelineStreamAssignment assignment) // set the new per-stream mapping final HashMultimap newMap = HashMultimap.create(multimap); newMap.removeAll(streamId); - pipelineIds.stream().map(pipelines::get).forEach(pipeline -> newMap.put(streamId, pipeline)); + pipelineIds.stream() + .map(pipelines::get) + .filter(Objects::nonNull) + .forEach(pipeline -> newMap.put(streamId, pipeline)); streamPipelineAssignments.set(ImmutableSetMultimap.copyOf(newMap)); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index e80e043873e8..c221c5966899 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -20,12 +20,10 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.eventbus.EventBus; -import com.google.common.util.concurrent.Uninterruptibles; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; -import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; @@ -42,7 +40,6 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import static com.google.common.collect.Sets.newHashSet; import static org.junit.Assert.assertEquals; @@ -78,28 +75,30 @@ public void testCreateMessage() { null) )); - final Map> functions = Maps.newHashMap(); - functions.put(CreateMessage.NAME, new CreateMessage()); - functions.put(StringCoercion.NAME, new StringCoercion()); - - final FunctionRegistry functionRegistry = new FunctionRegistry(functions); - final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry); - final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); final PipelineStreamAssignment pipelineStreamAssignment = PipelineStreamAssignment.create(null, "default", newHashSet("cde")); when(pipelineStreamAssignmentService.loadAll()).thenReturn( - newHashSet(pipelineStreamAssignment)) - ; + newHashSet(pipelineStreamAssignment) + ); + + final Map> functions = Maps.newHashMap(); + functions.put(CreateMessage.NAME, new CreateMessage()); + functions.put(StringCoercion.NAME, new StringCoercion()); + + final PipelineRuleParser parser = setupParser(functions); final PipelineInterpreter interpreter = new PipelineInterpreter( - ruleSourceService, pipelineSourceService, pipelineStreamAssignmentService, parser, mock(Journal.class), mock(MetricRegistry.class), - Executors.newSingleThreadScheduledExecutor(), mock(EventBus.class) + ruleSourceService, + pipelineSourceService, + pipelineStreamAssignmentService, + parser, + mock(Journal.class), + mock(MetricRegistry.class), + Executors.newSingleThreadScheduledExecutor(), + mock(EventBus.class) ); - interpreter.handlePipelineChanges(PipelinesChangedEvent.create(Collections.emptySet(),Collections.emptySet())); - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); - interpreter.handlePipelineAssignmentChanges(pipelineStreamAssignment); Message msg = new Message("original message", "test", Tools.nowUTC()); final Messages processed = interpreter.process(msg); @@ -108,4 +107,9 @@ ruleSourceService, pipelineSourceService, pipelineStreamAssignmentService, parse assertEquals(2, messages.length); } + private PipelineRuleParser setupParser(Map> functions) { + final FunctionRegistry functionRegistry = new FunctionRegistry(functions); + return new PipelineRuleParser(functionRegistry); + } + } \ No newline at end of file From 821a769f2feaae02ca86581278aae50b9a06628c Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 3 Feb 2016 16:46:53 +0100 Subject: [PATCH 060/521] fix initial loading of rules, pipelines, stream assignments pending additional tests --- .../processors/PipelineInterpreter.java | 237 +++++------------- .../processors/PipelineInterpreterTest.java | 2 - 2 files changed, 65 insertions(+), 174 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index d177cbcf4cd9..01a382b6df58 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -18,9 +18,6 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -31,10 +28,6 @@ import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.ast.Rule; @@ -58,41 +51,31 @@ import org.graylog2.plugin.streams.Stream; import org.graylog2.shared.buffers.processors.ProcessBufferProcessor; import org.graylog2.shared.journal.Journal; -import org.jooq.lambda.Seq; import org.jooq.lambda.tuple.Tuple2; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import javax.inject.Inject; -import javax.inject.Named; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static com.codahale.metrics.MetricRegistry.name; -import static com.google.common.cache.CacheLoader.asyncReloading; import static org.jooq.lambda.tuple.Tuple.tuple; public class PipelineInterpreter implements MessageProcessor { private static final Logger log = LoggerFactory.getLogger(PipelineInterpreter.class); + private final RuleSourceService ruleSourceService; private final PipelineSourceService pipelineSourceService; + private final PipelineStreamAssignmentService pipelineStreamAssignmentService; private final PipelineRuleParser pipelineRuleParser; private final Journal journal; private final Meter filteredOutMessages; - private final LoadingCache ruleCache; - private final ListeningScheduledExecutorService scheduledExecutorService; private final AtomicReference> currentPipelines = new AtomicReference<>(ImmutableMap.of()); private final AtomicReference> streamPipelineAssignments = new AtomicReference<>(ImmutableSetMultimap.of()); @@ -104,29 +87,76 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, PipelineRuleParser pipelineRuleParser, Journal journal, MetricRegistry metricRegistry, - @Named("daemonScheduler") ScheduledExecutorService scheduledExecutorService, @ClusterEventBus EventBus clusterBus) { + this.ruleSourceService = ruleSourceService; this.pipelineSourceService = pipelineSourceService; + this.pipelineStreamAssignmentService = pipelineStreamAssignmentService; this.pipelineRuleParser = pipelineRuleParser; + this.journal = journal; - this.scheduledExecutorService = MoreExecutors.listeningDecorator(scheduledExecutorService); this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); + + // listens to cluster wide Rule, Pipeline and pipeline stream assignment changes clusterBus.register(this); - ruleCache = CacheBuilder.newBuilder() - .build(asyncReloading(new RuleLoader(ruleSourceService, pipelineRuleParser), scheduledExecutorService)); - - // prime the cache with all presently stored rules - try { - final List ruleIds = ruleSourceService.loadAll().stream().map(RuleSource::id).collect(Collectors.toList()); - log.info("Compiling {} processing rules", ruleIds.size()); - // TODO this sucks, because it is completely async and we can't use listenable futures to trigger the pipeline updates - ruleCache.getAll(ruleIds); - triggerPipelineUpdate(); - } catch (ExecutionException ignored) { + + reload(); + } + + private void reload() { + // read all rules and compile them + Map ruleNameMap = Maps.newHashMap(); + for (RuleSource ruleSource : ruleSourceService.loadAll()) { + Rule rule; + try { + rule = pipelineRuleParser.parseRule(ruleSource.source()); + } catch (ParseException e) { + rule = Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id()); + } + ruleNameMap.put(rule.name(), rule); + } + + Map pipelineIdMap = Maps.newHashMap(); + // read all pipelines and compile them + for (PipelineSource pipelineSource : pipelineSourceService.loadAll()) { + Pipeline pipeline; + try { + pipeline = pipelineRuleParser.parsePipeline(pipelineSource); + } catch (ParseException e) { + pipeline = Pipeline.empty("Failed to parse pipeline" + pipelineSource.id()); + } + pipelineIdMap.put(pipelineSource.id(), pipeline); } - // initialize all assignments - pipelineStreamAssignmentService.loadAll().forEach(this::handlePipelineAssignmentChanges); + // resolve all rules in the stages + pipelineIdMap.values().stream() + .flatMap(pipeline -> { + log.info("Resolving pipeline {}", pipeline.name()); + return pipeline.stages().stream(); + }) + .forEach(stage -> { + final List resolvedRules = stage.ruleReferences().stream(). + map(ref -> { + Rule rule = ruleNameMap.get(ref); + if (rule == null) { + rule = Rule.alwaysFalse("Unresolved rule " + ref); + } + log.info("Resolved rule `{}` to {}", ref, rule); + return rule; + }) + .collect(Collectors.toList()); + stage.setRules(resolvedRules); + }); + currentPipelines.set(ImmutableMap.copyOf(pipelineIdMap)); + + // read all stream assignments of those pipelines to allow processing messages through them + final HashMultimap assignments = HashMultimap.create(); + for (PipelineStreamAssignment streamAssignment : pipelineStreamAssignmentService.loadAll()) { + streamAssignment.pipelineIds().stream() + .map(pipelineIdMap::get) + .filter(Objects::nonNull) + .forEach(pipeline -> assignments.put(streamAssignment.streamId(), pipeline)); + } + streamPipelineAssignments.set(ImmutableSetMultimap.copyOf(assignments)); } @@ -280,15 +310,11 @@ public Messages process(Messages messages) { @Subscribe public void handleRuleChanges(RulesChangedEvent event) { event.deletedRuleIds().forEach(id -> { - ruleCache.invalidate(id); log.info("Invalidated rule {}", id); }); event.updatedRuleIds().forEach(id -> { - ruleCache.refresh(id); log.info("Refreshing rule {}", id); }); - - triggerPipelineUpdate(); } @Subscribe @@ -299,144 +325,11 @@ public void handlePipelineChanges(PipelinesChangedEvent event) { event.updatedPipelineIds().forEach(id -> { log.info("Refreshing pipeline {}", id); }); - - triggerPipelineUpdate(); } @Subscribe public void handlePipelineAssignmentChanges(PipelineStreamAssignment assignment) { - // rebuild the stream -> pipelines multimap - // default stream is represented as "default" in the map - final String streamId = assignment.streamId(); - final Set pipelineIds = assignment.pipelineIds(); - - final ImmutableSetMultimap multimap = streamPipelineAssignments.get(); - final ImmutableMap pipelines = currentPipelines.get(); - - // set the new per-stream mapping - final HashMultimap newMap = HashMultimap.create(multimap); - newMap.removeAll(streamId); - pipelineIds.stream() - .map(pipelines::get) - .filter(Objects::nonNull) - .forEach(pipeline -> newMap.put(streamId, pipeline)); - - streamPipelineAssignments.set(ImmutableSetMultimap.copyOf(newMap)); - } - - private void triggerPipelineUpdate() { - Futures.addCallback( - scheduledExecutorService.schedule(new PipelineResolver(), 500, TimeUnit.MILLISECONDS), - new FutureCallback>() { - @Override - public void onSuccess(@Nullable ImmutableSet result) { - // TODO how do we deal with concurrent updates? canceling earlier attempts? - if (result == null) { - currentPipelines.set(ImmutableMap.of()); - } else { - currentPipelines.set(Maps.uniqueIndex(result, Pipeline::id)); - } - } - @Override - public void onFailure(Throwable t) { - // do not touch the existing pipeline configuration - log.error("Unable to update pipeline processor", t); - } - }); - } - - private static class RuleLoader extends CacheLoader { - private final RuleSourceService ruleSourceService; - private final PipelineRuleParser pipelineRuleParser; - - public RuleLoader(RuleSourceService ruleSourceService, PipelineRuleParser pipelineRuleParser) { - this.ruleSourceService = ruleSourceService; - this.pipelineRuleParser = pipelineRuleParser; - } - - @Override - public Map loadAll(Iterable keys) throws Exception { - final Map all = Maps.newHashMap(); - final HashSet keysToLoad = Sets.newHashSet(keys); - for (RuleSource ruleSource : ruleSourceService.loadAll()) { - if (!keysToLoad.isEmpty()) { - if (!keysToLoad.contains(ruleSource.id())) { - continue; - } - } - try { - all.put(ruleSource.id(), pipelineRuleParser.parseRule(ruleSource.source())); - } catch (ParseException e) { - log.error("Unable to parse rule: " + e.getMessage()); - all.put(ruleSource.id(), Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id())); - } - } - return all; - } - - @Override - public Rule load(@Nullable String ruleId) throws Exception { - final RuleSource ruleSource = ruleSourceService.load(ruleId); - try { - return pipelineRuleParser.parseRule(ruleSource.source()); - } catch (ParseException e) { - log.error("Unable to parse rule: " + e.getMessage()); - // return dummy rule - return Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id()); - } - } - } - - private class PipelineResolver implements Callable> { - private final Logger log = LoggerFactory.getLogger(PipelineResolver.class); - - - @Override - public ImmutableSet call() throws Exception { - log.info("Updating pipeline processor after rule/pipeline update"); - - final Collection allPipelineSources = pipelineSourceService.loadAll(); - log.info("Found {} pipelines to resolve", allPipelineSources.size()); - - // compile all pipelines - Set pipelines = Sets.newHashSetWithExpectedSize(allPipelineSources.size()); - for (PipelineSource source : allPipelineSources) { - try { - final Pipeline pipeline = pipelineRuleParser.parsePipeline(source); - pipelines.add(pipeline); - log.info("Parsed pipeline {} with {} stages", pipeline.name(), pipeline.stages().size()); - } catch (ParseException e) { - log.warn("Unable to compile pipeline {}: {}", source.title(), e); - } - } - - // change the rule cache to be able to quickly look up rules by name - final Map nameRuleMap = - Seq.toMap(Seq.seq(ruleCache.asMap()) - .map(entry -> tuple(entry.v2().name(), entry.v2()))); - - // resolve all rules - pipelines.stream() - .flatMap(pipeline -> { - log.info("Resolving pipeline {}", pipeline.name()); - return pipeline.stages().stream(); - }) - .forEach(stage -> { - final List resolvedRules = stage.ruleReferences().stream(). - map(ref -> { - Rule rule = nameRuleMap.get(ref); - if (rule == null) { - rule = Rule.alwaysFalse("Unresolved rule " + ref); - } - log.info("Resolved rule `{}` to {}", ref, rule); - return rule; - }) - .collect(Collectors.toList()); - stage.setRules(resolvedRules); - }); - - return ImmutableSet.copyOf(pipelines); - } + log.info("Pipeline stream assignment changed: {}", assignment); } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index c221c5966899..e4b2310e269a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -39,7 +39,6 @@ import java.util.Collections; import java.util.Map; -import java.util.concurrent.Executors; import static com.google.common.collect.Sets.newHashSet; import static org.junit.Assert.assertEquals; @@ -96,7 +95,6 @@ public void testCreateMessage() { parser, mock(Journal.class), mock(MetricRegistry.class), - Executors.newSingleThreadScheduledExecutor(), mock(EventBus.class) ); From a15925ed90b2207e1a789cec1f0eb355ceb9b29b Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 4 Feb 2016 14:46:39 +0100 Subject: [PATCH 061/521] trigger wholesale rule/pipeline etc reload after any changes this can likely be optimized to load less things, but it is happening in the background and should not interfere at the moment --- .../processors/PipelineInterpreter.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 01a382b6df58..e329df2c4399 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -56,11 +56,14 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; +import javax.inject.Named; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -75,6 +78,7 @@ public class PipelineInterpreter implements MessageProcessor { private final PipelineStreamAssignmentService pipelineStreamAssignmentService; private final PipelineRuleParser pipelineRuleParser; private final Journal journal; + private final ScheduledExecutorService scheduler; private final Meter filteredOutMessages; private final AtomicReference> currentPipelines = new AtomicReference<>(ImmutableMap.of()); @@ -87,6 +91,7 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, PipelineRuleParser pipelineRuleParser, Journal journal, MetricRegistry metricRegistry, + @Named("daemonScheduler") ScheduledExecutorService scheduler, @ClusterEventBus EventBus clusterBus) { this.ruleSourceService = ruleSourceService; this.pipelineSourceService = pipelineSourceService; @@ -94,6 +99,7 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, this.pipelineRuleParser = pipelineRuleParser; this.journal = journal; + this.scheduler = scheduler; this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); // listens to cluster wide Rule, Pipeline and pipeline stream assignment changes @@ -102,7 +108,8 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, reload(); } - private void reload() { + // this should not run in parallel + private synchronized void reload() { // read all rules and compile them Map ruleNameMap = Maps.newHashMap(); for (RuleSource ruleSource : ruleSourceService.loadAll()) { @@ -315,6 +322,7 @@ public void handleRuleChanges(RulesChangedEvent event) { event.updatedRuleIds().forEach(id -> { log.info("Refreshing rule {}", id); }); + scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } @Subscribe @@ -325,11 +333,13 @@ public void handlePipelineChanges(PipelinesChangedEvent event) { event.updatedPipelineIds().forEach(id -> { log.info("Refreshing pipeline {}", id); }); + scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } @Subscribe public void handlePipelineAssignmentChanges(PipelineStreamAssignment assignment) { log.info("Pipeline stream assignment changed: {}", assignment); + scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } } From 9c45f47da698c1dd912b90765576ce44333c95e5 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 4 Feb 2016 18:28:01 +0100 Subject: [PATCH 062/521] fix tests remove obsolete file --- .../processors/NaiveRuleProcessor.java | 171 ------------------ .../processors/PipelineInterpreterTest.java | 2 + 2 files changed, 2 insertions(+), 171 deletions(-) delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java deleted file mode 100644 index 11928ba0193e..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/NaiveRuleProcessor.java +++ /dev/null @@ -1,171 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor.processors; - -import com.codahale.metrics.Meter; -import com.codahale.metrics.MetricRegistry; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.eventbus.EventBus; -import com.google.common.eventbus.Subscribe; -import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.Rule; -import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; -import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; -import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; -import org.graylog.plugins.pipelineprocessor.parser.ParseException; -import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.RuleSource; -import org.graylog2.events.ClusterEventBus; -import org.graylog2.plugin.Message; -import org.graylog2.plugin.Messages; -import org.graylog2.plugin.messageprocessors.MessageProcessor; -import org.graylog2.shared.buffers.processors.ProcessBufferProcessor; -import org.graylog2.shared.journal.Journal; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.inject.Named; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.stream.Collectors; - -import static com.codahale.metrics.MetricRegistry.name; -import static com.google.common.cache.CacheLoader.asyncReloading; - -public class NaiveRuleProcessor implements MessageProcessor { - private static final Logger log = LoggerFactory.getLogger(NaiveRuleProcessor.class); - - private final Journal journal; - private final Meter filteredOutMessages; - private final LoadingCache ruleCache; - - @Inject - public NaiveRuleProcessor(RuleSourceService ruleSourceService, - PipelineRuleParser pipelineRuleParser, - Journal journal, - MetricRegistry metricRegistry, - @Named("daemonScheduler") ScheduledExecutorService scheduledExecutorService, - @ClusterEventBus EventBus clusterBus) { - this.journal = journal; - this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); - clusterBus.register(this); - ruleCache = CacheBuilder.newBuilder() - .build(asyncReloading(new RuleLoader(ruleSourceService, pipelineRuleParser), scheduledExecutorService)); - // prime the cache with all presently stored rules - try { - final List ruleIds = ruleSourceService.loadAll().stream().map(RuleSource::id).collect(Collectors.toList()); - log.info("Compiling {} processing rules", ruleIds.size()); - ruleCache.getAll(ruleIds); - } catch (ExecutionException ignored) {} - } - - @Override - public Messages process(Messages messages) { - for (Map.Entry entry : ruleCache.asMap().entrySet()) { - final Rule rule = entry.getValue(); - log.info("Evaluating rule {}", rule.name()); - - for (Message message : messages) { - try { - final EvaluationContext context = new EvaluationContext(message); - if (rule.when().evaluateBool(context)) { - log.info("[✓] Message {} matches condition", message.getId()); - - for (Statement statement : rule.then()) { - statement.evaluate(context); - } - - } else { - log.info("[✕] Message {} does not match condition", message.getId()); - } - } catch (Exception e) { - log.error("Unable to process message " + message.getId() + ", running rule " + rule.name(), e); - } - if (message.getFilterOut()) { - log.info("[✝] Message {} was filtered out", message.getId()); - - filteredOutMessages.mark(); - journal.markJournalOffsetCommitted(message.getJournalOffset()); - } - } - } - return messages; - } - - @Subscribe - public void handleRuleChanges(RulesChangedEvent event) { - event.deletedRuleIds().forEach(id -> { - ruleCache.invalidate(id); - log.info("Invalidated rule {}", id); - }); - event.updatedRuleIds().forEach(id -> { - ruleCache.refresh(id); - log.info("Refreshing rule {}", id); - }); - } - - private static class RuleLoader extends CacheLoader { - private final RuleSourceService ruleSourceService; - private final PipelineRuleParser pipelineRuleParser; - - public RuleLoader(RuleSourceService ruleSourceService, PipelineRuleParser pipelineRuleParser) { - this.ruleSourceService = ruleSourceService; - this.pipelineRuleParser = pipelineRuleParser; - } - - @Override - public Map loadAll(Iterable keys) throws Exception { - final Map all = Maps.newHashMap(); - final HashSet keysToLoad = Sets.newHashSet(keys); - for (RuleSource ruleSource : ruleSourceService.loadAll()) { - if (!keysToLoad.isEmpty()) { - if (!keysToLoad.contains(ruleSource.id())) { - continue; - } - } - try { - all.put(ruleSource.id(), pipelineRuleParser.parseRule(ruleSource.source())); - } catch (ParseException e) { - log.error("Unable to parse rule: " + e.getMessage()); - all.put(ruleSource.id(), Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id())); - } - } - return all; - } - - @Override - public Rule load(@Nullable String ruleId) throws Exception { - final RuleSource ruleSource = ruleSourceService.load(ruleId); - try { - return pipelineRuleParser.parseRule(ruleSource.source()); - } catch (ParseException e) { - log.error("Unable to parse rule: " + e.getMessage()); - // return dummy rule - return Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id()); - } - } - } -} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index e4b2310e269a..867f796ee6a2 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -39,6 +39,7 @@ import java.util.Collections; import java.util.Map; +import java.util.concurrent.Executors; import static com.google.common.collect.Sets.newHashSet; import static org.junit.Assert.assertEquals; @@ -95,6 +96,7 @@ public void testCreateMessage() { parser, mock(Journal.class), mock(MetricRegistry.class), + Executors.newScheduledThreadPool(1), mock(EventBus.class) ); From 3974ccb248d4a747efc59e610f33ddd831078eb4 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 5 Feb 2016 17:52:47 +0100 Subject: [PATCH 063/521] add benchmark project - update to 2.0.0-alpha.2-SNAPSHOT - reduce log level to debug in interpreter --- benchmarks/pom.xml | 189 ++++++++++++ .../pipeline/PipelineBenchmark.java | 268 ++++++++++++++++++ .../processors/PipelineInterpreter.java | 36 +-- src/test/resources/log4j2-test.xml | 1 + 4 files changed, 476 insertions(+), 18 deletions(-) create mode 100644 benchmarks/pom.xml create mode 100644 benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml new file mode 100644 index 000000000000..13207b9d945e --- /dev/null +++ b/benchmarks/pom.xml @@ -0,0 +1,189 @@ + + + + 4.0.0 + + org.graylog.benchmarks + pipeline-benchmarks + 1.0.0-SNAPSHOT + jar + + Message processing benchmarks + + + 3.3 + + + + + + org.graylog2 + graylog2-server + ${graylog.version} + + + org.graylog2 + graylog2-plugin + ${graylog.version} + + + org.graylog.plugins + pipeline-processor + 1.0.0-SNAPSHOT + + + + junit + junit + 4.12 + + + org.mockito + mockito-core + 2.0.40-beta + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + UTF-8 + 1.11.3 + 1.8 + pipeline-benchmarks + 2.0.0-alpha.2-SNAPSHOT + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${javac.target} + ${javac.target} + ${javac.target} + + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + maven-clean-plugin + 2.5 + + + maven-deploy-plugin + 2.8.1 + + + maven-install-plugin + 2.5.1 + + + maven-jar-plugin + 2.4 + + + maven-javadoc-plugin + 2.9.1 + + + maven-resources-plugin + 2.6 + + + maven-site-plugin + 3.3 + + + maven-source-plugin + 2.2.1 + + + maven-surefire-plugin + 2.17 + + + + + + diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java new file mode 100644 index 000000000000..c3d7eea90359 --- /dev/null +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2014, Oracle America, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Oracle nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.graylog.benchmarks.pipeline; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog.plugins.pipelineprocessor.rest.RuleSource; +import org.graylog2.Configuration; +import org.graylog2.database.NotFoundException; +import org.graylog2.filters.ExtractorFilter; +import org.graylog2.filters.FilterService; +import org.graylog2.filters.RulesFilter; +import org.graylog2.filters.StaticFieldFilter; +import org.graylog2.filters.StreamMatcherFilter; +import org.graylog2.inputs.InputService; +import org.graylog2.messageprocessors.MessageFilterChainProcessor; +import org.graylog2.notifications.NotificationService; +import org.graylog2.plugin.LocalMetricRegistry; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.ServerStatus; +import org.graylog2.plugin.Tools; +import org.graylog2.plugin.filters.MessageFilter; +import org.graylog2.rules.DroolsEngine; +import org.graylog2.shared.journal.Journal; +import org.graylog2.shared.stats.ThroughputStats; +import org.graylog2.streams.StreamFaultManager; +import org.graylog2.streams.StreamMetrics; +import org.graylog2.streams.StreamRouter; +import org.graylog2.streams.StreamRouterEngine; +import org.graylog2.streams.StreamService; +import org.mockito.Mockito; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +import static com.google.common.collect.Sets.newHashSet; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +@SuppressWarnings("Duplicates") +public class PipelineBenchmark { + + @State(Scope.Benchmark) + public static class InterpreterState { + + private final PipelineInterpreter interpreter; + + public InterpreterState() { + final RuleSourceService ruleSourceService = mock(RuleSourceService.class); + when(ruleSourceService.loadAll()).thenReturn(Collections.singleton( + RuleSource.create("abc", + "title", + "description", + "rule \"add\"\n" + + "when string(message.`message`) == \"original message\"\n" + + "then\n" + + " set_field(\"field\", \"derived message\");\n" + + "end", + Tools.nowUTC(), + null) + )); + + final PipelineSourceService pipelineSourceService = mock(PipelineSourceService.class); + when(pipelineSourceService.loadAll()).thenReturn(Collections.singleton( + PipelineSource.create("cde", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"add\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); + final PipelineStreamAssignment pipelineStreamAssignment = PipelineStreamAssignment.create(null, + "default", + newHashSet("cde")); + when(pipelineStreamAssignmentService.loadAll()).thenReturn( + newHashSet(pipelineStreamAssignment) + ); + + final Map> functions = Maps.newHashMap(); + functions.put(SetField.NAME, new SetField()); + functions.put(StringCoercion.NAME, new StringCoercion()); + + final PipelineRuleParser parser = setupParser(functions); + + interpreter = new PipelineInterpreter( + ruleSourceService, + pipelineSourceService, + pipelineStreamAssignmentService, + parser, + mock(Journal.class), + mock(MetricRegistry.class), + Executors.newScheduledThreadPool(1), + mock(EventBus.class) + ); + } + + private PipelineRuleParser setupParser(Map> functions) { + final FunctionRegistry functionRegistry = new FunctionRegistry(functions); + return new PipelineRuleParser(functionRegistry); + } + + } + + @Benchmark + public void testPipeline(final InterpreterState state) { + Message msg = new Message("original message", "test", Tools.nowUTC()); + state.interpreter.process(msg); + } + + @State(Scope.Benchmark) + public static class MessageFilterState { + + @Param("false") + public boolean noDrools; + + public MessageFilterChainProcessor filterChain; + + @Setup + public void setup() { + try { + // common objects + final Set filters = Sets.newHashSet(); + final LocalMetricRegistry metricRegistry = new LocalMetricRegistry(); + final ServerStatus serverStatus = mock(ServerStatus.class); + when(serverStatus.getDetailedMessageRecordingStrategy()).thenReturn(ServerStatus.MessageDetailRecordingStrategy.NEVER); + + final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setDaemon(true).build()); + + final InputService inputService = mock(InputService.class); + // extractors for the single input we are pretending to have + when(inputService.find(anyString())).thenReturn(null); + + when(inputService.getExtractors(any())).thenReturn(Lists.newArrayList()); + + // static fields + Map.Entry staticField = new HashMap.SimpleEntry<>("field", "derived message"); + when(inputService.getStaticFields(any())).thenReturn(ImmutableList.of(staticField)); + + // stream router + final StreamService streamService = mock(StreamService.class); + final StreamRouterEngine.Factory engineFactory = mock(StreamRouterEngine.Factory.class); + final StreamMetrics streamMetrics = new StreamMetrics(metricRegistry); + final StreamFaultManager streamFaultManager = new StreamFaultManager( + new Configuration(), + streamMetrics, + mock(NotificationService.class), + streamService + ); + ExecutorService daemonExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).build()); + when(engineFactory.create(any(), any())).thenReturn( + new StreamRouterEngine(Collections.emptyList(), + daemonExecutor, + streamFaultManager, + streamMetrics) + ); + final StreamRouter streamRouter = new StreamRouter(streamService, + serverStatus, + engineFactory, + scheduler); + + // drools + final DroolsEngine droolsEngine = new DroolsEngine(Collections.emptySet()); + final FilterService filterService = mock(FilterService.class); + when(filterService.loadAll()).thenReturn(Collections.emptySet()); + + filters.add(new ExtractorFilter(inputService)); + filters.add(new StaticFieldFilter(inputService)); + filters.add(new StreamMatcherFilter(streamRouter, mock(ThroughputStats.class))); + if (!noDrools) { + filters.add(new RulesFilter(droolsEngine, filterService)); + } + + filterChain = new MessageFilterChainProcessor(metricRegistry, + filters, + mock(Journal.class), + serverStatus); + } catch (NotFoundException e) { + e.printStackTrace(); + } + } + } + + @Benchmark + public void testMessageFilterChain(final MessageFilterState state) { + Message msg = new Message("original message", "test", Tools.nowUTC()); + state.filterChain.process(msg); + } + + public static T mock(Class classToMock) { + return Mockito.mock(classToMock, withSettings().stubOnly()); + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(PipelineBenchmark.class.getSimpleName()) + .warmupIterations(5) + .measurementIterations(5) + .threads(1) + .forks(1) + .build(); + + new Runner(opt).run(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index e329df2c4399..22f86fc13319 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -137,7 +137,7 @@ private synchronized void reload() { // resolve all rules in the stages pipelineIdMap.values().stream() .flatMap(pipeline -> { - log.info("Resolving pipeline {}", pipeline.name()); + log.debug("Resolving pipeline {}", pipeline.name()); return pipeline.stages().stream(); }) .forEach(stage -> { @@ -147,7 +147,7 @@ private synchronized void reload() { if (rule == null) { rule = Rule.alwaysFalse("Unresolved rule " + ref); } - log.info("Resolved rule `{}` to {}", ref, rule); + log.debug("Resolved rule `{}` to {}", ref, rule); return rule; }) .collect(Collectors.toList()); @@ -200,11 +200,11 @@ public Messages process(Messages messages) { if (processingBlacklist.contains(tuple(msgId, "default"))) { // already processed default pipeline for this message pipelinesToRun = ImmutableSet.of(); - log.info("[{}] already processed default stream, skipping", msgId); + log.debug("[{}] already processed default stream, skipping", msgId); } else { // get the default stream pipeline assignments for this message pipelinesToRun = streamAssignment.get("default"); - log.info("[{}] running default stream pipelines: [{}]", + log.debug("[{}] running default stream pipelines: [{}]", msgId, pipelinesToRun.stream().map(Pipeline::name).toArray()); } @@ -217,7 +217,7 @@ public Messages process(Messages messages) { pipelinesToRun = ImmutableSet.copyOf(streamsIds.stream() .flatMap(streamId -> streamAssignment.get(streamId).stream()) .collect(Collectors.toSet())); - log.info("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); + log.debug("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); } final StageIterator stages = new StageIterator(pipelinesToRun); @@ -232,12 +232,12 @@ public Messages process(Messages messages) { final Pipeline pipeline = pair.v2(); if (!pipelinesToProceedWith.isEmpty() && !pipelinesToProceedWith.contains(pipeline)) { - log.info("[{}] previous stage result prevents further processing of pipeline `{}`", + log.debug("[{}] previous stage result prevents further processing of pipeline `{}`", msgId, pipeline.name()); continue; } - log.info("[{}] evaluating rule conditions in stage {}: match {}", + log.debug("[{}] evaluating rule conditions in stage {}: match {}", msgId, stage.stage(), stage.matchAll() ? "all" : "either"); @@ -249,14 +249,14 @@ public Messages process(Messages messages) { final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); for (Rule rule : stage.getRules()) { if (rule.when().evaluateBool(context)) { - log.info("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); + log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); rulesToRun.add(rule); } else { - log.info("[{}] rule `{}` does not match", msgId, rule.name()); + log.debug("[{}] rule `{}` does not match", msgId, rule.name()); } } for (Rule rule : rulesToRun) { - log.info("[{}] rule `{}` matched running actions", msgId, rule.name()); + log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); for (Statement statement : rule.then()) { statement.evaluate(context); } @@ -268,7 +268,7 @@ public Messages process(Messages messages) { // record that it is ok to proceed with the pipeline if ((stage.matchAll() && (rulesToRun.size() == stage.getRules().size())) || (rulesToRun.size() > 0)) { - log.info("[{}] stage for pipeline `{}` required match: {}, ok to proceed with next stage", + log.debug("[{}] stage for pipeline `{}` required match: {}, ok to proceed with next stage", msgId, pipeline.name(), stage.matchAll() ? "all" : "either"); pipelinesToProceedWith.add(pipeline); } @@ -301,11 +301,11 @@ public Messages process(Messages messages) { } // 6. go to 1 and iterate over all messages again until no more streams are being assigned if (!addedStreams || message.getFilterOut()) { - log.info("[{}] no new streams matches or dropped message, not running again", msgId); + log.debug("[{}] no new streams matches or dropped message, not running again", msgId); fullyProcessed.add(message); } else { // process again, we've added a stream - log.info("[{}] new streams assigned, running again for those streams", msgId); + log.debug("[{}] new streams assigned, running again for those streams", msgId); toProcess.add(message); } } @@ -317,10 +317,10 @@ public Messages process(Messages messages) { @Subscribe public void handleRuleChanges(RulesChangedEvent event) { event.deletedRuleIds().forEach(id -> { - log.info("Invalidated rule {}", id); + log.debug("Invalidated rule {}", id); }); event.updatedRuleIds().forEach(id -> { - log.info("Refreshing rule {}", id); + log.debug("Refreshing rule {}", id); }); scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } @@ -328,17 +328,17 @@ public void handleRuleChanges(RulesChangedEvent event) { @Subscribe public void handlePipelineChanges(PipelinesChangedEvent event) { event.deletedPipelineIds().forEach(id -> { - log.info("Invalidated pipeline {}", id); + log.debug("Invalidated pipeline {}", id); }); event.updatedPipelineIds().forEach(id -> { - log.info("Refreshing pipeline {}", id); + log.debug("Refreshing pipeline {}", id); }); scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } @Subscribe public void handlePipelineAssignmentChanges(PipelineStreamAssignment assignment) { - log.info("Pipeline stream assignment changed: {}", assignment); + log.debug("Pipeline stream assignment changed: {}", assignment); scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index 4e08ed87f895..61a7c409ae4f 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -8,6 +8,7 @@ + From 976778f27176272f1c04a1009a29291bf5cfd43c Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sat, 6 Feb 2016 14:59:57 +0100 Subject: [PATCH 064/521] tweak benchmark --- .../graylog/benchmarks/pipeline/PipelineBenchmark.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index c3d7eea90359..dd5d034680a8 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -62,6 +62,7 @@ import org.graylog2.notifications.NotificationService; import org.graylog2.plugin.LocalMetricRegistry; import org.graylog2.plugin.Message; +import org.graylog2.plugin.Messages; import org.graylog2.plugin.ServerStatus; import org.graylog2.plugin.Tools; import org.graylog2.plugin.filters.MessageFilter; @@ -166,9 +167,9 @@ private PipelineRuleParser setupParser(Map T mock(Class classToMock) { From 3d57b7b6307470c39afd24058846165afe2c1c8f Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 9 Feb 2016 17:29:07 +0100 Subject: [PATCH 065/521] add remove_field function --- .../PipelineProcessorModule.java | 3 ++ .../functions/messages/RemoveField.java | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index b87a4940529b..d6320659bab1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -28,6 +28,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; @@ -64,6 +65,8 @@ protected void configure() { // message related functions addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(SetField.NAME, SetField.class); + addMessageProcessorFunction(RemoveField.NAME, RemoveField.class); + addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); addMessageProcessorFunction(CreateMessage.NAME, CreateMessage.class); addMessageProcessorFunction(RouteToStream.NAME, RouteToStream.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java new file mode 100644 index 000000000000..4a26780e2a44 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -0,0 +1,34 @@ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Optional; + +public class RemoveField implements Function { + + public static final String NAME = "remove_field"; + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Optional field = args.evaluated("field", context, String.class); + if (!field.isPresent()) { + throw new IllegalArgumentException(); + } + context.currentMessage().removeField(field.get()); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(ImmutableList.of(ParameterDescriptor.string("field"))) + .build(); + } +} From e48e266d7914c70f40438724259fb17cbf53c263 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 9 Feb 2016 17:30:17 +0100 Subject: [PATCH 066/521] renamed array and map literal constructor functions they were misleading because they didn't actually do the object access but constructed the values --- .../{ArrayExpression.java => ArrayLiteralExpression.java} | 4 ++-- .../{MapExpression.java => MapLiteralExpression.java} | 4 ++-- .../pipelineprocessor/parser/PipelineRuleParser.java | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) rename src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/{ArrayExpression.java => ArrayLiteralExpression.java} (92%) rename src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/{MapExpression.java => MapLiteralExpression.java} (93%) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java similarity index 92% rename from src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java index 2a45c146a766..45aa8e5abde8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java @@ -22,10 +22,10 @@ import java.util.List; import java.util.stream.Collectors; -public class ArrayExpression implements Expression { +public class ArrayLiteralExpression implements Expression { private final List elements; - public ArrayExpression(List elements) { + public ArrayLiteralExpression(List elements) { this.elements = elements; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java similarity index 93% rename from src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapExpression.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java index ca0fe269e976..e224801976ae 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java @@ -24,10 +24,10 @@ import java.util.HashMap; import java.util.Map; -public class MapExpression implements Expression { +public class MapLiteralExpression implements Expression { private final HashMap map; - public MapExpression(HashMap map) { + public MapLiteralExpression(HashMap map) { this.map = map; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 5cf24552f554..c7cd502a14b7 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -36,7 +36,7 @@ import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; -import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; @@ -49,7 +49,7 @@ import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; -import org.graylog.plugins.pipelineprocessor.ast.expressions.MapExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; @@ -471,7 +471,7 @@ public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { @Override public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); - exprs.put(ctx, new ArrayExpression(elements)); + exprs.put(ctx, new ArrayLiteralExpression(elements)); } @Override @@ -482,7 +482,7 @@ public void exitMapLiteralExpr(RuleLangParser.MapLiteralExprContext ctx) { final Expression value = exprs.get(propAssignmentContext.expression()); map.put(key, value); } - exprs.put(ctx, new MapExpression(map)); + exprs.put(ctx, new MapLiteralExpression(map)); } @Override From ee2ed4665b408c2a2a04c2a2e02b3e29bbc39034 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 10 Feb 2016 13:18:20 +0100 Subject: [PATCH 067/521] add indexed access for arrays, lists, iterables and maps array-like values can only be indexed with longs maps can only use strings type checker ensures indexable type and index type --- pom.xml | 12 ++ .../pipelineprocessor/parser/RuleLang.g4 | 2 +- .../expressions/IndexedAccessExpression.java | 83 ++++++++++++ .../functions/messages/RemoveField.java | 16 +++ .../parser/PipelineRuleParser.java | 43 ++++++ .../parser/errors/IncompatibleIndexType.java | 23 ++++ .../parser/errors/NonIndexableType.java | 19 +++ .../IndexedAccessExpressionTest.java | 124 ++++++++++++++++++ .../parser/PipelineRuleParserTest.java | 34 ++++- .../parser/indexedAccess.txt | 6 + .../parser/indexedAccessWrongIndexType.txt | 6 + .../parser/indexedAccessWrongType.txt | 6 + 12 files changed, 371 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java create mode 100644 src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt diff --git a/pom.xml b/pom.xml index c2a9fa8da2a0..e9a303ed3c76 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,12 @@ 2.0.40-beta test + + org.assertj + assertj-core + 3.3.0 + test + org.jooq jool @@ -176,6 +182,12 @@ log4j-over-slf4j test + + org.assertj + assertj-core + 3.3.0 + test + diff --git a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 index 2128410980d0..83c5d3cfc787 100644 --- a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 @@ -77,7 +77,7 @@ ruleDeclaration expression : MessageRef '.' field=expression # MessageRef | fieldSet=expression '.' field=expression # Nested - | fieldSet=expression '[' expression ']' # Array + | array=expression '[' index=expression ']' # IndexedAccess | functionCall # Func | left=expression comparison=('<=' | '>=' | '>' | '<') right=expression # Comparison | left=expression equality=('==' | '!=') right=expression # Equality diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java new file mode 100644 index 000000000000..a0440366b0c3 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java @@ -0,0 +1,83 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.collect.Iterables; +import com.google.common.primitives.Ints; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.lang.reflect.Array; +import java.util.List; +import java.util.Map; + +public class IndexedAccessExpression implements Expression { + private final Expression indexableObject; + private final Expression index; + + public IndexedAccessExpression(Expression indexableObject, Expression index) { + this.indexableObject = indexableObject; + this.index = index; + } + + @Override + public boolean isConstant() { + return indexableObject.isConstant() && index.isConstant(); + } + + @Override + public Object evaluate(EvaluationContext context) { + final Object idxObj = this.index.evaluate(context); + final Object indexable = indexableObject.evaluate(context); + + if (idxObj instanceof Long) { + int idx = Ints.saturatedCast((long) idxObj); + if (indexable.getClass().isArray()) { + return Array.get(indexable, idx); + } else if (indexable instanceof List) { + return ((List) indexable).get(idx); + } else if (indexable instanceof Iterable) { + return Iterables.get((Iterable) indexable, idx); + } + throw new IllegalArgumentException("Object '" + indexable + "' is not an Array, List or Iterable."); + } else if (idxObj instanceof String) { + final String idx = idxObj.toString(); + if (indexable instanceof Map) { + return ((Map) indexable).get(idx); + } + throw new IllegalArgumentException("Object '" + indexable + "' is not a Map."); + } + throw new IllegalArgumentException("Index '" + idxObj + "' is not a Long or String."); + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return indexableObject.toString() + "[" + index.toString() + "]"; + } + + public Expression getIndexableObject() { + return indexableObject; + } + + public Expression getIndex() { + return index; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java index 4a26780e2a44..71ba0a653338 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.messages; import com.google.common.collect.ImmutableList; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index c7cd502a14b7..2648aaaa4b06 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -36,6 +36,7 @@ import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; @@ -63,9 +64,11 @@ import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleType; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleTypes; import org.graylog.plugins.pipelineprocessor.parser.errors.MissingRequiredParam; +import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType; import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; import org.graylog.plugins.pipelineprocessor.parser.errors.SyntaxError; @@ -531,6 +534,16 @@ public void exitFunc(RuleLangParser.FuncContext ctx) { exprs.put(ctx, exprs.get(ctx.functionCall())); parseContext.addInnerNode(ctx); } + + @Override + public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { + final Expression array = exprs.get(ctx.array); + final Expression index = exprs.get(ctx.index); + + final IndexedAccessExpression expr = new IndexedAccessExpression(array, index); + exprs.put(ctx, expr); + log.info("IDXACCESS: ctx {} => {}", ctx, expr); + } } private class RuleTypeAnnotator extends RuleLangBaseListener { @@ -641,6 +654,36 @@ public void exitEveryRule(ParserRuleContext ctx) { } } + @Override + public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { + final IndexedAccessExpression idxExpr = (IndexedAccessExpression) parseContext.expressions().get( + ctx); + + final Class indexableType = idxExpr.getIndexableObject().getType(); + final Class indexType = idxExpr.getIndex().getType(); + + final boolean isMap = Map.class.isAssignableFrom(indexableType); + if (indexableType.isArray() + || List.class.isAssignableFrom(indexableType) + || Iterable.class.isAssignableFrom(indexableType) + || isMap) { + // then check if the index type is compatible, must be long for array-like and string for map-like types + if (isMap) { + if (!String.class.equals(indexType)) { + // add type error + parseContext.addError(new IncompatibleIndexType(ctx, String.class, indexType)); + } + } else { + if (!Long.class.equals(indexType)) { + parseContext.addError(new IncompatibleIndexType(ctx, Long.class, indexType)); + } + } + } else { + // not an indexable type + parseContext.addError(new NonIndexableType(ctx, indexableType)); + } + + } } /** diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java new file mode 100644 index 000000000000..85b993c4a0fd --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java @@ -0,0 +1,23 @@ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleIndexType extends ParseError { + private final Class expected; + private final Class actual; + + public IncompatibleIndexType(RuleLangParser.IndexedAccessContext ctx, + Class expected, + Class actual) { + super("incompatible_index_type", ctx); + this.expected = expected; + this.actual = actual; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + " when indexing" + positionString(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java new file mode 100644 index 000000000000..e13a5f0106f9 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java @@ -0,0 +1,19 @@ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class NonIndexableType extends ParseError { + private final Class indexableType; + + public NonIndexableType(RuleLangParser.IndexedAccessContext ctx, Class indexableType) { + super("non_indexable", ctx); + this.indexableType = indexableType; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Cannot index value of type " + indexableType.getSimpleName() + positionString(); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java new file mode 100644 index 000000000000..274a8527745b --- /dev/null +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java @@ -0,0 +1,124 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +public class IndexedAccessExpressionTest { + + private EvaluationContext context; + + @Before + public void setup() { + context = new EvaluationContext(new Message("test message", "test", DateTime.parse("2010-07-30T16:03:25Z"))); + } + + @Test + public void accessArray() { + int ary[] = new int[] {23}; + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(ary), num(0)); + + final Object evaluate = idxExpr.evaluate(context); + assertThat(evaluate).isOfAnyClassIn(Integer.class); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void accessList() { + final ImmutableList list = ImmutableList.of(23); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(list), num(0)); + + final Object evaluate = idxExpr.evaluate(context); + assertThat(evaluate).isOfAnyClassIn(Integer.class); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void accessIterable() { + final Iterable iterable = () -> new AbstractIterator() { + private boolean done = false; + + @Override + protected Integer computeNext() { + if (done) { + return endOfData(); + } + done = true; + return 23; + } + }; + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(iterable), num(0)); + + final Object evaluate = idxExpr.evaluate(context); + assertThat(evaluate).isOfAnyClassIn(Integer.class); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void accessMap() { + final ImmutableMap map = ImmutableMap.of("string", 23); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(map), string("string")); + + final Object evaluate = idxExpr.evaluate(context); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void invalidObject() { + final IndexedAccessExpression expression = new IndexedAccessExpression(obj(23), num(0)); + + // this should throw an exception + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> expression.evaluate(context)); + } + + private static Expression num(long idx) { + return new LongExpression(idx); + } + + private static Expression string(String string) { + return new StringExpression(string); + } + + private static ConstantObjectExpression obj(Object object) { + return new ConstantObjectExpression(object); + } + + private static class ConstantObjectExpression extends ConstantExpression { + private final Object object; + + protected ConstantObjectExpression(Object object) { + super(object.getClass()); + this.object = object; + } + + @Override + public Object evaluate(EvaluationContext context) { + return object; + } + } +} \ No newline at end of file diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index e600f98bfec0..ac79bf8564df 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -30,11 +30,13 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; -import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType; +import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType; import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; @@ -446,6 +448,34 @@ public void pipelineDeclaration() throws Exception { stage2.ruleReferences().toArray()); } + @Test + public void indexedAccess() { + final Rule rule = parser.parseRule(ruleForTest()); + + evaluateRule(rule, new Message("hallo", "test", DateTime.now())); + assertTrue("condition should be true", actionsTriggered.get()); + } + + @Test + public void indexedAccessWrongType() { + try { + parser.parseRule(ruleForTest()); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(NonIndexableType.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + + @Test + public void indexedAccessWrongIndexType() { + try { + parser.parseRule(ruleForTest()); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(IncompatibleIndexType.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(message); if (rule.when().evaluateBool(context)) { diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt new file mode 100644 index 000000000000..db37fbb31e5e --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt @@ -0,0 +1,6 @@ +rule "indexed array and map access" +when + ["first","second"][0] == "first" and {third: "a value"}["third"] == "a value" +then + trigger_test(); +end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt new file mode 100644 index 000000000000..93e221c3073f --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt @@ -0,0 +1,6 @@ +rule "indexed array and map access" +when + ["first"][true] == "first" +then + trigger_test(); +end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt new file mode 100644 index 000000000000..2ffae9939978 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt @@ -0,0 +1,6 @@ +rule "indexed array and map access" +when + one_arg("not an array")[0] == "first" +then + trigger_test(); +end \ No newline at end of file From b4b82758065ced3f892af84af45c1099530673c2 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 10 Feb 2016 15:50:52 +0100 Subject: [PATCH 068/521] updated licenses --- .../pipeline/PipelineBenchmark.java | 39 ++++++------------- .../parser/errors/IncompatibleIndexType.java | 16 ++++++++ .../parser/errors/NonIndexableType.java | 16 ++++++++ 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index dd5d034680a8..1f322f8abb08 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -1,34 +1,19 @@ -/* - * Copyright (c) 2014, Oracle America, Inc. - * All rights reserved. +/** + * This file is part of Graylog Pipeline Processor. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * * Neither the name of Oracle nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - * THE POSSIBILITY OF SUCH DAMAGE. + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . */ - package org.graylog.benchmarks.pipeline; import com.codahale.metrics.MetricRegistry; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java index 85b993c4a0fd..22a8f0dcd90d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java index e13a5f0106f9..a68bea25a84a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; From b125f3aa119c871ffb70ea1640902aa8cf46699a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 10 Feb 2016 17:48:28 +0100 Subject: [PATCH 069/521] dummy stream assignment component --- src/web/PipelineStreamComponent.jsx | 26 ++++++++++++++++++++++++++ src/web/PipelinesPage.jsx | 21 ++++++++++++++------- 2 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 src/web/PipelineStreamComponent.jsx diff --git a/src/web/PipelineStreamComponent.jsx b/src/web/PipelineStreamComponent.jsx new file mode 100644 index 000000000000..e11306c7e5eb --- /dev/null +++ b/src/web/PipelineStreamComponent.jsx @@ -0,0 +1,26 @@ +import React, {PropTypes} from 'react'; +import {Row, Col, Button} from 'react-bootstrap'; + +const PipelineStreamComponent = React.createClass({ + propTypes: { + pipelines: PropTypes.array.isRequired, + streams: PropTypes.array.isRequired, + assignments: PropTypes.array.isRequired, + }, + render() { + let streamAssignments = [ +
  • Todo
    • + ]; + return ( + + +

    Stream binding

    +
      + {streamAssignments} +
    + +     ); + }, +}); + +export default PipelineStreamComponent; \ No newline at end of file diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx index 81d6e143a66e..c2fc910f702d 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/PipelinesPage.jsx @@ -5,7 +5,7 @@ import { Row, Col } from 'react-bootstrap'; import PageHeader from 'components/common/PageHeader'; import Spinner from 'components/common/Spinner'; -import CurrentUserStore from 'stores/users/CurrentUserStore'; +import PipelineStreamComponent from 'PipelineStreamComponent'; import PipelinesActions from 'PipelinesActions'; import PipelinesStore from 'PipelinesStore'; @@ -17,30 +17,37 @@ import RulesComponent from 'RulesComponent'; const PipelinesPage = React.createClass({ - mixins: [Reflux.connect(CurrentUserStore), Reflux.connect(PipelinesStore), Reflux.connect(RulesStore)], - + mixins: [ + Reflux.connect(PipelinesStore), + Reflux.connect(RulesStore), + ], + contextTypes: { + storeProvider: React.PropTypes.object, + }, getInitialState() { return { pipelines: undefined, rules: undefined, + streams: undefined, + assignments: [], } }, componentDidMount() { PipelinesActions.list(); RulesActions.list(); - }, - loadData() { - PipelinesActions.list(); + var store = this.context.storeProvider.getStore('Streams'); + store.listStreams().then((streams) => this.setState({streams: streams})); }, render() { let content; - if (!this.state.pipelines || !this.state.rules) { + if (!this.state.pipelines || !this.state.rules || !this.state.streams) { content = ; } else { content = [ + , , ]; From df6f6671055107094546f9fb5c5a4903fb668983 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 12 Feb 2016 10:33:11 +0100 Subject: [PATCH 070/521] Bump Graylog server dependency to 2.0.0-alpha.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bd164b3f9cc6..7c60c1b85596 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-alpha.2-SNAPSHOT + 2.0.0-alpha.3-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From 8c96efc51bb270721302a412125863e4f63b0452 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 12 Feb 2016 14:07:41 +0100 Subject: [PATCH 071/521] Update graylog-web-manifests to 2.0.0-alpha.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 61451694cf30..8858ea9e44c7 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "devDependencies": { "babel-core": "^5.8.25", "babel-loader": "^5.3.2", - "graylog-web-manifests": "2.0.0-SNAPSHOT-1", + "graylog-web-manifests": "^2.0.0-alpha.3", "graylog-web-plugin": "~0.0.18", "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", From bcdce263606659dbc185a5b0ec3cdf89c3adffdc Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 12 Feb 2016 14:36:28 +0100 Subject: [PATCH 072/521] Overhaul function/param descriptors and precompute constant function arguments To avoid reevaluating function arguments which never change for each message and are potentially expensive to calculate (either because their expression tree is deep or the function needs to construct other objects) the Function interface now contains a callback to provide precomputed values for each constant argument. This callback is only invoked for provably constant expressions, so it is safe to implement for any function. The recommended way is to inherit from AbstractFunction which implements the straightforward precompute function by simply memoizing the evaluated value. For special cases, like the RegexMatch function, which precomputes a Pattern.compile from a constant parameter descriptors take a java.util.function.Function now, which gets invoked with the computed value. The default is the identity function. Parameter descriptors are now responsible for evaluating argument expressions, and are fully generic. --- .../pipelineprocessor/EvaluationContext.java | 21 +++ .../PipelineProcessorModule.java | 4 + .../expressions/ArrayLiteralExpression.java | 2 +- .../ast/expressions/EqualityExpression.java | 8 +- .../expressions/FieldAccessExpression.java | 6 +- .../ast/expressions/FunctionExpression.java | 9 +- .../ast/expressions/MapLiteralExpression.java | 2 +- .../ast/functions/AbstractFunction.java | 17 +++ .../ast/functions/Function.java | 35 ++++- .../ast/functions/FunctionArgs.java | 52 +++++++- .../ast/functions/FunctionDescriptor.java | 21 ++- .../ast/functions/ParameterDescriptor.java | 120 +++++++++++++----- .../functions/BooleanCoercion.java | 9 +- .../functions/DoubleCoercion.java | 15 ++- .../functions/FromInput.java | 16 ++- .../functions/LongCoercion.java | 10 +- .../functions/RegexMatch.java | 111 ++++++++++++++++ .../functions/StringCoercion.java | 12 +- .../functions/messages/CreateMessage.java | 9 +- .../functions/messages/DropMessage.java | 4 +- .../functions/messages/HasField.java | 12 +- .../functions/messages/RemoveField.java | 15 +-- .../functions/messages/RouteToStream.java | 10 +- .../functions/messages/SetField.java | 19 ++- .../parser/PipelineRuleParser.java | 13 +- .../parser/PipelineRuleParserTest.java | 59 +++++---- .../pipelineprocessor/parser/regexMatch.txt | 8 ++ 27 files changed, 479 insertions(+), 140 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/parser/regexMatch.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java index 65d12470a108..03fcecff9b02 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java @@ -21,16 +21,33 @@ import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; import org.graylog2.plugin.Messages; +import org.joda.time.DateTime; import java.util.List; import java.util.Map; public class EvaluationContext { + private static final EvaluationContext EMPTY_CONTEXT = new EvaluationContext() { + @Override + public void addCreatedMessage(Message newMessage) { + // cannot add messages to empty context + } + + @Override + public void define(String identifier, Class type, Object value) { + // cannot define any variables in empty context + } + }; + private final Message message; private Map ruleVars; private List createdMessages = Lists.newArrayList(); + private EvaluationContext() { + this(new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"))); // first Graylog release + } + public EvaluationContext(Message message) { this.message = message; ruleVars = Maps.newHashMap(); @@ -60,6 +77,10 @@ public void clearCreatedMessages() { createdMessages.clear(); } + public static EvaluationContext emptyContext() { + return EMPTY_CONTEXT; + } + public class TypedValue { private final Class type; private final Object value; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index d6320659bab1..bc1fb984c4c0 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -24,6 +24,7 @@ import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; import org.graylog.plugins.pipelineprocessor.functions.FromInput; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; +import org.graylog.plugins.pipelineprocessor.functions.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; @@ -73,6 +74,9 @@ protected void configure() { // input related functions addMessageProcessorFunction(FromInput.NAME, FromInput.class); + + // generic functions + addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java index 45aa8e5abde8..bb71eb8bfc59 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java @@ -31,7 +31,7 @@ public ArrayLiteralExpression(List elements) { @Override public boolean isConstant() { - return false; + return elements.stream().allMatch(Expression::isConstant); } @Override diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java index 7f147fffde17..bf61c510428d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java @@ -38,7 +38,13 @@ public Class getType() { @Override public boolean evaluateBool(EvaluationContext context) { - final boolean equals = left.evaluate(context).equals(right.evaluate(context)); + final Object left = this.left.evaluate(context); + final Object right = this.right.evaluate(context); + if (left == null) { + // TODO log error + return false; + } + final boolean equals = left.equals(right); if (checkEquality) { return equals; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java index 4702a9781443..eb3420ead146 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -44,7 +44,11 @@ public Object evaluate(EvaluationContext context) { final Object bean = this.object.evaluate(context); final String fieldName = field.evaluate(context).toString(); try { - final Object property = PropertyUtils.getProperty(bean, fieldName); + Object property = PropertyUtils.getProperty(bean, fieldName); + if (property == null) { + // in case the bean is a Map, try again with a simple property, it might be masked by the Map + property = PropertyUtils.getSimpleProperty(bean, fieldName); + } log.debug("[field access] property {} of bean {}: {}", fieldName, bean.getClass().getTypeName(), property); return property; } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java index d261623b123a..6956e39773c8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java @@ -27,10 +27,13 @@ public class FunctionExpression implements Expression { private final Function function; private final FunctionDescriptor descriptor; - public FunctionExpression(Function function, FunctionArgs args) { + public FunctionExpression(FunctionArgs args) { this.args = args; - this.function = function; - this.descriptor = function.descriptor(); + this.function = args.getFunction(); + this.descriptor = this.function.descriptor(); + + // precomputes all constant arguments to avoid dynamically recomputing trees on every invocation + this.function.preprocessArgs(args); } public Function getFunction() { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java index e224801976ae..d192b4e1a1f2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java @@ -33,7 +33,7 @@ public MapLiteralExpression(HashMap map) { @Override public boolean isConstant() { - return false; + return map.values().stream().allMatch(Expression::isConstant); } @Override diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java new file mode 100644 index 000000000000..47fb994b1552 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java @@ -0,0 +1,17 @@ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +/** + * Helper Function implementation which evaluates and memoizes all constant FunctionArgs. + * + * @param the return type + */ +public abstract class AbstractFunction implements Function { + + @Override + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { + return arg.evaluate(EvaluationContext.emptyContext()); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index 4ef0b79a22cc..4296a70e9e92 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -18,10 +18,17 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; public interface Function { - Function ERROR_FUNCTION = new Function() { + Logger log = LoggerFactory.getLogger(Function.class); + + Function ERROR_FUNCTION = new AbstractFunction() { @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { return null; @@ -37,6 +44,32 @@ public FunctionDescriptor descriptor() { } }; + default void preprocessArgs(FunctionArgs args) { + for (Map.Entry e : args.getConstantArgs().entrySet()) { + try { + final Object value = preComputeConstantArgument(args, e.getKey(), e.getValue()); + if (value != null) { + args.setPreComputedValue(e.getKey(), value); + } + } catch (Exception exception) { + log.warn("Unable to precompute argument value for " + e.getKey(), exception); + } + } + + } + + /** + * Implementations should provide a non-null value for each argument they wish to pre-compute. + *
    + * Examples include compile a Pattern from a regex string, which will never change during the lifetime of the function. + * If any part of the expression tree depends on external values this method will not be called, e.g. if the regex depends on a message field. + * @param args the function args for this functions, usually you don't need this + * @param name the name of the argument to potentially precompute + * @param arg the expression tree for the argument + * @return the precomputed value for the argument or null if the value should be dynamically calculated for each invocation + */ + Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg); + T evaluate(FunctionArgs args, EvaluationContext context); FunctionDescriptor descriptor(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index 784adc443c8f..054d8e745ed7 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -16,6 +16,7 @@ */ package org.graylog.plugins.pipelineprocessor.ast.functions; +import com.google.common.collect.Maps; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; @@ -23,7 +24,9 @@ import javax.annotation.Nullable; import java.util.Collections; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import static com.google.common.base.MoreObjects.firstNonNull; @@ -32,7 +35,13 @@ public class FunctionArgs { @Nonnull private final Map args; - public FunctionArgs(Map args) { + private final Map constantValues = Maps.newHashMap(); + private final Function function; + private final FunctionDescriptor descriptor; + + public FunctionArgs(Function func, Map args) { + function = func; + descriptor = function.descriptor(); this.args = firstNonNull(args, Collections.emptyMap()); } @@ -42,13 +51,34 @@ public Map getArgs() { } @Nonnull + public Map getConstantArgs() { + return args.entrySet().stream() + .filter(e -> e.getValue().isConstant()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + @Nonnull + @Deprecated public Optional evaluated(String name, EvaluationContext context, Class argumentType) { + return Optional.ofNullable(required(name, context, argumentType)); + } + + @Nullable + @Deprecated + public T required(String name, EvaluationContext context, Class argumentType) { + final ParameterDescriptor param = descriptor.param(name); + + final Object precomputedValue = constantValues.get(name); + if (precomputedValue != null) { + return (T)param.transformedType().cast(precomputedValue); + } final Expression valueExpr = expression(name); if (valueExpr == null) { - return Optional.empty(); + return null; } final Object value = valueExpr.evaluate(context); - return Optional.ofNullable(argumentType.cast(value)); + final Object transformed = param.transform().apply(value); + return (T)param.transformedType().cast(transformed); } public boolean isPresent(String key) { @@ -60,4 +90,20 @@ public Expression expression(String key) { return args.get(key); } + public Object getPreComputedValue(String name) { + return constantValues.get(name); + } + + public void setPreComputedValue(@Nonnull String name, @Nonnull Object value) { + Objects.requireNonNull(value); + constantValues.put(name, value); + } + + public Function getFunction() { + return function; + } + + public ParameterDescriptor param(String name) { + return descriptor.param(name); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java index 419ae7196e1b..b65a73e80c34 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java @@ -18,6 +18,8 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; @AutoValue public abstract class FunctionDescriptor { @@ -26,10 +28,16 @@ public abstract class FunctionDescriptor { public abstract boolean pure(); - public abstract Class returnType(); + public abstract Class returnType(); public abstract ImmutableList params(); + public abstract ImmutableMap paramMap(); + + public ParameterDescriptor param(String name) { + return paramMap().get(name); + } + public static Builder builder() { //noinspection unchecked return new AutoValue_FunctionDescriptor.Builder().pure(false); @@ -37,11 +45,18 @@ public static Builder builder() { @AutoValue.Builder public static abstract class Builder { - public abstract FunctionDescriptor build(); + public abstract FunctionDescriptor autoBuild(); + + public FunctionDescriptor build() { + return paramMap(Maps.uniqueIndex(params(), ParameterDescriptor::name)) + .autoBuild(); + } public abstract Builder name(String name); public abstract Builder pure(boolean pure); - public abstract Builder returnType(Class type); + public abstract Builder returnType(Class type); public abstract Builder params(ImmutableList params); + public abstract Builder paramMap(ImmutableMap map); + public abstract ImmutableList params(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index e8c766c31aeb..d387246d1f67 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -17,67 +17,119 @@ package org.graylog.plugins.pipelineprocessor.ast.functions; import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +import javax.annotation.Nullable; +import java.util.Optional; @AutoValue -public abstract class ParameterDescriptor { +public abstract class ParameterDescriptor { + + public abstract Class type(); - public abstract Class type(); + public abstract Class transformedType(); public abstract String name(); public abstract boolean optional(); - public static Builder param() { - return new AutoValue_ParameterDescriptor.Builder().optional(false); + public abstract java.util.function.Function transform(); + + public static Builder param() { + return new AutoValue_ParameterDescriptor.Builder().optional(false); } - public static ParameterDescriptor string(String name) { - return param().string(name).build(); + public static Builder string(String name) { + return string(name, String.class); } - public static ParameterDescriptor object(String name) { - return param().object(name).build(); + public static Builder string(String name, Class transformedClass) { + return ParameterDescriptor.param().type(String.class).transformedType(transformedClass).name(name); } - public static ParameterDescriptor integer(String name) { - return param().integer(name).build(); + public static Builder object(String name) { + return object(name, Object.class); } - public static ParameterDescriptor floating(String name) { - return param().floating(name).build(); + public static Builder object(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Object.class).transformedType(transformedClass).name(name); } - public static ParameterDescriptor bool(String name) { - return param().bool(name).build(); + public static Builder integer(String name) { + return integer(name, Long.class); } - @AutoValue.Builder - public static abstract class Builder { - public abstract Builder type(Class type); - public abstract Builder name(String name); - public abstract Builder optional(boolean optional); - public abstract ParameterDescriptor build(); + public static Builder integer(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Long.class).transformedType(transformedClass).name(name); + } + public static Builder floating(String name) { + return floating(name, Double.class); + } + public static Builder floating(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Double.class).transformedType(transformedClass).name(name); + } - public Builder string(String name) { - return type(String.class).name(name); - } - public Builder object(String name) { - return type(Object.class).name(name); - } - public Builder floating(String name) { - return type(Double.class).name(name); - } - public Builder integer(String name) { - return type(Long.class).name(name); + public static Builder bool(String name) { + return bool(name, Boolean.class); + } + + public static Builder bool(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Boolean.class).transformedType(transformedClass).name(name); + } + + public static Builder type(String name, Class typeClass) { + return type(name, typeClass, typeClass); + } + + public static Builder type(String name, Class typeClass, Class transformedClass) { + return ParameterDescriptor.param().type(typeClass).transformedType(transformedClass).name(name); + } + + public Optional eval(FunctionArgs args, EvaluationContext context, Class type) { + return Optional.ofNullable(evalRequired(args, context, type)); + } + + public X evalRequired(FunctionArgs args, EvaluationContext context, Class type) { + final Object precomputedValue = args.getPreComputedValue(name()); + if (precomputedValue != null) { + return type.cast(transformedType().cast(precomputedValue)); } - public Builder bool(String name) { - return type(Boolean.class).name(name); + final Expression valueExpr = args.expression(name()); + if (valueExpr == null) { + return null; } + final Object value = valueExpr.evaluate(context); + return type.cast(transform().apply(type().cast(value))); + } + + @AutoValue.Builder + public static abstract class Builder { + public abstract Builder type(Class type); + public abstract Builder transformedType(Class type); + public abstract Builder name(String name); + public abstract Builder optional(boolean optional); - public Builder optional() { + public Builder optional() { return optional(true); } + abstract ParameterDescriptor autoBuild(); + public ParameterDescriptor build() { + try { + transform(); + } catch (IllegalStateException ignored) { + // unfortunately there's no "hasTransform" method in autovalue + //noinspection unchecked + transform((java.util.function.Function) java.util.function.Function.identity()); + } + return autoBuild(); + } + + public abstract Builder transform(@Nullable java.util.function.Function transform); + @Nullable + public abstract java.util.function.Function transform(); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java index 49f0f6939a97..309bf3a6f57a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java @@ -17,22 +17,21 @@ package org.graylog.plugins.pipelineprocessor.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; -public class BooleanCoercion implements Function { +public class BooleanCoercion extends AbstractFunction { public static final String NAME = "bool"; private static final String VALUE = "value"; @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse("false"); - return Boolean.parseBoolean(evaluated.toString()); + return args.param(VALUE).evalRequired(args, context, Boolean.class); } @Override @@ -40,7 +39,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(of(object(VALUE))) + .params(of(object(VALUE, Boolean.class).transform(o -> Boolean.parseBoolean(String.valueOf(o))).build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java index 597ca4f25139..8b1e5a0d00d6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java @@ -18,7 +18,7 @@ import com.google.common.primitives.Doubles; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; @@ -27,7 +27,7 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; -public class DoubleCoercion implements Function { +public class DoubleCoercion extends AbstractFunction { public static final String NAME = "double"; @@ -36,8 +36,13 @@ public class DoubleCoercion implements Function { @Override public Double evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); - return (Double) firstNonNull(Doubles.tryParse(evaluated.toString()), args.evaluated(DEFAULT, context, Double.class).orElse(0d)); + final Object evaluated = args.required(VALUE, context, Object.class); + final Double defaultValue = args.evaluated(DEFAULT, context, Double.class).orElse(0d); + if (evaluated == null) { + return defaultValue; + } + return firstNonNull(Doubles.tryParse(evaluated.toString()), + defaultValue); } @Override @@ -46,7 +51,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Double.class) .params(of( - object(VALUE), + object(VALUE).build(), param().optional().name(DEFAULT).type(Double.class).build() )) .build(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java index bc53e86943a9..62d9118b7382 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -17,7 +17,7 @@ package org.graylog.plugins.pipelineprocessor.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog2.plugin.IOState; @@ -27,11 +27,13 @@ import javax.inject.Inject; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; -public class FromInput implements Function { +public class FromInput extends AbstractFunction { public static final String NAME = "from_input"; + public static final String ID_ARG = "id"; + public static final String NAME_ARG = "name"; private final InputRegistry inputRegistry; @@ -42,11 +44,11 @@ public FromInput(InputRegistry inputRegistry) { @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - String id = args.evaluated("id", context, String.class).orElse(""); + String id = args.evaluated(ID_ARG, context, String.class).orElse(""); MessageInput input = null; if ("".equals(id)) { - final String name = args.evaluated("name", context, String.class).orElse(""); + final String name = args.evaluated(NAME_ARG, context, String.class).orElse(""); for (IOState messageInputIOState : inputRegistry.getInputStates()) { final MessageInput messageInput = messageInputIOState.getStoppable(); if (messageInput.getTitle().equalsIgnoreCase(name)) { @@ -74,8 +76,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of( - param().optional().string("id").build(), - param().optional().string("name").build())) + string(ID_ARG).optional().build(), + string(NAME_ARG).optional().build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java index 4f5def52e3fc..6fed69a770e7 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java @@ -17,17 +17,17 @@ package org.graylog.plugins.pipelineprocessor.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; import static com.google.common.primitives.Longs.tryParse; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.integer; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; -public class LongCoercion implements Function { +public class LongCoercion extends AbstractFunction { public static final String NAME = "long"; @@ -48,8 +48,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Long.class) .params(of( - object(VALUE), - param().optional().integer(DEFAULT).build() + object(VALUE).build(), + integer(DEFAULT).optional().build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java new file mode 100644 index 000000000000..b63f28bb3c66 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java @@ -0,0 +1,111 @@ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.google.common.collect.ForwardingMap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.google.common.collect.ImmutableList.of; + +public class RegexMatch extends AbstractFunction { + + public static final String PATTERN_ARG = "pattern"; + public static final String VALUE_ARG = "value"; + public static final String GROUP_NAMES_ARG = "group_names"; + public static final String NAME = "regex"; + + @Override + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { + final String stringValue = (String) super.preComputeConstantArgument(args, name, arg); + switch (name) { + case PATTERN_ARG: + return Pattern.compile(stringValue); + } + return stringValue; + } + + @Override + public RegexMatchResult evaluate(FunctionArgs args, EvaluationContext context) { + final Pattern regex = args.required(PATTERN_ARG, context, Pattern.class); + final String value = args.required(VALUE_ARG, context, String.class); + if (regex == null || value == null) { + throw new IllegalArgumentException(); + } + //noinspection unchecked + final List groupNames = + (List) args.evaluated(GROUP_NAMES_ARG, context, List.class) + .orElse(Collections.emptyList()); + + final Matcher matcher = regex.matcher(value); + final boolean matches = matcher.matches(); + + return new RegexMatchResult(matches, matcher.toMatchResult(), groupNames); + + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .pure(true) + .returnType(RegexMatchResult.class) + .params(of( + ParameterDescriptor.string(PATTERN_ARG, Pattern.class).transform(Pattern::compile).build(), + ParameterDescriptor.string(VALUE_ARG).build(), + ParameterDescriptor.type(GROUP_NAMES_ARG, List.class).optional().build() + )) + .build(); + } + + /** + * The bean returned into the rule engine. It implements Map so rules can access it directly. + *
    + * At the same time there's an additional matches bean property to quickly check whether the regex has matched at all. + */ + public static class RegexMatchResult extends ForwardingMap { + private final boolean matches; + private final ImmutableMap groups; + + public RegexMatchResult(boolean matches, MatchResult matchResult, List groupNames) { + this.matches = matches; + ImmutableMap.Builder builder = ImmutableMap.builder(); + + if (matches) { + // arggggh! not 0 based. + final int groupCount = matchResult.groupCount(); + for (int i = 1; i <= groupCount; i++) { + final String groupValue = matchResult.group(i); + // try to get a group name, if that fails use a 0-based index as the name + final String groupName = Iterables.get(groupNames, i - 1, null); + builder.put(groupName != null ? groupName : String.valueOf(i - 1), groupValue); + } + } + groups = builder.build(); + } + + public boolean isMatches() { + return matches; + } + + public Map getGroups() { + return groups; + } + + @Override + protected Map delegate() { + return getGroups(); + } + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java index 75b8e5801c9d..cab09c9e2c97 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java @@ -17,15 +17,15 @@ package org.graylog.plugins.pipelineprocessor.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; -public class StringCoercion implements Function { +public class StringCoercion extends AbstractFunction { public static final String NAME = "string"; @@ -34,7 +34,7 @@ public class StringCoercion implements Function { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); + final Object evaluated = args.required(VALUE, context, Object.class); if (evaluated instanceof String) { return (String) evaluated; } else { @@ -48,8 +48,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of( - object(VALUE), - param().optional().string(DEFAULT).build() + object(VALUE).build(), + string(DEFAULT).optional().build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java index a70741daadbf..493d071e1088 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java @@ -17,7 +17,7 @@ package org.graylog.plugins.pipelineprocessor.functions.messages; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog2.plugin.Message; @@ -28,8 +28,9 @@ import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; -public class CreateMessage implements Function { +public class CreateMessage extends AbstractFunction { public static final String NAME = "create_message"; @@ -61,8 +62,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Message.class) .params(of( - param().string(MESSAGE_ARG).optional().build(), - param().string(SOURCE_ARG).optional().build(), + string(MESSAGE_ARG).optional().build(), + string(SOURCE_ARG).optional().build(), param().name(TIMESTAMP_ARG).type(DateTime.class).optional().build() )) .build(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java index 28af07efa4a3..bfc51005d448 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog2.plugin.Message; @@ -27,7 +27,7 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; -public class DropMessage implements Function { +public class DropMessage extends AbstractFunction { public static final String NAME = "drop_message"; public static final String MESSAGE_ARG = "message"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java index c6a3828cb810..797ae69d598a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -18,21 +18,19 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; -import java.util.Optional; - -public class HasField implements Function { +public class HasField extends AbstractFunction { public static final String NAME = "has_field"; @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final Optional field = args.evaluated("field", context, String.class); - return context.currentMessage().hasField(field.orElse(null)); + final String field = args.required("field", context, String.class); + return context.currentMessage().hasField(field); } @Override @@ -40,7 +38,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(ImmutableList.of(ParameterDescriptor.string("field"))) + .params(ImmutableList.of(ParameterDescriptor.string("field").build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java index 71ba0a653338..e4f377982808 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -18,24 +18,19 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; -import java.util.Optional; - -public class RemoveField implements Function { +public class RemoveField extends AbstractFunction { public static final String NAME = "remove_field"; @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final Optional field = args.evaluated("field", context, String.class); - if (!field.isPresent()) { - throw new IllegalArgumentException(); - } - context.currentMessage().removeField(field.get()); + final String field = args.required("field", context, String.class); + context.currentMessage().removeField(field); return null; } @@ -44,7 +39,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(ImmutableList.of(ParameterDescriptor.string("field"))) + .params(ImmutableList.of(ParameterDescriptor.string("field").build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java index 640f165e108b..862cb288838b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -20,7 +20,7 @@ import com.google.common.collect.Maps; import com.google.inject.Inject; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog2.database.NotFoundException; @@ -28,9 +28,9 @@ import org.graylog2.streams.StreamService; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; -public class RouteToStream implements Function { +public class RouteToStream extends AbstractFunction { public static final String NAME = "route_to_stream"; private static final String ID_ARG = "id"; @@ -81,8 +81,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(of( - param().optional().string(NAME_ARG).build(), - param().optional().string(ID_ARG).build())) + string(NAME_ARG).optional().build(), + string(ID_ARG).optional().build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index ff40e0e964e4..d3047071747d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -16,18 +16,17 @@ */ package org.graylog.plugins.pipelineprocessor.functions.messages; +import com.google.common.base.Strings; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; -import java.util.Optional; - import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; -public class SetField implements Function { +public class SetField extends AbstractFunction { public static final String NAME = "set_field"; public static final String FIELD = "field"; @@ -35,11 +34,11 @@ public class SetField implements Function { @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final Optional field = args.evaluated(FIELD, context, Object.class); - final Optional value = args.evaluated(VALUE, context, Object.class); + final String field = args.required(FIELD, context, String.class); + final Object value = args.required(VALUE, context, Object.class); - if (field.isPresent() && !field.get().toString().isEmpty()) { - context.currentMessage().addField(field.get().toString(), value.get()); + if (!Strings.isNullOrEmpty(field)) { + context.currentMessage().addField(field, value); } return null; } @@ -49,8 +48,8 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(of(string(FIELD), - object(VALUE))) + .params(of(string(FIELD).build(), + object(VALUE).build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 2648aaaa4b06..74e32b5cb4dd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -341,8 +341,9 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { } } - final FunctionExpression expr = new FunctionExpression(functionRegistry.resolveOrError(name), - new FunctionArgs(argsMap)); + final FunctionExpression expr = new FunctionExpression( + new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) + ); log.info("FUNC: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -519,12 +520,16 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { parseContext.addError(new UndeclaredVariable(ctx)); } final Expression expr; - if (idIsFieldAccess) { + String type; + // if the identifier is also a declared variable name prefer the variable + if (idIsFieldAccess && !definedVars.contains(identifierName)) { expr = new FieldRefExpression(identifierName); + type = "FIELDREF"; } else { expr = new VarRefExpression(identifierName); + type = "VARREF"; } - log.info("VAR: ctx {} => {}", ctx, expr); + log.info("{}: ctx {} => {}", type, ctx, expr); exprs.put(ctx, expr); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index ac79bf8564df..27b6cf36a72b 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -25,12 +25,14 @@ import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; +import org.graylog.plugins.pipelineprocessor.functions.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; @@ -64,7 +66,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -84,7 +85,7 @@ public class PipelineRuleParserTest { @BeforeClass public static void registerFunctions() { final Map> functions = Maps.newHashMap(); - functions.put("nein", new Function() { + functions.put("nein", new AbstractFunction() { @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { return false; @@ -99,7 +100,7 @@ public FunctionDescriptor descriptor() { .build(); } }); - functions.put("doch", new Function() { + functions.put("doch", new AbstractFunction() { @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { return true; @@ -114,7 +115,7 @@ public FunctionDescriptor descriptor() { .build(); } }); - functions.put("double_valued_func", new Function() { + functions.put("double_valued_func", new AbstractFunction() { @Override public Double evaluate(FunctionArgs args, EvaluationContext context) { return 0d; @@ -129,7 +130,7 @@ public FunctionDescriptor descriptor() { .build(); } }); - functions.put("one_arg", new Function() { + functions.put("one_arg", new AbstractFunction() { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { return args.evaluated("one", context, String.class).orElse(""); @@ -140,11 +141,11 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("one_arg") .returnType(String.class) - .params(of(ParameterDescriptor.string("one"))) + .params(of(ParameterDescriptor.string("one").build())) .build(); } }); - functions.put("concat", new Function() { + functions.put("concat", new AbstractFunction() { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { final Object one = args.evaluated("one", context, Object.class).orElse(""); @@ -159,14 +160,14 @@ public FunctionDescriptor descriptor() { .name("concat") .returnType(String.class) .params(of( - ParameterDescriptor.string("one"), - ParameterDescriptor.object("two"), - ParameterDescriptor.object("three") + ParameterDescriptor.string("one").build(), + ParameterDescriptor.object("two").build(), + ParameterDescriptor.object("three").build() )) .build(); } }); - functions.put("trigger_test", new Function() { + functions.put("trigger_test", new AbstractFunction() { @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { actionsTriggered.set(true); @@ -182,7 +183,7 @@ public FunctionDescriptor descriptor() { .build(); } }); - functions.put("optional", new Function() { + functions.put("optional", new AbstractFunction() { @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { return true; @@ -194,15 +195,15 @@ public FunctionDescriptor descriptor() { .name("optional") .returnType(Boolean.class) .params(of( - ParameterDescriptor.bool("a"), - ParameterDescriptor.string("b"), - param().floating("c").optional().build(), - ParameterDescriptor.integer("d") + ParameterDescriptor.bool("a").build(), + ParameterDescriptor.string("b").build(), + ParameterDescriptor.floating("c").optional().build(), + ParameterDescriptor.integer("d").build() )) .build(); } }); - functions.put("customObject", new Function() { + functions.put("customObject", new AbstractFunction() { @Override public CustomObject evaluate(FunctionArgs args, EvaluationContext context) { return new CustomObject(args.evaluated("default", context, String.class).orElse("")); @@ -213,11 +214,11 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("customObject") .returnType(CustomObject.class) - .params(of(ParameterDescriptor.string("default"))) + .params(of(ParameterDescriptor.string("default").build())) .build(); } }); - functions.put("keys", new Function() { + functions.put("keys", new AbstractFunction() { @Override public List evaluate(FunctionArgs args, EvaluationContext context) { final Optional map = args.evaluated("map", context, Map.class); @@ -229,11 +230,11 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("keys") .returnType(List.class) - .params(of(param().name("map").type(Map.class).build())) + .params(of(ParameterDescriptor.type("map", Map.class).build())) .build(); } }); - functions.put("sort", new Function() { + functions.put("sort", new AbstractFunction() { @Override public Collection evaluate(FunctionArgs args, EvaluationContext context) { final Collection collection = args.evaluated("collection", @@ -247,7 +248,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("sort") .returnType(Collection.class) - .params(of(param().name("collection").type(Collection.class).build())) + .params(of(ParameterDescriptor.type("collection", Collection.class).build())) .build(); } }); @@ -255,6 +256,7 @@ public FunctionDescriptor descriptor() { functions.put(StringCoercion.NAME, new StringCoercion()); functions.put(SetField.NAME, new SetField()); functions.put(HasField.NAME, new HasField()); + functions.put(RegexMatch.NAME, new RegexMatch()); functionRegistry = new FunctionRegistry(functions); } @@ -476,6 +478,19 @@ public void indexedAccessWrongIndexType() { } } + @Test + public void regexMatch() { + try { + final Rule rule = parser.parseRule(ruleForTest()); + final Message message = evaluateRule(rule); + assertNotNull(message); + assertTrue(message.hasField("matched_regex")); + assertTrue(message.hasField("group_1")); + } catch (ParseException e) { + fail("Should parse"); + } + } + private Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(message); if (rule.when().evaluateBool(context)) { diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/regexMatch.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/regexMatch.txt new file mode 100644 index 000000000000..7a0d6c22745d --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/regexMatch.txt @@ -0,0 +1,8 @@ +rule "regexMatch" +when + regex(".*(cde).*", "abcdefg").matches == true +then + let result = regex(".*(cde).*", "abcdefg"); + set_field("group_1", result["0"]); + set_field("matched_regex", result.matches); +end \ No newline at end of file From 57407ed1813eb7e63ea08acee79d7a548767d63a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 12 Feb 2016 14:54:49 +0100 Subject: [PATCH 073/521] remove deprecated functionargs eval methods --- .../ast/functions/FunctionArgs.java | 26 ------------------- .../functions/DoubleCoercion.java | 4 +-- .../functions/FromInput.java | 4 +-- .../functions/LongCoercion.java | 4 +-- .../functions/RegexMatch.java | 6 ++--- .../functions/StringCoercion.java | 4 +-- .../functions/messages/CreateMessage.java | 6 ++--- .../functions/messages/DropMessage.java | 2 +- .../functions/messages/HasField.java | 5 ++-- .../functions/messages/RemoveField.java | 5 ++-- .../functions/messages/RouteToStream.java | 4 +-- .../functions/messages/SetField.java | 4 +-- .../parser/PipelineRuleParserTest.java | 14 +++++----- 13 files changed, 32 insertions(+), 56 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index 054d8e745ed7..b157d7cca9f3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -17,7 +17,6 @@ package org.graylog.plugins.pipelineprocessor.ast.functions; import com.google.common.collect.Maps; -import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import javax.annotation.Nonnull; @@ -25,7 +24,6 @@ import java.util.Collections; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; import static com.google.common.base.MoreObjects.firstNonNull; @@ -57,30 +55,6 @@ public Map getConstantArgs() { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - @Nonnull - @Deprecated - public Optional evaluated(String name, EvaluationContext context, Class argumentType) { - return Optional.ofNullable(required(name, context, argumentType)); - } - - @Nullable - @Deprecated - public T required(String name, EvaluationContext context, Class argumentType) { - final ParameterDescriptor param = descriptor.param(name); - - final Object precomputedValue = constantValues.get(name); - if (precomputedValue != null) { - return (T)param.transformedType().cast(precomputedValue); - } - final Expression valueExpr = expression(name); - if (valueExpr == null) { - return null; - } - final Object value = valueExpr.evaluate(context); - final Object transformed = param.transform().apply(value); - return (T)param.transformedType().cast(transformed); - } - public boolean isPresent(String key) { return args.containsKey(key); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java index 8b1e5a0d00d6..70f39bda466f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java @@ -36,8 +36,8 @@ public class DoubleCoercion extends AbstractFunction { @Override public Double evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.required(VALUE, context, Object.class); - final Double defaultValue = args.evaluated(DEFAULT, context, Double.class).orElse(0d); + final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); + final Double defaultValue = args.param(DEFAULT).eval(args, context, Double.class).orElse(0d); if (evaluated == null) { return defaultValue; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java index 62d9118b7382..4a4fbb7bef05 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -44,11 +44,11 @@ public FromInput(InputRegistry inputRegistry) { @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - String id = args.evaluated(ID_ARG, context, String.class).orElse(""); + String id = args.param(ID_ARG).eval(args, context, String.class).orElse(""); MessageInput input = null; if ("".equals(id)) { - final String name = args.evaluated(NAME_ARG, context, String.class).orElse(""); + final String name = args.param(NAME_ARG).eval(args, context, String.class).orElse(""); for (IOState messageInputIOState : inputRegistry.getInputStates()) { final MessageInput messageInput = messageInputIOState.getStoppable(); if (messageInput.getTitle().equalsIgnoreCase(name)) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java index 6fed69a770e7..55899c82f9ae 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java @@ -36,8 +36,8 @@ public class LongCoercion extends AbstractFunction { @Override public Long evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.evaluated(VALUE, context, Object.class).orElse(new Object()); - final Long defaultValue = args.evaluated(DEFAULT, context, Long.class).orElse(0L); + final Object evaluated = args.param(VALUE).eval(args, context, Object.class).orElse(new Object()); + final Long defaultValue = args.param(DEFAULT).eval(args, context, Long.class).orElse(0L); return firstNonNull(tryParse(evaluated.toString()), defaultValue); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java index b63f28bb3c66..34ff4b669a1f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java @@ -38,14 +38,14 @@ public Object preComputeConstantArgument(FunctionArgs args, String name, Express @Override public RegexMatchResult evaluate(FunctionArgs args, EvaluationContext context) { - final Pattern regex = args.required(PATTERN_ARG, context, Pattern.class); - final String value = args.required(VALUE_ARG, context, String.class); + final Pattern regex = args.param(PATTERN_ARG).evalRequired(args, context, Pattern.class); + final String value = args.param(VALUE_ARG).evalRequired(args, context, String.class); if (regex == null || value == null) { throw new IllegalArgumentException(); } //noinspection unchecked final List groupNames = - (List) args.evaluated(GROUP_NAMES_ARG, context, List.class) + (List) args.param(GROUP_NAMES_ARG).eval(args, context, List.class) .orElse(Collections.emptyList()); final Matcher matcher = regex.matcher(value); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java index cab09c9e2c97..9baa228eee43 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java @@ -34,11 +34,11 @@ public class StringCoercion extends AbstractFunction { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.required(VALUE, context, Object.class); + final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); if (evaluated instanceof String) { return (String) evaluated; } else { - return args.evaluated(DEFAULT, context, String.class).orElse(""); + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java index 493d071e1088..553215b8a6c2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java @@ -40,13 +40,13 @@ public class CreateMessage extends AbstractFunction { @Override public Message evaluate(FunctionArgs args, EvaluationContext context) { - final Optional optMessage = args.evaluated(MESSAGE_ARG, context, String.class); + final Optional optMessage = args.param(MESSAGE_ARG).eval(args, context, String.class); final String message = optMessage.isPresent() ? optMessage.get() : context.currentMessage().getMessage(); - final Optional optSource = args.evaluated(SOURCE_ARG, context, String.class); + final Optional optSource = args.param(SOURCE_ARG).eval(args, context, String.class); final String source = optSource.isPresent() ? optSource.get() : context.currentMessage().getSource(); - final Optional optTimestamp = args.evaluated(TIMESTAMP_ARG, context, DateTime.class); + final Optional optTimestamp = args.param(TIMESTAMP_ARG).eval(args, context, DateTime.class); final DateTime timestamp = optTimestamp.isPresent() ? optTimestamp.get() : Tools.nowUTC(); final Message newMessage = new Message(message, source, timestamp); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java index bfc51005d448..8f018536b86f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java @@ -36,7 +36,7 @@ public class DropMessage extends AbstractFunction { public Void evaluate(FunctionArgs args, EvaluationContext context) { final Optional message; if (args.isPresent("message")) { - message = args.evaluated("message", context, Message.class); + message = args.param("message").eval(args, context, Message.class); } else { message = Optional.of(context.currentMessage()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java index 797ae69d598a..25e868a5a701 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -26,10 +26,11 @@ public class HasField extends AbstractFunction { public static final String NAME = "has_field"; + public static final String FIELD = "field"; @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final String field = args.required("field", context, String.class); + final String field = args.param(FIELD).evalRequired(args, context, String.class); return context.currentMessage().hasField(field); } @@ -38,7 +39,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(ImmutableList.of(ParameterDescriptor.string("field").build())) + .params(ImmutableList.of(ParameterDescriptor.string(FIELD).build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java index e4f377982808..f683b2d33df1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -26,10 +26,11 @@ public class RemoveField extends AbstractFunction { public static final String NAME = "remove_field"; + public static final String FIELD = "field"; @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final String field = args.required("field", context, String.class); + final String field = args.param(FIELD).evalRequired(args, context, String.class); context.currentMessage().removeField(field); return null; } @@ -39,7 +40,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(ImmutableList.of(ParameterDescriptor.string("field").build())) + .params(ImmutableList.of(ParameterDescriptor.string(FIELD).build())) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java index 862cb288838b..c5108454b55d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -45,11 +45,11 @@ public RouteToStream(StreamService streamService) { @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - String id = args.evaluated(ID_ARG, context, String.class).orElse(""); + String id = args.param(ID_ARG).eval(args, context, String.class).orElse(""); final Stream stream; if ("".equals(id)) { - final String name = args.evaluated(NAME_ARG, context, String.class).orElse(""); + final String name = args.param(NAME_ARG).eval(args, context, String.class).orElse(""); if ("".equals(name)) { return null; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index d3047071747d..dc3a28e076f3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -34,8 +34,8 @@ public class SetField extends AbstractFunction { @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final String field = args.required(FIELD, context, String.class); - final Object value = args.required(VALUE, context, Object.class); + final String field = args.param(FIELD).evalRequired(args, context, String.class); + final Object value = args.param(VALUE).evalRequired(args, context, Object.class); if (!Strings.isNullOrEmpty(field)) { context.currentMessage().addField(field, value); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index 27b6cf36a72b..7b98ed822d0d 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -133,7 +133,7 @@ public FunctionDescriptor descriptor() { functions.put("one_arg", new AbstractFunction() { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - return args.evaluated("one", context, String.class).orElse(""); + return args.param("one").eval(args, context, String.class).orElse(""); } @Override @@ -148,9 +148,9 @@ public FunctionDescriptor descriptor() { functions.put("concat", new AbstractFunction() { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object one = args.evaluated("one", context, Object.class).orElse(""); - final Object two = args.evaluated("two", context, Object.class).orElse(""); - final Object three = args.evaluated("three", context, Object.class).orElse(""); + final Object one = args.param("one").eval(args, context, Object.class).orElse(""); + final Object two = args.param("two").eval(args, context, Object.class).orElse(""); + final Object three = args.param("three").eval(args, context, Object.class).orElse(""); return one.toString() + two.toString() + three.toString(); } @@ -206,7 +206,7 @@ public FunctionDescriptor descriptor() { functions.put("customObject", new AbstractFunction() { @Override public CustomObject evaluate(FunctionArgs args, EvaluationContext context) { - return new CustomObject(args.evaluated("default", context, String.class).orElse("")); + return new CustomObject(args.param("default").eval(args, context, String.class).orElse("")); } @Override @@ -221,7 +221,7 @@ public FunctionDescriptor descriptor() { functions.put("keys", new AbstractFunction() { @Override public List evaluate(FunctionArgs args, EvaluationContext context) { - final Optional map = args.evaluated("map", context, Map.class); + final Optional map = args.param("map").eval(args, context, Map.class); return Lists.newArrayList(map.orElse(Collections.emptyMap()).keySet()); } @@ -237,7 +237,7 @@ public FunctionDescriptor descriptor() { functions.put("sort", new AbstractFunction() { @Override public Collection evaluate(FunctionArgs args, EvaluationContext context) { - final Collection collection = args.evaluated("collection", + final Collection collection = args.param("collection").eval(args, context, Collection.class).orElse(Collections.emptyList()); return Ordering.natural().sortedCopy(collection); From ccb65e0d0342221f3aca855886fb0ee16b7ef94a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sat, 13 Feb 2016 17:25:07 +0100 Subject: [PATCH 074/521] make autoBuild package local --- .../pipelineprocessor/ast/functions/FunctionDescriptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java index b65a73e80c34..3f4625368eb1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java @@ -45,7 +45,7 @@ public static Builder builder() { @AutoValue.Builder public static abstract class Builder { - public abstract FunctionDescriptor autoBuild(); + abstract FunctionDescriptor autoBuild(); public FunctionDescriptor build() { return paramMap(Maps.uniqueIndex(params(), ParameterDescriptor::name)) From 5b9e233fbc258943a0069a7bfdfe9d9ecd72af45 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Sat, 13 Feb 2016 17:26:10 +0100 Subject: [PATCH 075/521] add various string functions - upper/lower/swap case - un-/capitalize - abbreviate move package for regex function --- .../PipelineProcessorModule.java | 18 ++++++- .../functions/strings/Abbreviate.java | 41 ++++++++++++++++ .../functions/strings/Capitalize.java | 25 ++++++++++ .../functions/strings/Lowercase.java | 25 ++++++++++ .../functions/{ => strings}/RegexMatch.java | 2 +- .../strings/StringUtilsFunction.java | 47 +++++++++++++++++++ .../functions/strings/SwapCase.java | 26 ++++++++++ .../functions/strings/Uncapitalize.java | 25 ++++++++++ .../functions/strings/Uppercase.java | 25 ++++++++++ .../parser/PipelineRuleParserTest.java | 2 +- 10 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{ => strings}/RegexMatch.java (98%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index bc1fb984c4c0..4c9be8f9fa87 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -24,7 +24,10 @@ import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; import org.graylog.plugins.pipelineprocessor.functions.FromInput; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; +import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; @@ -32,6 +35,9 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamResource; @@ -77,6 +83,16 @@ protected void configure() { // generic functions addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class); + + // string functions + addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class); + addMessageProcessorFunction(Capitalize.NAME, Capitalize.class); + addMessageProcessorFunction(Lowercase.NAME, Lowercase.class); + addMessageProcessorFunction(SwapCase.NAME, SwapCase.class); + addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); + addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); + + } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java new file mode 100644 index 000000000000..dfb2cb407689 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -0,0 +1,41 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.primitives.Ints.saturatedCast; + +public class Abbreviate extends AbstractFunction { + + public static final String NAME = "abbreviate"; + private static final String VALUE = "value"; + private static final String WIDTH = "width"; + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = args.param(VALUE).evalRequired(args, context, String.class); + final Long maxWidth = args.param(WIDTH).evalRequired(args, context, Long.class); + + return StringUtils.abbreviate(value, saturatedCast(maxWidth)); + } + + @Override + public FunctionDescriptor descriptor() { + ImmutableList.Builder params = ImmutableList.builder(); + params.add(); + + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(ImmutableList.of( + ParameterDescriptor.string(VALUE).build(), + ParameterDescriptor.string(WIDTH).build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java new file mode 100644 index 000000000000..6fe306475685 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Capitalize extends StringUtilsFunction { + + public static final String NAME = "capitalize"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected boolean isLocaleAware() { + return false; + } + + @Override + protected String apply(String value, Locale unused) { + return StringUtils.capitalize(value); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java new file mode 100644 index 000000000000..49599ca42f88 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Lowercase extends StringUtilsFunction { + + public static final String NAME = "lowercase"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected boolean isLocaleAware() { + return true; + } + + @Override + protected String apply(String value, Locale locale) { + return StringUtils.lowerCase(value, locale); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java similarity index 98% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index 34ff4b669a1f..528e73d77cc7 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -1,4 +1,4 @@ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.strings; import com.google.common.collect.ForwardingMap; import com.google.common.collect.ImmutableMap; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java new file mode 100644 index 000000000000..fda942b00e1a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java @@ -0,0 +1,47 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Locale; + +public abstract class StringUtilsFunction extends AbstractFunction { + + private static final String VALUE = "value"; + private static final String LOCALE = "locale"; + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = args.param(VALUE).evalRequired(args, context, String.class); + Locale locale = Locale.ENGLISH; + if (isLocaleAware()) { + locale = args.param(LOCALE).eval(args, context, Locale.class).orElse(Locale.ENGLISH); + } + return apply(value, locale); + } + + @Override + public FunctionDescriptor descriptor() { + ImmutableList.Builder params = ImmutableList.builder(); + params.add(ParameterDescriptor.string(VALUE).build()); + if (isLocaleAware()) { + params.add(ParameterDescriptor.string(LOCALE, + Locale.class).optional().transform(Locale::forLanguageTag).build()); + } + return FunctionDescriptor.builder() + .name(getName()) + .returnType(String.class) + .params(params.build()) + .build(); + } + + protected abstract String getName(); + + protected abstract boolean isLocaleAware(); + + protected abstract String apply(String value, Locale locale); +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java new file mode 100644 index 000000000000..a220a14c7531 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java @@ -0,0 +1,26 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import javax.annotation.Nullable; +import java.util.Locale; + +public class SwapCase extends StringUtilsFunction { + + public static final String NAME = "swapcase"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected boolean isLocaleAware() { + return false; + } + + @Override + protected String apply(String value, Locale unused) { + return StringUtils.swapCase(value); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java new file mode 100644 index 000000000000..57c39d72cdcf --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Uncapitalize extends StringUtilsFunction { + + public static final String NAME = "uncapitalize"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected boolean isLocaleAware() { + return false; + } + + @Override + protected String apply(String value, Locale unused) { + return StringUtils.uncapitalize(value); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java new file mode 100644 index 000000000000..b64eb516fcbb --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Uppercase extends StringUtilsFunction { + + public static final String NAME = "uppercase"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected boolean isLocaleAware() { + return true; + } + + @Override + protected String apply(String value, Locale locale) { + return StringUtils.upperCase(value, locale); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index 7b98ed822d0d..bad17766295a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -32,7 +32,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; From 6aff93303e149c48358c1de593e0385618bd0a80 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 15 Feb 2016 18:57:09 +0100 Subject: [PATCH 076/521] add parse_json, select_jsonpath and set_fields functions - parse json into JsonNode - allow multiple jsonpath selections from the document - setting multiple fields (a map really) at once various cleanups due to type erasures. --- .../PipelineProcessorModule.java | 15 ++- .../ast/functions/ParameterDescriptor.java | 1 + .../functions/json/JsonParse.java | 51 +++++++ .../functions/json/SelectJsonPath.java | 79 +++++++++++ .../functions/messages/SetFields.java | 37 ++++++ .../pipelineprocessor/BaseParserTest.java | 66 +++++++++ .../functions/FunctionsSnippetsTest.java | 125 ++++++++++++++++++ .../parser/PipelineRuleParserTest.java | 64 +-------- .../pipelineprocessor/functions/jsonpath.txt | 10 ++ 9 files changed, 384 insertions(+), 64 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java create mode 100644 src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java create mode 100644 src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 4c9be8f9fa87..b489f935e253 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -24,17 +24,20 @@ import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; import org.graylog.plugins.pipelineprocessor.functions.FromInput; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; -import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; -import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; -import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; +import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; +import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; +import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; @@ -72,6 +75,7 @@ protected void configure() { // message related functions addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(SetField.NAME, SetField.class); + addMessageProcessorFunction(SetFields.NAME, SetFields.class); addMessageProcessorFunction(RemoveField.NAME, RemoveField.class); addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); @@ -92,6 +96,9 @@ protected void configure() { addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); + // json + addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); + addMessageProcessorFunction(SelectJsonPath.NAME, SelectJsonPath.class); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index d387246d1f67..b722679e5e96 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -104,6 +104,7 @@ public X evalRequired(FunctionArgs args, EvaluationContext context, Class { public abstract Builder type(Class type); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java new file mode 100644 index 000000000000..7e614e3f2e08 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java @@ -0,0 +1,51 @@ +package org.graylog.plugins.pipelineprocessor.functions.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +import java.io.IOException; + +import static com.google.common.collect.ImmutableList.of; + +public class JsonParse extends AbstractFunction { + private static final Logger log = LoggerFactory.getLogger(JsonParse.class); + public static final String NAME = "parse_json"; + + private final ObjectMapper objectMapper; + + @Inject + public JsonParse(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + public JsonNode evaluate(FunctionArgs args, EvaluationContext context) { + final String value = args.param("value").evalRequired(args, context, String.class); + try { + return objectMapper.readTree(value); + } catch (IOException e) { + log.warn("Unable to parse json", e); + } + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(JsonNode.class) + .params(of( + ParameterDescriptor.string("value").build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java new file mode 100644 index 000000000000..416c16ead768 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -0,0 +1,79 @@ +package org.graylog.plugins.pipelineprocessor.functions.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.TypeLiteral; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.google.common.collect.ImmutableList.of; + +public class SelectJsonPath extends AbstractFunction> { + + public static final String NAME = "select_jsonpath"; + private final Configuration configuration; + + @Inject + public SelectJsonPath(ObjectMapper objectMapper) { + configuration = Configuration.builder() + .options(Option.SUPPRESS_EXCEPTIONS) + .jsonProvider(new JacksonJsonNodeJsonProvider(objectMapper)) + .build(); + } + + @Override + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { + if ("paths".equals(name)) { + final Object o = super.preComputeConstantArgument(args, name, arg); + //noinspection unchecked + final HashMap map = (HashMap) o; + return map.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + stringStringEntry -> JsonPath.compile(stringStringEntry.getValue()) + )); + } + return super.preComputeConstantArgument(args, name, arg); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + final JsonNode json = args.param("json").evalRequired(args, context, JsonNode.class); + //noinspection unchecked + final Map paths = args.param("paths").evalRequired(args, context, Map.class); + + return paths.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().read(json, configuration) + )); + } + + @Override + public FunctionDescriptor> descriptor() { + //noinspection unchecked + return FunctionDescriptor.>builder() + .name(NAME) + .returnType((Class>) new TypeLiteral>() {}.getRawType()) + .params(of( + ParameterDescriptor.type("json", JsonNode.class).build(), + ParameterDescriptor + .type("paths", Map.class, Map.class) + .build() + )) + .build(); + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java new file mode 100644 index 000000000000..e481006ccb98 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -0,0 +1,37 @@ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; + +public class SetFields extends AbstractFunction { + + public static final String NAME = "set_fields"; + public static final String FIELDS = "fields"; + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + //noinspection unchecked + final Map fields = args.param(FIELDS).evalRequired(args, context, Map.class); + context.currentMessage().addFields(fields); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of( + ParameterDescriptor.type(FIELDS, Map.class).build() + )) + .build(); + } + +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java new file mode 100644 index 000000000000..15a50ca31be8 --- /dev/null +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java @@ -0,0 +1,66 @@ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.base.Charsets; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog2.plugin.Message; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.rules.TestName; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicBoolean; + +public class BaseParserTest { + protected static final AtomicBoolean actionsTriggered = new AtomicBoolean(false); + protected static FunctionRegistry functionRegistry; + + @org.junit.Rule + public TestName name = new TestName(); + protected PipelineRuleParser parser; + + @Before + public void setup() { + parser = new PipelineRuleParser(functionRegistry); + // initialize before every test! + actionsTriggered.set(false); + } + + protected Message evaluateRule(Rule rule, Message message) { + final EvaluationContext context = new EvaluationContext(message); + if (rule.when().evaluateBool(context)) { + + for (Statement statement : rule.then()) { + statement.evaluate(context); + } + return message; + } else { + return null; + } + } + + @Nullable + protected Message evaluateRule(Rule rule) { + final Message message = new Message("hello test", "source", DateTime.now()); + return evaluateRule(rule, message); + } + + protected String ruleForTest() { + try { + final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); + final Path path = Paths.get(resource.toURI()); + final byte[] bytes = Files.readAllBytes(path); + return new String(bytes, Charsets.UTF_8); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java new file mode 100644 index 000000000000..9721c5c98d98 --- /dev/null +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -0,0 +1,125 @@ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import org.graylog.plugins.pipelineprocessor.BaseParserTest; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; +import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; +import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; +import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Tools; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class FunctionsSnippetsTest extends BaseParserTest { + + @BeforeClass + public static void registerFunctions() { + final Map> functions = Maps.newHashMap(); + + functions.put(BooleanCoercion.NAME, new BooleanCoercion()); + functions.put(DoubleCoercion.NAME, new DoubleCoercion()); + functions.put(LongCoercion.NAME, new LongCoercion()); + functions.put(StringCoercion.NAME, new StringCoercion()); + + // message related functions + functions.put(HasField.NAME, new HasField()); + functions.put(SetField.NAME, new SetField()); + functions.put(SetFields.NAME, new SetFields()); + functions.put(RemoveField.NAME, new RemoveField()); + + functions.put(DropMessage.NAME, new DropMessage()); + functions.put(CreateMessage.NAME, new CreateMessage()); + // TODO needs mock + //functions.put(RouteToStream.NAME, new RouteToStream())); + + // input related functions + // TODO needs mock + //functions.put(FromInput.NAME, new FromInput()); + + // generic functions + functions.put(RegexMatch.NAME, new RegexMatch()); + + // string functions + functions.put(Abbreviate.NAME, new Abbreviate()); + functions.put(Capitalize.NAME, new Capitalize()); + functions.put(Lowercase.NAME, new Lowercase()); + functions.put(SwapCase.NAME, new SwapCase()); + functions.put(Uncapitalize.NAME, new Uncapitalize()); + functions.put(Uppercase.NAME, new Uppercase()); + + final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + functions.put(JsonParse.NAME, new JsonParse(objectMapper)); + functions.put(SelectJsonPath.NAME, new SelectJsonPath(objectMapper)); + + functionRegistry = new FunctionRegistry(functions); + } + + @Test + public void jsonpath() { + final String json = "{\n" + + " \"store\": {\n" + + " \"book\": [\n" + + " {\n" + + " \"category\": \"reference\",\n" + + " \"author\": \"Nigel Rees\",\n" + + " \"title\": \"Sayings of the Century\",\n" + + " \"price\": 8.95\n" + + " },\n" + + " {\n" + + " \"category\": \"fiction\",\n" + + " \"author\": \"Evelyn Waugh\",\n" + + " \"title\": \"Sword of Honour\",\n" + + " \"price\": 12.99\n" + + " },\n" + + " {\n" + + " \"category\": \"fiction\",\n" + + " \"author\": \"Herman Melville\",\n" + + " \"title\": \"Moby Dick\",\n" + + " \"isbn\": \"0-553-21311-3\",\n" + + " \"price\": 8.99\n" + + " },\n" + + " {\n" + + " \"category\": \"fiction\",\n" + + " \"author\": \"J. R. R. Tolkien\",\n" + + " \"title\": \"The Lord of the Rings\",\n" + + " \"isbn\": \"0-395-19395-8\",\n" + + " \"price\": 22.99\n" + + " }\n" + + " ],\n" + + " \"bicycle\": {\n" + + " \"color\": \"red\",\n" + + " \"price\": 19.95\n" + + " }\n" + + " },\n" + + " \"expensive\": 10\n" + + "}"; + + final Rule rule = parser.parseRule(ruleForTest()); + final Message message = evaluateRule(rule, new Message(json, "test", Tools.nowUTC())); + + assertThat(message.hasField("author_first")).isTrue(); + assertThat(message.hasField("author_last")).isTrue(); + + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index bad17766295a..15116c18507a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -16,11 +16,11 @@ */ package org.graylog.plugins.pipelineprocessor.parser; -import com.google.common.base.Charsets; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; +import org.graylog.plugins.pipelineprocessor.BaseParserTest; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.ast.Rule; @@ -30,12 +30,11 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType; import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType; @@ -46,24 +45,14 @@ import org.joda.time.DateTime; import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.rules.TestName; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; + import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.collect.ImmutableList.of; import static org.junit.Assert.assertArrayEquals; @@ -72,15 +61,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class PipelineRuleParserTest { - - @org.junit.Rule - public TestName name = new TestName(); - - private PipelineRuleParser parser; - private static FunctionRegistry functionRegistry; - - private static final AtomicBoolean actionsTriggered = new AtomicBoolean(false); +public class PipelineRuleParserTest extends BaseParserTest { @BeforeClass public static void registerFunctions() { @@ -260,13 +241,6 @@ public FunctionDescriptor descriptor() { functionRegistry = new FunctionRegistry(functions); } - @Before - public void setup() { - parser = new PipelineRuleParser(functionRegistry); - // initialize before every test! - actionsTriggered.set(false); - } - @After public void tearDown() { parser = null; @@ -491,36 +465,6 @@ public void regexMatch() { } } - private Message evaluateRule(Rule rule, Message message) { - final EvaluationContext context = new EvaluationContext(message); - if (rule.when().evaluateBool(context)) { - - for (Statement statement : rule.then()) { - statement.evaluate(context); - } - return message; - } else { - return null; - } - } - - @Nullable - private Message evaluateRule(Rule rule) { - final Message message = new Message("hello test", "source", DateTime.now()); - return evaluateRule(rule, message); - } - - private String ruleForTest() { - try { - final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); - final Path path = Paths.get(resource.toURI()); - final byte[] bytes = Files.readAllBytes(path); - return new String(bytes, Charsets.UTF_8); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - public static class CustomObject { private final String id; diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt new file mode 100644 index 000000000000..81c231bb1212 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt @@ -0,0 +1,10 @@ +rule "jsonpath" +when true +then + let x = parse_json(string(message.`message`)); + let new_fields = select_jsonpath(x, + { author_first: "$['store']['book'][0]['author']", + author_last: "$['store']['book'][-1:]['author']" + }); + set_fields(new_fields); +end \ No newline at end of file From e3d7578b4c7d9b1beff1131308219c9429ec7907 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 09:35:31 +0100 Subject: [PATCH 077/521] add license plugin to build - update licenses everywhere, this is GPL3 --- benchmarks/pom.xml | 32 ++++++++++++++++++- .../pipeline/PipelineBenchmark.java | 16 ++++++++++ .../ast/functions/AbstractFunction.java | 16 ++++++++++ .../functions/json/JsonParse.java | 16 ++++++++++ .../functions/json/SelectJsonPath.java | 16 ++++++++++ .../functions/messages/SetFields.java | 16 ++++++++++ .../functions/strings/Abbreviate.java | 16 ++++++++++ .../functions/strings/Capitalize.java | 16 ++++++++++ .../functions/strings/Lowercase.java | 16 ++++++++++ .../functions/strings/RegexMatch.java | 16 ++++++++++ .../strings/StringUtilsFunction.java | 16 ++++++++++ .../functions/strings/SwapCase.java | 16 ++++++++++ .../functions/strings/Uncapitalize.java | 16 ++++++++++ .../functions/strings/Uppercase.java | 16 ++++++++++ .../pipelineprocessor/BaseParserTest.java | 16 ++++++++++ .../functions/FunctionsSnippetsTest.java | 16 ++++++++++ 16 files changed, 271 insertions(+), 1 deletion(-) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 13207b9d945e..1bd04208e56c 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -92,7 +92,7 @@ THE POSSIBILITY OF SUCH DAMAGE. 1.11.3 1.8 pipeline-benchmarks - 2.0.0-alpha.2-SNAPSHOT + 2.0.0-alpha.3-SNAPSHOT @@ -107,6 +107,10 @@ THE POSSIBILITY OF SUCH DAMAGE. ${javac.target} + + com.mycila + license-maven-plugin + org.apache.maven.plugins maven-shade-plugin @@ -182,6 +186,32 @@ THE POSSIBILITY OF SUCH DAMAGE. maven-surefire-plugin 2.17 + + com.mycila + license-maven-plugin + 2.11 + +
    com/mycila/maven/plugin/license/templates/GPL-3.txt
    + + ${project.organization.name} + Graylog + + + **/src/main/java/** + **/src/test/java/** + + + graylog2-web-interface/plugin/** + +     + + + + check + + + +     diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index 1f322f8abb08..e099832696a1 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ /** * This file is part of Graylog Pipeline Processor. * diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java index 47fb994b1552..c668f2e07628 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.ast.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java index 7e614e3f2e08..c887a7951c16 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.json; import com.fasterxml.jackson.databind.JsonNode; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java index 416c16ead768..9d7e1dab7981 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.json; import com.fasterxml.jackson.databind.JsonNode; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java index e481006ccb98..2229fc279bde 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.messages; import org.graylog.plugins.pipelineprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java index dfb2cb407689..2386904caeee 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import com.google.common.collect.ImmutableList; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java index 6fe306475685..cc3e85ac13ba 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java index 49599ca42f88..f03609918737 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index 528e73d77cc7..c0dce7fa6d31 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import com.google.common.collect.ForwardingMap; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java index fda942b00e1a..ba4df872f882 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import com.google.common.collect.ImmutableList; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java index a220a14c7531..5e13b25a67ad 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java index 57c39d72cdcf..e1dc73994992 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java index b64eb516fcbb..32396cb9bbf7 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import org.apache.commons.lang3.StringUtils; diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java index 15a50ca31be8..35d6772c2416 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor; import com.google.common.base.Charsets; diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 9721c5c98d98..f9a95bbde5f4 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions; import com.fasterxml.jackson.databind.ObjectMapper; From 482d8bf28f06f2b063ed70b426b69f8118ddbbfc Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 12:18:19 +0100 Subject: [PATCH 078/521] allow signed floats and integer literals --- .../plugins/pipelineprocessor/parser/RuleLang.g4 | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 index 83c5d3cfc787..27c2d93ebb5b 100644 --- a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 @@ -79,6 +79,10 @@ expression | fieldSet=expression '.' field=expression # Nested | array=expression '[' index=expression ']' # IndexedAccess | functionCall # Func +// | ('+'|'-') expression # SignedExpression +//// | ('~'|'!') expression # BitwiseNot +// | expression ('*'|'/'|'%') expression # Multiplication +// | expression ('+'|'-') expression # Addition | left=expression comparison=('<=' | '>=' | '>' | '<') right=expression # Comparison | left=expression equality=('==' | '!=') right=expression # Equality | left=expression and=And right=expression # And @@ -151,22 +155,22 @@ Integer fragment DecimalIntegerLiteral - : DecimalNumeral IntegerTypeSuffix? + : Sign? DecimalNumeral IntegerTypeSuffix? ; fragment HexIntegerLiteral - : HexNumeral IntegerTypeSuffix? + : Sign? HexNumeral IntegerTypeSuffix? ; fragment OctalIntegerLiteral - : OctalNumeral IntegerTypeSuffix? + : Sign? OctalNumeral IntegerTypeSuffix? ; fragment BinaryIntegerLiteral - : BinaryNumeral IntegerTypeSuffix? + : Sign? BinaryNumeral IntegerTypeSuffix? ; fragment @@ -272,8 +276,8 @@ BinaryDigitOrUnderscore // Floats Float - : DecimalFloatingPointLiteral - | HexadecimalFloatingPointLiteral + : Sign? DecimalFloatingPointLiteral + | Sign? HexadecimalFloatingPointLiteral ; fragment From 901e22954efd7699cc1b8e9e0eeee0a1a55a5cc5 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 12:20:17 +0100 Subject: [PATCH 079/521] add substring function and test - simply expose StringUtils.substring(string, start, end) - end is optional, is "string.length()" if omitted slightly refactored test classes and moved extracted functions module for future use of injection in tests --- .../PipelineProcessorModule.java | 74 +----------------- .../functions/ProcessorFunctionsModule.java | 78 +++++++++++++++++++ .../functions/strings/Substring.java | 38 +++++++++ .../pipelineprocessor/BaseParserTest.java | 29 +++++++ .../functions/FunctionsSnippetsTest.java | 13 +++- .../parser/PipelineRuleParserTest.java | 3 +- .../pipelineprocessor/functions/substring.txt | 15 ++++ 7 files changed, 174 insertions(+), 76 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index b489f935e253..0d67b8279371 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -16,31 +16,7 @@ */ package org.graylog.plugins.pipelineprocessor; -import com.google.inject.Binder; -import com.google.inject.TypeLiteral; -import com.google.inject.multibindings.MapBinder; -import org.graylog.plugins.pipelineprocessor.ast.functions.Function; -import org.graylog.plugins.pipelineprocessor.functions.BooleanCoercion; -import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion; -import org.graylog.plugins.pipelineprocessor.functions.FromInput; -import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; -import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; -import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; -import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; -import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; -import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; -import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; -import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; -import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; -import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; -import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; -import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; -import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; -import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; -import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; -import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; -import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog.plugins.pipelineprocessor.functions.ProcessorFunctionsModule; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamResource; @@ -66,52 +42,6 @@ protected void configure() { addRestResource(PipelineResource.class); addRestResource(PipelineStreamResource.class); - // built-in functions - addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); - addMessageProcessorFunction(DoubleCoercion.NAME, DoubleCoercion.class); - addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); - addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); - - // message related functions - addMessageProcessorFunction(HasField.NAME, HasField.class); - addMessageProcessorFunction(SetField.NAME, SetField.class); - addMessageProcessorFunction(SetFields.NAME, SetFields.class); - addMessageProcessorFunction(RemoveField.NAME, RemoveField.class); - - addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); - addMessageProcessorFunction(CreateMessage.NAME, CreateMessage.class); - addMessageProcessorFunction(RouteToStream.NAME, RouteToStream.class); - - // input related functions - addMessageProcessorFunction(FromInput.NAME, FromInput.class); - - // generic functions - addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class); - - // string functions - addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class); - addMessageProcessorFunction(Capitalize.NAME, Capitalize.class); - addMessageProcessorFunction(Lowercase.NAME, Lowercase.class); - addMessageProcessorFunction(SwapCase.NAME, SwapCase.class); - addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); - addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); - - // json - addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); - addMessageProcessorFunction(SelectJsonPath.NAME, SelectJsonPath.class); - - } - - protected void addMessageProcessorFunction(String name, Class> functionClass) { - addMessageProcessorFunction(binder(), name, functionClass); - } - - public static MapBinder> processorFunctionBinder(Binder binder) { - return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral>() {}); - } - - public static void addMessageProcessorFunction(Binder binder, String name, Class> functionClass) { - processorFunctionBinder(binder).addBinding(name).to(functionClass); - + install(new ProcessorFunctionsModule()); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java new file mode 100644 index 000000000000..635c5a5b1c83 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -0,0 +1,78 @@ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.google.inject.Binder; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; +import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; +import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; +import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; +import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog2.plugin.PluginModule; + +public class ProcessorFunctionsModule extends PluginModule { + @Override + protected void configure() { + // built-in functions + addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); + addMessageProcessorFunction(DoubleCoercion.NAME, DoubleCoercion.class); + addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); + addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); + + // message related functions + addMessageProcessorFunction(HasField.NAME, HasField.class); + addMessageProcessorFunction(SetField.NAME, SetField.class); + addMessageProcessorFunction(SetFields.NAME, SetFields.class); + addMessageProcessorFunction(RemoveField.NAME, RemoveField.class); + + addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); + addMessageProcessorFunction(CreateMessage.NAME, CreateMessage.class); + addMessageProcessorFunction(RouteToStream.NAME, RouteToStream.class); + + // input related functions + addMessageProcessorFunction(FromInput.NAME, FromInput.class); + + // generic functions + addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class); + + // string functions + addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class); + addMessageProcessorFunction(Capitalize.NAME, Capitalize.class); + addMessageProcessorFunction(Lowercase.NAME, Lowercase.class); + addMessageProcessorFunction(Substring.NAME, Substring.class); + addMessageProcessorFunction(SwapCase.NAME, SwapCase.class); + addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); + addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); + + // json + addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); + addMessageProcessorFunction(SelectJsonPath.NAME, SelectJsonPath.class); + + } + + protected void addMessageProcessorFunction(String name, Class> functionClass) { + addMessageProcessorFunction(binder(), name, functionClass); + } + + public static MapBinder> processorFunctionBinder(Binder binder) { + return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral>() {}); + } + + public static void addMessageProcessorFunction(Binder binder, String name, Class> functionClass) { + processorFunctionBinder(binder).addBinding(name).to(functionClass); + + } +} \ No newline at end of file diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java new file mode 100644 index 000000000000..9d4f65557dfa --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java @@ -0,0 +1,38 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.primitives.Ints; +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Substring extends AbstractFunction { + + public static final String NAME = "substring"; + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = args.param("value").evalRequired(args, context, String.class); + final int start = Ints.saturatedCast(args.param("start").evalRequired(args, context, Long.class)); + final int end = Ints.saturatedCast(args.param("end").eval(args, context, Long.class).orElse((long) value.length())); + + return StringUtils.substring(value, start, end); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + ParameterDescriptor.string("value").build(), + ParameterDescriptor.integer("start").build(), + ParameterDescriptor.integer("end").optional().build() + )) + .build(); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java index 35d6772c2416..ff28ded0837c 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java @@ -17,7 +17,12 @@ package org.graylog.plugins.pipelineprocessor; import com.google.common.base.Charsets; +import com.google.common.collect.Maps; import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; @@ -33,8 +38,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashMap; import java.util.concurrent.atomic.AtomicBoolean; +import static com.google.common.collect.ImmutableList.of; + public class BaseParserTest { protected static final AtomicBoolean actionsTriggered = new AtomicBoolean(false); protected static FunctionRegistry functionRegistry; @@ -43,6 +51,27 @@ public class BaseParserTest { public TestName name = new TestName(); protected PipelineRuleParser parser; + protected static HashMap> commonFunctions() { + final HashMap> functions = Maps.newHashMap(); + functions.put("trigger_test", new AbstractFunction() { + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + actionsTriggered.set(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("trigger_test") + .returnType(Void.class) + .params(of()) + .build(); + } + }); + return functions; + } + @Before public void setup() { parser = new PipelineRuleParser(functionRegistry); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index f9a95bbde5f4..2784835a15eb 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -17,7 +17,6 @@ package org.graylog.plugins.pipelineprocessor.functions; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; import org.graylog.plugins.pipelineprocessor.BaseParserTest; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; @@ -33,6 +32,7 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; @@ -51,7 +51,7 @@ public class FunctionsSnippetsTest extends BaseParserTest { @BeforeClass public static void registerFunctions() { - final Map> functions = Maps.newHashMap(); + final Map> functions = commonFunctions(); functions.put(BooleanCoercion.NAME, new BooleanCoercion()); functions.put(DoubleCoercion.NAME, new DoubleCoercion()); @@ -80,6 +80,7 @@ public static void registerFunctions() { functions.put(Abbreviate.NAME, new Abbreviate()); functions.put(Capitalize.NAME, new Capitalize()); functions.put(Lowercase.NAME, new Lowercase()); + functions.put(Substring.NAME, new Substring()); functions.put(SwapCase.NAME, new SwapCase()); functions.put(Uncapitalize.NAME, new Uncapitalize()); functions.put(Uppercase.NAME, new Uppercase()); @@ -138,4 +139,12 @@ public void jsonpath() { assertThat(message.hasField("author_last")).isTrue(); } + + @Test + public void substring() { + final Rule rule = parser.parseRule(ruleForTest()); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index 15116c18507a..e76271e46448 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -18,7 +18,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import org.graylog.plugins.pipelineprocessor.BaseParserTest; import org.graylog.plugins.pipelineprocessor.EvaluationContext; @@ -65,7 +64,7 @@ public class PipelineRuleParserTest extends BaseParserTest { @BeforeClass public static void registerFunctions() { - final Map> functions = Maps.newHashMap(); + final Map> functions = commonFunctions(); functions.put("nein", new AbstractFunction() { @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt new file mode 100644 index 000000000000..7d253bdb10da --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt @@ -0,0 +1,15 @@ +rule "substrings" +when + substring("abc", 0, 2) == "ab" && + substring("abc", 0, 2) == "ab" && + substring("abc", 2, 0) == "" && + substring("abc", 2, 4) == "c" && + substring("abc", 4, 6) == "" && + substring("abc", 2, 2) == "" && + substring("abc", -2, -1) == "b" && + substring("abc", -4, 2) == "ab" && + substring("abc", 1) == "bc" && + substring("abc", 0, -1) == "ab" +then + trigger_test(); +end \ No newline at end of file From 9bb4f6c3a197357cead88ea8124558bb807fc158 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 12:29:09 +0100 Subject: [PATCH 080/521] renamed conversion functions using a "to" adapted all test cases renamed the classes from *Coercion to *Conversion and moved to own package --- .../benchmarks/pipeline/PipelineBenchmark.java | 6 +++--- .../functions/ProcessorFunctionsModule.java | 12 ++++++++---- .../BooleanConversion.java} | 6 +++--- .../DoubleConversion.java} | 6 +++--- .../LongConversion.java} | 6 +++--- .../StringConversion.java} | 6 +++--- .../functions/FunctionsSnippetsTest.java | 12 ++++++++---- .../parser/PipelineRuleParserTest.java | 8 ++++---- .../processors/PipelineInterpreterTest.java | 4 ++-- .../plugins/pipelineprocessor/functions/jsonpath.txt | 2 +- .../plugins/pipelineprocessor/parser/messageRef.txt | 2 +- .../parser/messageRefQuotedField.txt | 2 +- .../pipelineprocessor/parser/typedFieldAccess.txt | 2 +- 13 files changed, 41 insertions(+), 33 deletions(-) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{BooleanCoercion.java => conversion/BooleanConversion.java} (90%) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{DoubleCoercion.java => conversion/DoubleConversion.java} (92%) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{LongCoercion.java => conversion/LongConversion.java} (92%) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/{StringCoercion.java => conversion/StringConversion.java} (92%) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index e099832696a1..98adc38e4089 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -43,7 +43,7 @@ import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; -import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; @@ -115,7 +115,7 @@ public InterpreterState() { "title", "description", "rule \"add\"\n" + - "when string(message.`message`) == \"original message\"\n" + + "when tostring(message.`message`) == \"original message\"\n" + "then\n" + " set_field(\"field\", \"derived message\");\n" + "end", @@ -144,7 +144,7 @@ public InterpreterState() { final Map> functions = Maps.newHashMap(); functions.put(SetField.NAME, new SetField()); - functions.put(StringCoercion.NAME, new StringCoercion()); + functions.put(StringConversion.NAME, new StringConversion()); final PipelineRuleParser parser = setupParser(functions); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 635c5a5b1c83..ae50450a5384 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -4,6 +4,10 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.conversion.BooleanConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -27,10 +31,10 @@ public class ProcessorFunctionsModule extends PluginModule { @Override protected void configure() { // built-in functions - addMessageProcessorFunction(BooleanCoercion.NAME, BooleanCoercion.class); - addMessageProcessorFunction(DoubleCoercion.NAME, DoubleCoercion.class); - addMessageProcessorFunction(LongCoercion.NAME, LongCoercion.class); - addMessageProcessorFunction(StringCoercion.NAME, StringCoercion.class); + addMessageProcessorFunction(BooleanConversion.NAME, BooleanConversion.class); + addMessageProcessorFunction(DoubleConversion.NAME, DoubleConversion.class); + addMessageProcessorFunction(LongConversion.NAME, LongConversion.class); + addMessageProcessorFunction(StringConversion.NAME, StringConversion.class); // message related functions addMessageProcessorFunction(HasField.NAME, HasField.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java similarity index 90% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java index 309bf3a6f57a..d7d47f29556c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/BooleanCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.conversion; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; @@ -24,8 +24,8 @@ import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; -public class BooleanCoercion extends AbstractFunction { - public static final String NAME = "bool"; +public class BooleanConversion extends AbstractFunction { + public static final String NAME = "tobool"; private static final String VALUE = "value"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java similarity index 92% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java index 70f39bda466f..6bbf04893836 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/DoubleCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.conversion; import com.google.common.primitives.Doubles; import org.graylog.plugins.pipelineprocessor.EvaluationContext; @@ -27,9 +27,9 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; -public class DoubleCoercion extends AbstractFunction { +public class DoubleConversion extends AbstractFunction { - public static final String NAME = "double"; + public static final String NAME = "todouble"; private static final String VALUE = "value"; private static final String DEFAULT = "default"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java similarity index 92% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java index 55899c82f9ae..b3074809b090 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/LongCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.conversion; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; @@ -27,9 +27,9 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.integer; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; -public class LongCoercion extends AbstractFunction { +public class LongConversion extends AbstractFunction { - public static final String NAME = "long"; + public static final String NAME = "tolong"; private static final String VALUE = "value"; private static final String DEFAULT = "default"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java similarity index 92% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 9baa228eee43..19b537634b8a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/StringCoercion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -14,7 +14,7 @@ * You should have received a copy of the GNU General Public License * along with Graylog Pipeline Processor. If not, see . */ -package org.graylog.plugins.pipelineprocessor.functions; +package org.graylog.plugins.pipelineprocessor.functions.conversion; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; @@ -25,9 +25,9 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; -public class StringCoercion extends AbstractFunction { +public class StringConversion extends AbstractFunction { - public static final String NAME = "string"; + public static final String NAME = "tostring"; private static final String VALUE = "value"; private static final String DEFAULT = "default"; diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 2784835a15eb..c4797a03a6d9 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -20,6 +20,10 @@ import org.graylog.plugins.pipelineprocessor.BaseParserTest; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.conversion.BooleanConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -53,10 +57,10 @@ public class FunctionsSnippetsTest extends BaseParserTest { public static void registerFunctions() { final Map> functions = commonFunctions(); - functions.put(BooleanCoercion.NAME, new BooleanCoercion()); - functions.put(DoubleCoercion.NAME, new DoubleCoercion()); - functions.put(LongCoercion.NAME, new LongCoercion()); - functions.put(StringCoercion.NAME, new StringCoercion()); + functions.put(BooleanConversion.NAME, new BooleanConversion()); + functions.put(DoubleConversion.NAME, new DoubleConversion()); + functions.put(LongConversion.NAME, new LongConversion()); + functions.put(StringConversion.NAME, new StringConversion()); // message related functions functions.put(HasField.NAME, new HasField()); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index e76271e46448..edd6868501e4 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -29,8 +29,8 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; -import org.graylog.plugins.pipelineprocessor.functions.LongCoercion; -import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; @@ -232,8 +232,8 @@ public FunctionDescriptor descriptor() { .build(); } }); - functions.put(LongCoercion.NAME, new LongCoercion()); - functions.put(StringCoercion.NAME, new StringCoercion()); + functions.put(LongConversion.NAME, new LongConversion()); + functions.put(StringConversion.NAME, new StringConversion()); functions.put(SetField.NAME, new SetField()); functions.put(HasField.NAME, new HasField()); functions.put(RegexMatch.NAME, new RegexMatch()); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index 867f796ee6a2..fc6d2fe2f245 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -24,7 +24,7 @@ import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; -import org.graylog.plugins.pipelineprocessor.functions.StringCoercion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; @@ -85,7 +85,7 @@ public void testCreateMessage() { final Map> functions = Maps.newHashMap(); functions.put(CreateMessage.NAME, new CreateMessage()); - functions.put(StringCoercion.NAME, new StringCoercion()); + functions.put(StringConversion.NAME, new StringConversion()); final PipelineRuleParser parser = setupParser(functions); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt index 81c231bb1212..0450176dcbb8 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt @@ -1,7 +1,7 @@ rule "jsonpath" when true then - let x = parse_json(string(message.`message`)); + let x = parse_json(tostring(message.`message`)); let new_fields = select_jsonpath(x, { author_first: "$['store']['book'][0]['author']", author_last: "$['store']['book'][-1:]['author']" diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt index 8285453124d2..59206e7e559f 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt @@ -1,5 +1,5 @@ rule "message field ref" -when long(value: message.responseCode, default: 200) >= 500 +when tolong(value: message.responseCode, default: 200) >= 500 then set_field(field: "response_category", value: "server_error"); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt index 7e1067903ba6..3c74dc266fc3 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt @@ -1,5 +1,5 @@ rule "test" -when string(message.`@specialfieldname`, "empty") == "string" +when tostring(message.`@specialfieldname`, "empty") == "string" then trigger_test(); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt index 727d06057b33..5518b85c7c03 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt @@ -1,6 +1,6 @@ rule "typed field access" when - long(customObject("1").id, 0) < 2 + tolong(customObject("1").id, 0) < 2 then trigger_test(); end \ No newline at end of file From 43ab9026fd0bc8a604b8f729fa71669eb4465be2 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 12:34:03 +0100 Subject: [PATCH 081/521] accessing the context's current message must be done with $message now renamed message reference from 'message' to '$message' to work around having to special case accessing the field message this really saves a lot of fuckeria and state handling in the parser, and $message is special enough to warrant a special name (it's a context field after all, and not explicitly declared in the rule) adapted test cases and benchmark --- .../java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java | 2 +- .../org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 | 2 +- .../graylog/plugins/pipelineprocessor/functions/jsonpath.txt | 2 +- .../org/graylog/plugins/pipelineprocessor/parser/messageRef.txt | 2 +- .../plugins/pipelineprocessor/parser/messageRefQuotedField.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index 98adc38e4089..91ec07b28562 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -115,7 +115,7 @@ public InterpreterState() { "title", "description", "rule \"add\"\n" + - "when tostring(message.`message`) == \"original message\"\n" + + "when tostring($message.message) == \"original message\"\n" + "then\n" + " set_field(\"field\", \"derived message\");\n" + "end", diff --git a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 index 27c2d93ebb5b..276da69f2523 100644 --- a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 @@ -138,7 +138,7 @@ Then: T H E N; End: E N D; Let: L E T; Match: M A T C H; -MessageRef: 'message'; +MessageRef: '$message'; Boolean : 'true'|'false' diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt index 0450176dcbb8..1f46a04629c2 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt @@ -1,7 +1,7 @@ rule "jsonpath" when true then - let x = parse_json(tostring(message.`message`)); + let x = parse_json(tostring($message.message)); let new_fields = select_jsonpath(x, { author_first: "$['store']['book'][0]['author']", author_last: "$['store']['book'][-1:]['author']" diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt index 59206e7e559f..9617b38895c3 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt @@ -1,5 +1,5 @@ rule "message field ref" -when tolong(value: message.responseCode, default: 200) >= 500 +when tolong(value: $message.responseCode, default: 200) >= 500 then set_field(field: "response_category", value: "server_error"); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt index 3c74dc266fc3..087e9c3d15bf 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt @@ -1,5 +1,5 @@ rule "test" -when tostring(message.`@specialfieldname`, "empty") == "string" +when tostring($message.`@specialfieldname`, "empty") == "string" then trigger_test(); end \ No newline at end of file From 1d152edacd7f60d055fa84ae7eab9bb7c2741c9e Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 14:32:27 +0100 Subject: [PATCH 082/521] also record parse error when no arguments were give to a function that expected at least one required parameter only complain about missing required paramters, do not include the optional ones in the error message --- .../pipelineprocessor/parser/PipelineRuleParser.java | 3 +++ .../pipelineprocessor/parser/errors/WrongNumberOfArgs.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 74e32b5cb4dd..357a17d15504 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -338,6 +338,9 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { i++; } } + } else if(! params.stream().allMatch(ParameterDescriptor::optional)) { + // no parameters given but some of them are required + parseContext.addError(new WrongNumberOfArgs(ctx, function, 0)); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java index 4e1ef0e6b004..3b09652441ea 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java @@ -18,8 +18,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; +import java.util.function.Predicate; + public class WrongNumberOfArgs extends ParseError { private final Function function; private final int argCount; @@ -35,7 +38,8 @@ public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, @JsonProperty("reason") @Override public String toString() { - return "Expected " + function.descriptor().params().size() + + final Predicate optional = ParameterDescriptor::optional; + return "Expected " + function.descriptor().params().stream().filter(optional.negate()).count() + " arguments but found " + argCount + " in call to function " + function.descriptor().name() + positionString(); From c1104df9a42d81bfdde9b1a33ec869385551d256 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 18:14:24 +0100 Subject: [PATCH 083/521] EqualityExpression special cases DateTime objects Joda's DateTime insists on times not being equal if their timezones are different but both refer to the same instant. Need to use isEqual instead :grumpy: --- .../ast/expressions/EqualityExpression.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java index bf61c510428d..43272bb44879 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java @@ -17,8 +17,13 @@ package org.graylog.plugins.pipelineprocessor.ast.expressions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EqualityExpression extends BinaryExpression implements LogicalExpression { + private static final Logger log = LoggerFactory.getLogger(EqualityExpression.class); + private final boolean checkEquality; public EqualityExpression(Expression left, Expression right, boolean checkEquality) { @@ -41,16 +46,36 @@ public boolean evaluateBool(EvaluationContext context) { final Object left = this.left.evaluate(context); final Object right = this.right.evaluate(context); if (left == null) { - // TODO log error + log.warn("left expression evaluated to null, returning false: {}", this.left); return false; } - final boolean equals = left.equals(right); + final boolean equals; + // sigh: DateTime::equals takes the chronology into account, so identical instants expressed in different timezones are not equal + if (left instanceof DateTime && right instanceof DateTime) { + equals = ((DateTime) left).isEqual((DateTime) right); + } else { + equals = left.equals(right); + } + + if (log.isTraceEnabled()) { + traceEquality(left, right, equals, checkEquality); + } if (checkEquality) { return equals; } return !equals; } + private void traceEquality(Object left, + Object right, + boolean equals, + boolean checkEquality) { + log.trace(checkEquality + ? "[{}] {} == {} : {} == {}" + : "[{}] {} != {} : {} != {}", + checkEquality == equals, this.left, this.right, left, right); + } + @Override public String toString() { return left.toString() + (checkEquality ? " == " : " != ") + right.toString(); From cdb88f44f9652b4e1295faf3c005bc0a2179af5b Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 18:15:48 +0100 Subject: [PATCH 084/521] don't call toString if it isn't overridden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - this horrible thing is necessary to prevent calling the default Object::toString method - probably slow as hell, but whatever, we'll optimize later™ --- .../functions/conversion/StringConversion.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 19b537634b8a..2b9d766e2282 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -38,7 +38,16 @@ public String evaluate(FunctionArgs args, EvaluationContext context) { if (evaluated instanceof String) { return (String) evaluated; } else { - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + try { + if ((evaluated.getClass().getMethod("toString").getDeclaringClass() != Object.class)) { + return evaluated.toString(); + } else { + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + } + } catch (NoSuchMethodException ignored) { + // should never happen because toString is always there + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + } } } From c8b42a38120b1c3cf90401f7b84ba7c24a29b2db Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 18:19:25 +0100 Subject: [PATCH 085/521] add missing substring function --- .../functions/strings/Substring.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java index 9d4f65557dfa..0f550ca4b10c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import com.google.common.primitives.Ints; From f3634e25ac36edb12077a70e07edfb35787b77ee Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 16 Feb 2016 18:19:51 +0100 Subject: [PATCH 086/521] add date functions: "now", "parse_date" - add timezone handling abstract helper class - tests - tweak log levels during tests --- .../pipeline/PipelineBenchmark.java | 16 ----- .../functions/ProcessorFunctionsModule.java | 21 ++++++ .../functions/dates/Now.java | 44 ++++++++++++ .../functions/dates/ParseDate.java | 55 ++++++++++++++ .../dates/TimezoneAwareFunction.java | 72 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 38 ++++++++++ src/test/resources/log4j2-test.xml | 1 + .../pipelineprocessor/functions/dates.txt | 13 ++++ 8 files changed, 244 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index 91ec07b28562..5015082e9cc0 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -1,19 +1,3 @@ -/** - * This file is part of Graylog. - * - * Graylog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog. If not, see . - */ /** * This file is part of Graylog Pipeline Processor. * diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index ae50450a5384..432065a0593a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions; import com.google.inject.Binder; @@ -8,6 +24,8 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.Now; +import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -65,6 +83,9 @@ protected void configure() { addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); addMessageProcessorFunction(SelectJsonPath.NAME, SelectJsonPath.class); + // dates + addMessageProcessorFunction(Now.NAME, Now.class); + addMessageProcessorFunction(ParseDate.NAME, ParseDate.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java new file mode 100644 index 000000000000..c1d2856ef33b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +public class Now extends TimezoneAwareFunction { + + public static final String NAME = "now"; + + @Override + protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + return DateTime.now(timezone); + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java new file mode 100644 index 000000000000..069feac57d7f --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java @@ -0,0 +1,55 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +public class ParseDate extends TimezoneAwareFunction { + + public static final String NAME = "parse_date"; + public static final String VALUE = "value"; + public static final String PATTERN = "pattern"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of( + ParameterDescriptor.string(VALUE).build(), + ParameterDescriptor.string(PATTERN).build() + ); + } + + @Override + public DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + final String dateString = args.param(VALUE).evalRequired(args, context, String.class); + final String pattern = args.param(PATTERN).evalRequired(args, context, String.class); + final DateTimeFormatter formatter = DateTimeFormat.forPattern(pattern).withZone(timezone); + + return formatter.parseDateTime(dateString); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java new file mode 100644 index 000000000000..267a5c10218a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java @@ -0,0 +1,72 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +public abstract class TimezoneAwareFunction extends AbstractFunction { + + public static final String TIMEZONE = "timezone"; + + @Override + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { + if (name.equals(TIMEZONE)) { + final String tz = (String) super.preComputeConstantArgument(args, name, arg); + return DateTimeZone.forID(tz); + } + return super.preComputeConstantArgument(args, name, arg); + } + + @Override + public DateTime evaluate(FunctionArgs args, EvaluationContext context) { + final DateTimeZone timezone = + args.param(TIMEZONE) + .eval(args, context, DateTimeZone.class) + .orElse(DateTimeZone.UTC); + + return evaluate(args, context, timezone); + } + + protected abstract DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone); + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(getName()) + .returnType(DateTime.class) + .params(ImmutableList.builder() + .addAll(params()) + .add(ParameterDescriptor + .string(TIMEZONE, DateTimeZone.class) + .optional() + .build()) + .build()) + .build(); + } + + protected abstract String getName(); + + protected abstract ImmutableList params(); +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index c4797a03a6d9..83111021d80e 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -24,6 +24,8 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.Now; +import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -41,18 +43,25 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog2.plugin.InstantMillisProvider; import org.graylog2.plugin.Message; import org.graylog2.plugin.Tools; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.joda.time.DateTime; +import org.joda.time.DateTimeUtils; import org.junit.BeforeClass; import org.junit.Test; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; public class FunctionsSnippetsTest extends BaseParserTest { + public static final DateTime GRAYLOG_EPOCH = DateTime.parse("2010-07-30T16:03:25Z"); + @BeforeClass public static void registerFunctions() { final Map> functions = commonFunctions(); @@ -93,6 +102,9 @@ public static void registerFunctions() { functions.put(JsonParse.NAME, new JsonParse(objectMapper)); functions.put(SelectJsonPath.NAME, new SelectJsonPath(objectMapper)); + functions.put(Now.NAME, new Now()); + functions.put(ParseDate.NAME, new ParseDate()); + functionRegistry = new FunctionRegistry(functions); } @@ -151,4 +163,30 @@ public void substring() { assertThat(actionsTriggered.get()).isTrue(); } + + @Test + public void dates() { + final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH); + DateTimeUtils.setCurrentMillisProvider(clock); + + try { + final Rule rule; + try { + rule = parser.parseRule(ruleForTest()); + } catch (ParseException e) { + fail("Should not fail to parse", e); + return; + } + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message).isNotEmpty(); + assertThat(message.hasField("year")).isTrue(); + assertThat(message.getField("year")).isEqualTo(2010); + assertThat(message.getField("timezone")).isEqualTo("UTC"); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + } } diff --git a/src/test/resources/log4j2-test.xml b/src/test/resources/log4j2-test.xml index 61a7c409ae4f..9ce70dff1c81 100644 --- a/src/test/resources/log4j2-test.xml +++ b/src/test/resources/log4j2-test.xml @@ -9,6 +9,7 @@ + diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt new file mode 100644 index 000000000000..aba160f8b671 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt @@ -0,0 +1,13 @@ +// now() is fixed, test uses different millisprovider! + +rule "dates" +when + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") == parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + now("CET") == now("UTC") && + now("CET") == now() +then + trigger_test(); + let date = parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ"); + set_field("year", date.year); + set_field("timezone", tostring(date.zone)); +end \ No newline at end of file From 0eac08d84ac8d8a26ea7d74543138faadcc54f84 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 17 Feb 2016 10:33:36 +0100 Subject: [PATCH 087/521] remove unused transform --- .../plugins/pipelineprocessor/functions/strings/RegexMatch.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index c0dce7fa6d31..69fd31fc952e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -78,7 +78,7 @@ public FunctionDescriptor descriptor() { .pure(true) .returnType(RegexMatchResult.class) .params(of( - ParameterDescriptor.string(PATTERN_ARG, Pattern.class).transform(Pattern::compile).build(), + ParameterDescriptor.string(PATTERN_ARG, Pattern.class).build(), ParameterDescriptor.string(VALUE_ARG).build(), ParameterDescriptor.type(GROUP_NAMES_ARG, List.class).optional().build() )) From 0f4cd214ec03035791bf3d3afcf8dc0ef66e4515 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 17 Feb 2016 15:54:53 +0100 Subject: [PATCH 088/521] add flex_parse_date function - using Natty parser underneath, like the original Flexible Date Converter --- .../functions/ProcessorFunctionsModule.java | 2 + .../functions/dates/FlexParseDate.java | 50 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 2 + .../pipelineprocessor/functions/dates.txt | 3 +- 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 432065a0593a..f1ef6aeee85a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -24,6 +24,7 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; @@ -86,6 +87,7 @@ protected void configure() { // dates addMessageProcessorFunction(Now.NAME, Now.class); addMessageProcessorFunction(ParseDate.NAME, ParseDate.class); + addMessageProcessorFunction(FlexParseDate.NAME, FlexParseDate.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java new file mode 100644 index 000000000000..263fcb0149c4 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java @@ -0,0 +1,50 @@ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import com.joestelmach.natty.DateGroup; +import com.joestelmach.natty.Parser; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.util.List; +import java.util.Optional; + +public class FlexParseDate extends TimezoneAwareFunction { + + public static final String VALUE = "value"; + public static final String NAME = "flex_parse_date"; + public static final String DEFAULT = "default"; + + @Override + protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + final String time = args.param(VALUE).evalRequired(args, context, String.class); + final DateTimeZone timeZone = args.param(TIMEZONE).evalRequired(args, context, DateTimeZone.class); + + final List dates = new Parser(timeZone.toTimeZone()).parse(time); + if (dates.size() == 0) { + final Optional defaultTime = args.param(DEFAULT).eval(args, context, DateTime.class); + if (defaultTime.isPresent()) { + return defaultTime.get(); + } + // TODO really? this should probably throw an exception of some sort to be handled in the interpreter + return null; + } + return new DateTime(dates.get(0).getDates().get(0), timeZone); + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of( + ParameterDescriptor.string(VALUE).build(), + ParameterDescriptor.type(DEFAULT, DateTime.class).optional().build() + ); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 83111021d80e..f438025efa0a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -24,6 +24,7 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; @@ -103,6 +104,7 @@ public static void registerFunctions() { functions.put(SelectJsonPath.NAME, new SelectJsonPath(objectMapper)); functions.put(Now.NAME, new Now()); + functions.put(FlexParseDate.NAME, new FlexParseDate()); functions.put(ParseDate.NAME, new ParseDate()); functionRegistry = new FunctionRegistry(functions); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt index aba160f8b671..ead44b739e6e 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt @@ -4,7 +4,8 @@ rule "dates" when parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") == parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && now("CET") == now("UTC") && - now("CET") == now() + now("CET") == now() && + flex_parse_date(value: "30th July 2010 18:03:25 ", timezone: "CET") == parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") then trigger_test(); let date = parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ"); From c4d1d6cd9e1d2e9d0b443176f27ef608c31e6251 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 17 Feb 2016 18:15:01 +0100 Subject: [PATCH 089/521] improve performace of 'tostring' function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - special case known classes and do not use reflection to detect whether they overrider .toString() - for other classes, use a threadlocal map to cache declaring classes instead of using reflection every time - added benchmark for previous and current versions, on dev laptop these show: DeclaringClassBenchmark.oldStringConversion thrpt 10 1500711.329 ± 22682.211 ops/s DeclaringClassBenchmark.stringConversion thrpt 10 7982409.552 ± 156661.740 ops/s --- .../pipeline/DeclaringClassBenchmark.java | 291 ++++++++++++++++++ .../conversion/StringConversion.java | 41 ++- 2 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java new file mode 100644 index 000000000000..f5800b53bfaf --- /dev/null +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java @@ -0,0 +1,291 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.benchmarks.pipeline; + +import com.google.common.collect.Maps; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.joda.time.DateTime; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class DeclaringClassBenchmark { + + @State(Scope.Benchmark) + public static class BState { + public EvaluationContext context; + + private final Map, Class> declaringClassCache; + Object[] objects; + Class[] declaringClass; + int i = 0; + public FunctionArgs args; + private StringConversion conversion; + private OldStringConversion oldconversion; + + public BState() { + declaringClassCache = new HashMap<>(); + try { + declaringClass = new Class[]{ + DateTime.class.getMethod("toString").getDeclaringClass(), + HashSet.class.getMethod("toString").getDeclaringClass(), + String.class.getMethod("toString").getDeclaringClass(), + ClassLoader.class.getMethod("toString").getDeclaringClass() + }; + objects = new Object[]{ + new DateTime(), + new HashSet<>(), + "hallo", + DeclaringClassBenchmark.class.getClassLoader() + }; + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } + context = EvaluationContext.emptyContext(); + args = new FunctionArgs(new StringConversion(), Maps.newHashMap()); + conversion = new StringConversion(); + oldconversion = new OldStringConversion(); + } + + + } + + @Benchmark + public void declaringClass(Blackhole bh, BState state) throws NoSuchMethodException { + final Object obj = state.objects[state.i++ % 4]; + if ((obj.getClass().getMethod("toString").getDeclaringClass() != Object.class)) { + bh.consume(obj.toString()); + } else { + bh.consume(obj); + } + } + + @Benchmark + public void declaringClassCached(Blackhole bh, BState state) throws NoSuchMethodException { + final Object obj = state.objects[state.i % 4]; + state.i++; + Class declaringClass = state.declaringClassCache.get(obj.getClass()); + if (declaringClass == null) { + declaringClass = obj.getClass().getMethod("toString").getDeclaringClass(); + state.declaringClassCache.put(obj.getClass(), declaringClass); + } + if ((declaringClass != Object.class)) { + bh.consume(obj.toString()); + } else { + bh.consume(obj); + } + } + + @Benchmark + public void declaringClassCachedAndSwitch(Blackhole bh, BState state) throws NoSuchMethodException { + final Object obj = state.objects[state.i % 4]; + state.i++; + + if (obj instanceof DateTime) { + bh.consume(obj.toString()); + } else if (obj instanceof HashSet) { + bh.consume(obj.toString()); + } else if (obj instanceof String) { + bh.consume(obj.toString()); + } else { + Class declaringClass = state.declaringClassCache.get(obj.getClass()); + if (declaringClass == null) { + declaringClass = obj.getClass().getMethod("toString").getDeclaringClass(); + state.declaringClassCache.put(obj.getClass(), declaringClass); + } + if ((declaringClass != Object.class)) { + bh.consume(obj.toString()); + } else { + bh.consume(obj); + } + } + } + + @Benchmark + public void instanceOfSwitch(Blackhole bh, BState state) { + final Object obj = state.objects[state.i++ % 4]; + if (obj instanceof DateTime) { + bh.consume(obj.toString()); + } else if (obj instanceof HashSet) { + bh.consume(obj.toString()); + } else if (obj instanceof String) { + bh.consume(obj.toString()); + } else { + bh.consume(obj); + } + } + + @Benchmark + public void toString(Blackhole bh, BState state) throws NoSuchMethodException { + final Object obj = state.objects[state.i++ % 4]; + bh.consume(obj.toString()); + } + + @Benchmark + public void stringConversion(Blackhole bh, BState state) { + state.args.setPreComputedValue(StringConversion.VALUE, state.objects[state.i++ % 4]); + + bh.consume(state.conversion.evaluate(state.args, state.context)); + } + + @Benchmark + public void oldStringConversion(Blackhole bh, BState state) { + state.args.setPreComputedValue(StringConversion.VALUE, state.objects[state.i++ % 4]); + + bh.consume(state.oldconversion.evaluate(state.args, state.context)); + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(DeclaringClassBenchmark.class.getSimpleName()) + .warmupIterations(2) + .measurementIterations(5) + .threads(1) + .forks(1) + .build(); + + new Runner(opt).run(); + } + + @SuppressWarnings("Duplicates") + public static class StringConversion extends AbstractFunction { + + public static final String NAME = "tostring"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + // this is per-thread to save an expensive concurrent hashmap access + private final ThreadLocal, Class>> declaringClassCache; + + public StringConversion() { + declaringClassCache = new ThreadLocal, Class>>() { + @Override + protected LinkedHashMap, Class> initialValue() { + return new LinkedHashMap, Class>() { + @Override + protected boolean removeEldestEntry(Map.Entry, Class> eldest) { + return size() > 1024; + } + }; + } + }; + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); + // fast path for the most common targets + if (evaluated instanceof String + || evaluated instanceof Number + || evaluated instanceof Boolean + || evaluated instanceof DateTime) { + return evaluated.toString(); + } else { + try { + // slow path, we aren't sure that the object's class actually overrides toString() so we'll look it up. + final Class klass = evaluated.getClass(); + final LinkedHashMap, Class> classCache = declaringClassCache.get(); + + Class declaringClass = classCache.get(klass); + if (declaringClass == null) { + declaringClass = klass.getMethod("toString").getDeclaringClass(); + classCache.put(klass, declaringClass); + } + if ((declaringClass != Object.class)) { + return evaluated.toString(); + } else { + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + } + } catch (NoSuchMethodException ignored) { + // should never happen because toString is always there + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + } + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + object(VALUE).build(), + string(DEFAULT).optional().build() + )) + .build(); + } + } + + @SuppressWarnings("Duplicates") + public static class OldStringConversion extends AbstractFunction { + + public static final String NAME = "tostring"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); + if (evaluated instanceof String) { + return (String) evaluated; + } else { + try { + if ((evaluated.getClass().getMethod("toString").getDeclaringClass() != Object.class)) { + return evaluated.toString(); + } else { + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + } + } catch (NoSuchMethodException ignored) { + // should never happen because toString is always there + return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + } + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + object(VALUE).build(), + string(DEFAULT).optional().build() + )) + .build(); + } + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 2b9d766e2282..03b7ec086e1c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -20,6 +20,10 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.joda.time.DateTime; + +import java.util.LinkedHashMap; +import java.util.Map; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; @@ -32,14 +36,45 @@ public class StringConversion extends AbstractFunction { private static final String VALUE = "value"; private static final String DEFAULT = "default"; + // this is per-thread to save an expensive concurrent hashmap access + private final ThreadLocal, Class>> declaringClassCache; + + public StringConversion() { + declaringClassCache = new ThreadLocal, Class>>() { + @Override + protected LinkedHashMap, Class> initialValue() { + return new LinkedHashMap, Class>() { + @Override + protected boolean removeEldestEntry(Map.Entry, Class> eldest) { + return size() > 1024; + } + }; + } + }; + } + @Override public String evaluate(FunctionArgs args, EvaluationContext context) { final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); - if (evaluated instanceof String) { - return (String) evaluated; + // fast path for the most common targets + //noinspection Duplicates + if (evaluated instanceof String + || evaluated instanceof Number + || evaluated instanceof Boolean + || evaluated instanceof DateTime) { + return evaluated.toString(); } else { try { - if ((evaluated.getClass().getMethod("toString").getDeclaringClass() != Object.class)) { + // slow path, we aren't sure that the object's class actually overrides toString() so we'll look it up. + final Class klass = evaluated.getClass(); + final LinkedHashMap, Class> classCache = declaringClassCache.get(); + + Class declaringClass = classCache.get(klass); + if (declaringClass == null) { + declaringClass = klass.getMethod("toString").getDeclaringClass(); + classCache.put(klass, declaringClass); + } + if ((declaringClass != Object.class)) { return evaluated.toString(); } else { return args.param(DEFAULT).eval(args, context, String.class).orElse(""); From 91b55a28a3285c3ee7e2e3be7431ce48fa85800a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 17 Feb 2016 18:15:16 +0100 Subject: [PATCH 090/521] update license --- .../functions/dates/FlexParseDate.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java index 263fcb0149c4..972652e33dc1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.dates; import com.google.common.collect.ImmutableList; From 3a9f4d2d5203af9fb128f2adf8c184a7c956e64a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 17 Feb 2016 19:40:03 +0100 Subject: [PATCH 091/521] added digest functions 'md5', 'sha1', 'sha256', 'sha512' - added tests - all functions return the hex string of the digest --- .../functions/ProcessorFunctionsModule.java | 10 ++++ .../functions/hashing/MD5.java | 34 +++++++++++++ .../functions/hashing/SHA1.java | 34 +++++++++++++ .../functions/hashing/SHA256.java | 34 +++++++++++++ .../functions/hashing/SHA512.java | 34 +++++++++++++ .../hashing/SingleArgStringFunction.java | 48 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 16 +++++++ .../pipelineprocessor/functions/digests.txt | 9 ++++ 8 files changed, 219 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index f1ef6aeee85a..1916e29ee5f6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -27,6 +27,10 @@ import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; +import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -88,6 +92,12 @@ protected void configure() { addMessageProcessorFunction(Now.NAME, Now.class); addMessageProcessorFunction(ParseDate.NAME, ParseDate.class); addMessageProcessorFunction(FlexParseDate.NAME, FlexParseDate.class); + + // hash digest + addMessageProcessorFunction(MD5.NAME, MD5.class); + addMessageProcessorFunction(SHA1.NAME, SHA1.class); + addMessageProcessorFunction(SHA256.NAME, SHA256.class); + addMessageProcessorFunction(SHA512.NAME, SHA512.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java new file mode 100644 index 000000000000..bc023e958d55 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class MD5 extends SingleArgStringFunction { + + public static final String NAME = "md5"; + + @Override + protected String getDigest(String value) { + return DigestUtils.md5Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java new file mode 100644 index 000000000000..ddaaad478790 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class SHA1 extends SingleArgStringFunction { + + public static final String NAME = "sha1"; + + @Override + protected String getDigest(String value) { + return DigestUtils.sha1Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java new file mode 100644 index 000000000000..ff4bcf4dc0de --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class SHA256 extends SingleArgStringFunction { + + public static final String NAME = "sha256"; + + @Override + protected String getDigest(String value) { + return DigestUtils.sha256Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java new file mode 100644 index 000000000000..2cc38f9d29e1 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class SHA512 extends SingleArgStringFunction { + + public static final String NAME = "sha512"; + + @Override + protected String getDigest(String value) { + return DigestUtils.sha512Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java new file mode 100644 index 000000000000..a1889ba9ae4f --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java @@ -0,0 +1,48 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public abstract class SingleArgStringFunction extends AbstractFunction { + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = args.param("value").evalRequired(args, context, String.class); + return getDigest(value); + } + + protected abstract String getDigest(String value); + + protected abstract String getName(); + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(getName()) + .returnType(String.class) + .params(of( + ParameterDescriptor.string("value").build()) + ) + .build(); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index f438025efa0a..b0f8d23e250b 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -27,6 +27,10 @@ import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; +import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -107,6 +111,10 @@ public static void registerFunctions() { functions.put(FlexParseDate.NAME, new FlexParseDate()); functions.put(ParseDate.NAME, new ParseDate()); + functions.put(MD5.NAME, new MD5()); + functions.put(SHA1.NAME, new SHA1()); + functions.put(SHA256.NAME, new SHA256()); + functions.put(SHA512.NAME, new SHA512()); functionRegistry = new FunctionRegistry(functions); } @@ -191,4 +199,12 @@ public void dates() { DateTimeUtils.setCurrentMillisSystem(); } } + + @Test + public void digests() { + final Rule rule = parser.parseRule(ruleForTest()); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt new file mode 100644 index 000000000000..3221ef67396d --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt @@ -0,0 +1,9 @@ +rule "digests" +when + md5("graylog") == "6f9efb466e043b9f3635827ce446e13c" && + sha1("graylog") == "6d88bccf40bf65b911fe79d78c7af98e382f0c1a" && + sha256("graylog") == "4bbdd5a829dba09d7a7ff4c1367be7d36a017b4267d728d31bd264f63debeaa6" && + sha512("graylog") == "f6cb3a96450fb9c9174299a651333c926cd67b6f5c25d8daeede1589ffa006f4dd31da4f0625b7f281051a34c8352b3a9c1a9babf90020360e911a380b5c3f4f" +then + trigger_test(); +end \ No newline at end of file From 335f7c31f6d3c2d81bb8310e7d8f877273044633 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 09:39:43 +0100 Subject: [PATCH 092/521] add contains function, fix abbreviate width parameter type - upgrade boolean valued function in logical expressions so the user can leave out the == true/false comparison - rename SwapCase -> Swapcase to match its function name better (cosmetic change) - add string function tests - move regex tests into more proper package --- .../functions/ProcessorFunctionsModule.java | 6 ++- .../functions/strings/Abbreviate.java | 2 +- .../functions/strings/Contains.java | 40 +++++++++++++++++++ .../functions/strings/SwapCase.java | 2 +- .../parser/PipelineRuleParser.java | 21 +++++++--- .../functions/FunctionsSnippetsTest.java | 33 ++++++++++++++- .../parser/PipelineRuleParserTest.java | 13 ------ .../{parser => functions}/regexMatch.txt | 0 .../pipelineprocessor/functions/strings.txt | 20 ++++++++++ 9 files changed, 113 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java rename src/test/resources/org/graylog/plugins/pipelineprocessor/{parser => functions}/regexMatch.txt (100%) create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 1916e29ee5f6..68315dc2cf6f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -42,10 +42,11 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; -import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Swapcase; import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; import org.graylog2.plugin.PluginModule; @@ -78,9 +79,10 @@ protected void configure() { // string functions addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class); addMessageProcessorFunction(Capitalize.NAME, Capitalize.class); + addMessageProcessorFunction(Contains.NAME, Contains.class); addMessageProcessorFunction(Lowercase.NAME, Lowercase.class); addMessageProcessorFunction(Substring.NAME, Substring.class); - addMessageProcessorFunction(SwapCase.NAME, SwapCase.class); + addMessageProcessorFunction(Swapcase.NAME, Swapcase.class); addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java index 2386904caeee..55ce750b8c7c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -50,7 +50,7 @@ public FunctionDescriptor descriptor() { .returnType(String.class) .params(ImmutableList.of( ParameterDescriptor.string(VALUE).build(), - ParameterDescriptor.string(WIDTH).build() + ParameterDescriptor.integer(WIDTH).build() )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java new file mode 100644 index 000000000000..a40c51d316f4 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java @@ -0,0 +1,40 @@ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Contains extends AbstractFunction { + + public static final String NAME = "contains"; + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final String value = args.param("value").evalRequired(args, context, String.class); + final String search = args.param("search").evalRequired(args, context, String.class); + final boolean ignoreCase = args.param("ignore_case").eval(args, context, Boolean.class).orElse(false); + if (ignoreCase) { + return StringUtils.containsIgnoreCase(value, search); + } else { + return StringUtils.contains(value, search); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + ParameterDescriptor.string("value").build(), + ParameterDescriptor.string("search").build(), + ParameterDescriptor.bool("ignore_case").optional().build() + )) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java index 5e13b25a67ad..f5ead84cae44 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java @@ -21,7 +21,7 @@ import javax.annotation.Nullable; import java.util.Locale; -public class SwapCase extends StringUtilsFunction { +public class Swapcase extends StringUtilsFunction { public static final String NAME = "swapcase"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 357a17d15504..8a77e458c1ff 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -388,7 +388,7 @@ public void exitNested(RuleLangParser.NestedContext ctx) { @Override public void exitNot(RuleLangParser.NotContext ctx) { - final LogicalExpression expression = (LogicalExpression) exprs.get(ctx.expression()); + final LogicalExpression expression = upgradeBoolFunctionExpression(ctx.expression()); final NotExpression expr = new NotExpression(expression); log.info("NOT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -396,17 +396,28 @@ public void exitNot(RuleLangParser.NotContext ctx) { @Override public void exitAnd(RuleLangParser.AndContext ctx) { - final LogicalExpression left = (LogicalExpression) exprs.get(ctx.left); - final LogicalExpression right = (LogicalExpression) exprs.get(ctx.right); + // if the expressions are function calls but boolean valued, upgrade them, + // we allow testing boolean valued functions without explicit comparison operator + final LogicalExpression left = upgradeBoolFunctionExpression(ctx.left); + final LogicalExpression right = upgradeBoolFunctionExpression(ctx.right); + final AndExpression expr = new AndExpression(left, right); log.info("AND: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } + private LogicalExpression upgradeBoolFunctionExpression(RuleLangParser.ExpressionContext leftExprContext) { + Expression leftExpr = exprs.get(leftExprContext); + if (leftExpr instanceof FunctionExpression && leftExpr.getType().equals(Boolean.class)) { + leftExpr = new BooleanValuedFunctionWrapper(leftExpr); + } + return (LogicalExpression) leftExpr; + } + @Override public void exitOr(RuleLangParser.OrContext ctx) { - final LogicalExpression left = (LogicalExpression) exprs.get(ctx.left); - final LogicalExpression right = (LogicalExpression) exprs.get(ctx.right); + final LogicalExpression left = upgradeBoolFunctionExpression(ctx.left); + final LogicalExpression right = upgradeBoolFunctionExpression(ctx.right); final OrExpression expr = new OrExpression(left, right); log.info("OR: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index b0f8d23e250b..04cb8bdb04b3 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -41,10 +41,11 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; -import org.graylog.plugins.pipelineprocessor.functions.strings.SwapCase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Swapcase; import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; @@ -55,6 +56,7 @@ import org.graylog2.shared.bindings.providers.ObjectMapperProvider; import org.joda.time.DateTime; import org.joda.time.DateTimeUtils; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -62,6 +64,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class FunctionsSnippetsTest extends BaseParserTest { @@ -97,9 +101,10 @@ public static void registerFunctions() { // string functions functions.put(Abbreviate.NAME, new Abbreviate()); functions.put(Capitalize.NAME, new Capitalize()); + functions.put(Contains.NAME, new Contains()); functions.put(Lowercase.NAME, new Lowercase()); functions.put(Substring.NAME, new Substring()); - functions.put(SwapCase.NAME, new SwapCase()); + functions.put(Swapcase.NAME, new Swapcase()); functions.put(Uncapitalize.NAME, new Uncapitalize()); functions.put(Uppercase.NAME, new Uppercase()); @@ -207,4 +212,28 @@ public void digests() { assertThat(actionsTriggered.get()).isTrue(); } + + @Test + public void regexMatch() { + try { + final Rule rule = parser.parseRule(ruleForTest()); + final Message message = evaluateRule(rule); + assertNotNull(message); + assertTrue(message.hasField("matched_regex")); + assertTrue(message.hasField("group_1")); + } catch (ParseException e) { + Assert.fail("Should parse"); + } + } + + @Test + public void strings() { + final Rule rule = parser.parseRule(ruleForTest()); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("has_xyz")).isInstanceOf(Boolean.class); + assertThat((boolean)message.getField("has_xyz")).isFalse(); + } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index edd6868501e4..ea318f9edc45 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -451,19 +451,6 @@ public void indexedAccessWrongIndexType() { } } - @Test - public void regexMatch() { - try { - final Rule rule = parser.parseRule(ruleForTest()); - final Message message = evaluateRule(rule); - assertNotNull(message); - assertTrue(message.hasField("matched_regex")); - assertTrue(message.hasField("group_1")); - } catch (ParseException e) { - fail("Should parse"); - } - } - public static class CustomObject { private final String id; diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/regexMatch.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt similarity index 100% rename from src/test/resources/org/graylog/plugins/pipelineprocessor/parser/regexMatch.txt rename to src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt new file mode 100644 index 000000000000..ee74d12c70b1 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt @@ -0,0 +1,20 @@ +// various string functions +rule "string tests" +when + contains("abcdef", "bc") && + lowercase("a MIXED bag of chArs") == "a mixed bag of chars" && + uppercase("a MIXED bag of chArs") == "A MIXED BAG OF CHARS" && + swapcase("Capitalized") == "cAPITALIZED" && + capitalize("hello") == "Hello" && + capitalize("hEllo") == "HEllo" && + uncapitalize("Hello") == "hello" && + uncapitalize("HEllo") == "hEllo" && + abbreviate("", 4) == "" && + abbreviate("abcdefg", 6) == "abc..." && + abbreviate("abcdefg", 7) == "abcdefg" && + abbreviate("abcdefg", 8) == "abcdefg" && + abbreviate("abcdefg", 4) == "a..." +then + set_field("has_xyz", contains("abcdef", "xyz")); + trigger_test(); +end \ No newline at end of file From 4162ea9c59caf6088e68f5bac8f3d6c31a3c71b3 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 14:04:37 +0100 Subject: [PATCH 093/521] properly implement toString in BooleanValuedFunctionWrapper --- .../ast/expressions/BooleanValuedFunctionWrapper.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java index a744e9089271..426cb00cb58a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -48,4 +48,9 @@ public Object evaluate(EvaluationContext context) { public Class getType() { return expr.getType(); } + + @Override + public String toString() { + return expr.toString(); + } } From e9c8d08349323083c126a8b573c13f2ed10cf478 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 14:05:07 +0100 Subject: [PATCH 094/521] rethrow exception during constant argument processing --- .../plugins/pipelineprocessor/ast/functions/Function.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index 4296a70e9e92..03c77a96aa38 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -53,6 +53,7 @@ default void preprocessArgs(FunctionArgs args) { } } catch (Exception exception) { log.warn("Unable to precompute argument value for " + e.getKey(), exception); + throw exception; } } From c41dbba1b13ed9111f4c623050bb4408e84cdbc3 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 14:05:34 +0100 Subject: [PATCH 095/521] implement toip conversion and cidr matching (both IPv4/v6) --- .../functions/ProcessorFunctionsModule.java | 6 ++ .../functions/ips/CidrMatch.java | 63 +++++++++++++++++++ .../functions/ips/IpAddress.java | 25 ++++++++ .../functions/ips/IpAddressConversion.java | 37 +++++++++++ .../functions/FunctionsSnippetsTest.java | 15 +++++ .../functions/ipMatching.txt | 7 +++ 6 files changed, 153 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 68315dc2cf6f..f52134a5cce9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -31,6 +31,8 @@ import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; +import org.graylog.plugins.pipelineprocessor.functions.ips.CidrMatch; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddressConversion; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -100,6 +102,10 @@ protected void configure() { addMessageProcessorFunction(SHA1.NAME, SHA1.class); addMessageProcessorFunction(SHA256.NAME, SHA256.class); addMessageProcessorFunction(SHA512.NAME, SHA512.class); + + // ip handling + addMessageProcessorFunction(CidrMatch.NAME, CidrMatch.class); + addMessageProcessorFunction(IpAddressConversion.NAME, IpAddressConversion.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java new file mode 100644 index 000000000000..d6d3a1e2d773 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java @@ -0,0 +1,63 @@ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.jboss.netty.handler.ipfilter.CIDR; + +import java.net.UnknownHostException; +import java.util.function.Function; + +import static com.google.common.collect.ImmutableList.of; + +public class CidrMatch extends AbstractFunction { + + public static final String NAME = "cidr_match"; + public static final String CIDR_PARAM = "cidr"; + public static final StringToCIDR STRING_TO_CIDR = new StringToCIDR(); + public static final String IP = "ip"; + + @Override + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { + final Object argument = super.preComputeConstantArgument(args, name, arg); + if (CIDR_PARAM.equals(name)) { + //noinspection unchecked + final Function name1 = (Function) args.param(CIDR_PARAM).transform(); + return name1.apply(argument.toString()); + } + return argument; + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final CIDR cidr = args.param(CIDR_PARAM).evalRequired(args, context, CIDR.class); + final IpAddress ipAddress = args.param(IP).evalRequired(args, context, IpAddress.class); + return cidr.contains(ipAddress.inetAddress()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + ParameterDescriptor.string(CIDR_PARAM, CIDR.class).transform(STRING_TO_CIDR).build(), + ParameterDescriptor.type(IP, IpAddress.class).build() + )) + .build(); + } + + private static class StringToCIDR implements Function { + @Override + public CIDR apply(String s) { + try { + return CIDR.newCIDR(s); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + } + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java new file mode 100644 index 000000000000..a1c2619dae0f --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import java.net.InetAddress; + +/** + * Graylog's rule language wrapper for InetAddress. + *
    + * The purpose of this class is to guard against accidentally accessing properties which can trigger name resolutions + * and to provide a known interface to deal with IP addresses. + *
    + * Almost all of the logic is in the actual InetAddress delegate object. + */ +public class IpAddress { + + private InetAddress address; + + public IpAddress(InetAddress address) { + this.address = address; + } + + + public InetAddress inetAddress() { + return address; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java new file mode 100644 index 000000000000..a026b361e616 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -0,0 +1,37 @@ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import com.google.common.net.InetAddresses; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.net.InetAddress; + +import static com.google.common.collect.ImmutableList.of; + +public class IpAddressConversion extends AbstractFunction { + + public static final String IP = "ip"; + public static final String NAME = "toip"; + + @Override + public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { + final String ipString = args.param(IP).evalRequired(args, context, String.class); + + final InetAddress inetAddress = InetAddresses.forString(ipString); + return new IpAddress(inetAddress); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(IpAddress.class) + .params(of( + ParameterDescriptor.string(IP).build() + )) + .build(); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 04cb8bdb04b3..6cd70e334d15 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -31,6 +31,8 @@ import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; +import org.graylog.plugins.pipelineprocessor.functions.ips.CidrMatch; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddressConversion; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; @@ -120,6 +122,10 @@ public static void registerFunctions() { functions.put(SHA1.NAME, new SHA1()); functions.put(SHA256.NAME, new SHA256()); functions.put(SHA512.NAME, new SHA512()); + + functions.put(IpAddressConversion.NAME, new IpAddressConversion()); + functions.put(CidrMatch.NAME, new CidrMatch()); + functionRegistry = new FunctionRegistry(functions); } @@ -236,4 +242,13 @@ public void strings() { assertThat(message.getField("has_xyz")).isInstanceOf(Boolean.class); assertThat((boolean)message.getField("has_xyz")).isFalse(); } + + @Test + public void ipMatching() { + final Rule rule = parser.parseRule(ruleForTest()); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } + } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt new file mode 100644 index 000000000000..2cc0c423c9ce --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt @@ -0,0 +1,7 @@ +rule "ip handling" +when + cidr_match("192.0.0.0/8", toip("192.168.1.50")) && + ! cidr_match("191.0.0.0/8", toip("192.168.1.50")) +then + trigger_test(); +end \ No newline at end of file From 08f182d7ef03694f12a89704709d8d31520f6abb Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 14:06:17 +0100 Subject: [PATCH 096/521] update license headers --- .../functions/ips/CidrMatch.java | 16 ++++++++++++++++ .../functions/ips/IpAddress.java | 16 ++++++++++++++++ .../functions/ips/IpAddressConversion.java | 16 ++++++++++++++++ .../functions/strings/Contains.java | 16 ++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java index d6d3a1e2d773..e3e0cef6262c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.ips; import org.graylog.plugins.pipelineprocessor.EvaluationContext; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java index a1c2619dae0f..8efafd915689 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.ips; import java.net.InetAddress; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index a026b361e616..3d894283d4f8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.ips; import com.google.common.net.InetAddresses; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java index a40c51d316f4..2d0618c011b9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.strings; import org.apache.commons.lang3.StringUtils; From 6cf6374e744a5eaf6ce43adc73c9d2c3896a0acb Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 14:43:56 +0100 Subject: [PATCH 097/521] add "anonymize" property to IpAddress - removes last octet in IP with 0 - add IpAddress to short-circuit StringConversion classes - add tests for v4/v6 --- .../conversion/StringConversion.java | 6 ++++-- .../functions/ips/IpAddress.java | 20 ++++++++++++++++++- .../functions/FunctionsSnippetsTest.java | 5 ++++- .../functions/ipMatching.txt | 2 ++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 03b7ec086e1c..c33059090676 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddress; import org.joda.time.DateTime; import java.util.LinkedHashMap; @@ -57,13 +58,14 @@ protected boolean removeEldestEntry(Map.Entry, Class> eldest) { public String evaluate(FunctionArgs args, EvaluationContext context) { final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); // fast path for the most common targets - //noinspection Duplicates if (evaluated instanceof String || evaluated instanceof Number || evaluated instanceof Boolean - || evaluated instanceof DateTime) { + || evaluated instanceof DateTime + || evaluated instanceof IpAddress) { return evaluated.toString(); } else { + //noinspection Duplicates try { // slow path, we aren't sure that the object's class actually overrides toString() so we'll look it up. final Class klass = evaluated.getClass(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java index 8efafd915689..86a5dc29dcdd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java @@ -16,7 +16,10 @@ */ package org.graylog.plugins.pipelineprocessor.functions.ips; +import com.google.common.net.InetAddresses; + import java.net.InetAddress; +import java.net.UnknownHostException; /** * Graylog's rule language wrapper for InetAddress. @@ -34,8 +37,23 @@ public IpAddress(InetAddress address) { this.address = address; } - public InetAddress inetAddress() { return address; } + + @Override + public String toString() { + return InetAddresses.toAddrString(address); + } + + public IpAddress getAnonymized() { + final byte[] address = this.address.getAddress(); + address[address.length-1] = 0x00; + try { + return new IpAddress(InetAddress.getByAddress(address)); + } catch (UnknownHostException e) { + // cannot happen, it's created from a valid InetAddress to begin with + throw new IllegalStateException(e); + } + } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 6cd70e334d15..70a6410bf581 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -246,9 +246,12 @@ public void strings() { @Test public void ipMatching() { final Rule rule = parser.parseRule(ruleForTest()); - evaluateRule(rule); + final Message message = evaluateRule(rule); assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("ip_anon")).isEqualTo("192.168.1.0"); + assertThat(message.getField("ipv6_anon")).isEqualTo("2001:db8::"); } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt index 2cc0c423c9ce..7c21ef8dec5f 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt @@ -3,5 +3,7 @@ when cidr_match("192.0.0.0/8", toip("192.168.1.50")) && ! cidr_match("191.0.0.0/8", toip("192.168.1.50")) then + set_field("ip_anon", tostring(toip("192.168.1.20").anonymized)); + set_field("ipv6_anon", tostring(toip("2001:db8::1").anonymized)); trigger_test(); end \ No newline at end of file From 746e071ef441fd420f593d7351774e5e1c822455 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 18 Feb 2016 18:52:30 +0100 Subject: [PATCH 098/521] refactor function parameter handling to provide generic types without unchecked casts By actually using the explicit parameter descriptors we can make sure the function code stays largely clean of casts. This also enables using the .transform lambdas more effectively, making overriding getPrecomputedConstantArgs unnecessary, simply providing a transform lambda is enough now. Migrated all existing functions to the new style (wow such changes). Fixed an improper boolean for tracking nested state, when treating identifiers as field names vs variable accesses. Maybe using a sigil for them would be good after all, but for now it works ok. The toip function now accepts "Object" as parameters, as all conversion functions should do, otherwise we need to build a better type inference engine, which turns out to be pretty much impossible because of liberal usage of "object" whenever we don't know what the type is. --- .../pipeline/DeclaringClassBenchmark.java | 34 ++++++------- .../ast/functions/Function.java | 9 ++-- .../ast/functions/ParameterDescriptor.java | 13 +++-- .../functions/FromInput.java | 13 +++-- .../conversion/BooleanConversion.java | 11 +++-- .../conversion/DoubleConversion.java | 18 +++++-- .../functions/conversion/LongConversion.java | 19 ++++++-- .../conversion/StringConversion.java | 18 +++---- .../functions/dates/FlexParseDate.java | 20 +++++--- .../functions/dates/ParseDate.java | 15 ++++-- .../dates/TimezoneAwareFunction.java | 25 ++++------ .../hashing/SingleArgStringFunction.java | 11 ++++- .../functions/ips/CidrMatch.java | 47 +++++++----------- .../functions/ips/IpAddress.java | 1 + .../functions/ips/IpAddressConversion.java | 10 ++-- .../functions/json/JsonParse.java | 6 ++- .../functions/json/SelectJsonPath.java | 48 +++++++++---------- .../functions/messages/CreateMessage.java | 24 +++++++--- .../functions/messages/DropMessage.java | 21 ++++---- .../functions/messages/HasField.java | 9 +++- .../functions/messages/RemoveField.java | 9 +++- .../functions/messages/RouteToStream.java | 14 ++++-- .../functions/messages/SetField.java | 17 +++++-- .../functions/messages/SetFields.java | 9 +++- .../functions/strings/Abbreviate.java | 15 ++++-- .../functions/strings/Contains.java | 21 +++++--- .../functions/strings/RegexMatch.java | 34 ++++++------- .../strings/StringUtilsFunction.java | 17 +++++-- .../functions/strings/Substring.java | 21 +++++--- .../parser/PipelineRuleParser.java | 19 ++++---- .../functions/FunctionsSnippetsTest.java | 4 +- .../parser/PipelineRuleParserTest.java | 48 ++++++++++++------- .../functions/ipMatching.txt | 2 +- 33 files changed, 363 insertions(+), 239 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java index f5800b53bfaf..78ff72942bf9 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java @@ -21,6 +21,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.joda.time.DateTime; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Scope; @@ -154,14 +155,14 @@ public void toString(Blackhole bh, BState state) throws NoSuchMethodException { @Benchmark public void stringConversion(Blackhole bh, BState state) { - state.args.setPreComputedValue(StringConversion.VALUE, state.objects[state.i++ % 4]); + state.args.setPreComputedValue("value", state.objects[state.i++ % 4]); bh.consume(state.conversion.evaluate(state.args, state.context)); } @Benchmark public void oldStringConversion(Blackhole bh, BState state) { - state.args.setPreComputedValue(StringConversion.VALUE, state.objects[state.i++ % 4]); + state.args.setPreComputedValue("value", state.objects[state.i++ % 4]); bh.consume(state.oldconversion.evaluate(state.args, state.context)); } @@ -183,8 +184,8 @@ public static class StringConversion extends AbstractFunction { public static final String NAME = "tostring"; - private static final String VALUE = "value"; - private static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam = object("value").build(); + private final ParameterDescriptor defaultParam = string("default").optional().build(); // this is per-thread to save an expensive concurrent hashmap access private final ThreadLocal, Class>> declaringClassCache; @@ -205,7 +206,7 @@ protected boolean removeEldestEntry(Map.Entry, Class> eldest) { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); + final Object evaluated = valueParam.required(args, context); // fast path for the most common targets if (evaluated instanceof String || evaluated instanceof Number @@ -226,11 +227,11 @@ public String evaluate(FunctionArgs args, EvaluationContext context) { if ((declaringClass != Object.class)) { return evaluated.toString(); } else { - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + return defaultParam.optional(args, context).orElse(""); } } catch (NoSuchMethodException ignored) { // should never happen because toString is always there - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + return defaultParam.optional(args, context).orElse(""); } } } @@ -241,8 +242,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of( - object(VALUE).build(), - string(DEFAULT).optional().build() + valueParam, + defaultParam )) .build(); } @@ -253,12 +254,13 @@ public static class OldStringConversion extends AbstractFunction { public static final String NAME = "tostring"; - private static final String VALUE = "value"; - private static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam = object("value").build(); + private final ParameterDescriptor defaultParam = string("default").optional().build(); + @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); + final Object evaluated = valueParam.required(args, context); if (evaluated instanceof String) { return (String) evaluated; } else { @@ -266,11 +268,11 @@ public String evaluate(FunctionArgs args, EvaluationContext context) { if ((evaluated.getClass().getMethod("toString").getDeclaringClass() != Object.class)) { return evaluated.toString(); } else { - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + return defaultParam.optional(args, context).orElse(""); } } catch (NoSuchMethodException ignored) { // should never happen because toString is always there - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + return defaultParam.optional(args, context).orElse(""); } } } @@ -281,8 +283,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of( - object(VALUE).build(), - string(DEFAULT).optional().build() + valueParam, + defaultParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index 03c77a96aa38..cdd46b30a52a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -46,13 +46,16 @@ public FunctionDescriptor descriptor() { default void preprocessArgs(FunctionArgs args) { for (Map.Entry e : args.getConstantArgs().entrySet()) { + final String name = e.getKey(); try { - final Object value = preComputeConstantArgument(args, e.getKey(), e.getValue()); + final Object value = preComputeConstantArgument(args, name, e.getValue()); if (value != null) { - args.setPreComputedValue(e.getKey(), value); + //noinspection unchecked + final ParameterDescriptor param = (ParameterDescriptor) args.param(name); + args.setPreComputedValue(name, param.transform().apply(value)); } } catch (Exception exception) { - log.warn("Unable to precompute argument value for " + e.getKey(), exception); + log.warn("Unable to precompute argument value for " + name, exception); throw exception; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index b722679e5e96..0abaab4a340d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -87,23 +87,22 @@ public static Builder type(String name, Class typeClas return ParameterDescriptor.param().type(typeClass).transformedType(transformedClass).name(name); } - public Optional eval(FunctionArgs args, EvaluationContext context, Class type) { - return Optional.ofNullable(evalRequired(args, context, type)); - } - - public X evalRequired(FunctionArgs args, EvaluationContext context, Class type) { + public R required(FunctionArgs args, EvaluationContext context) { final Object precomputedValue = args.getPreComputedValue(name()); if (precomputedValue != null) { - return type.cast(transformedType().cast(precomputedValue)); + return transformedType().cast(precomputedValue); } final Expression valueExpr = args.expression(name()); if (valueExpr == null) { return null; } final Object value = valueExpr.evaluate(context); - return type.cast(transform().apply(type().cast(value))); + return transformedType().cast(transform().apply(type().cast(value))); } + public Optional optional(FunctionArgs args, EvaluationContext context) { + return Optional.ofNullable(required(args, context)); + } @AutoValue.Builder public static abstract class Builder { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java index 4a4fbb7bef05..85ab0a274a16 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog2.plugin.IOState; import org.graylog2.plugin.inputs.MessageInput; import org.graylog2.shared.inputs.InputRegistry; @@ -36,19 +37,23 @@ public class FromInput extends AbstractFunction { public static final String NAME_ARG = "name"; private final InputRegistry inputRegistry; + private final ParameterDescriptor idParam; + private final ParameterDescriptor nameParam; @Inject public FromInput(InputRegistry inputRegistry) { this.inputRegistry = inputRegistry; + idParam = string(ID_ARG).optional().build(); + nameParam = string(NAME_ARG).optional().build(); } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - String id = args.param(ID_ARG).eval(args, context, String.class).orElse(""); + String id = idParam.optional(args, context).orElse(""); MessageInput input = null; if ("".equals(id)) { - final String name = args.param(NAME_ARG).eval(args, context, String.class).orElse(""); + final String name = nameParam.optional(args, context).orElse(""); for (IOState messageInputIOState : inputRegistry.getInputStates()) { final MessageInput messageInput = messageInputIOState.getStoppable(); if (messageInput.getTitle().equalsIgnoreCase(name)) { @@ -76,8 +81,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of( - string(ID_ARG).optional().build(), - string(NAME_ARG).optional().build())) + idParam, + nameParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java index d7d47f29556c..a8081196f396 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; @@ -27,11 +28,15 @@ public class BooleanConversion extends AbstractFunction { public static final String NAME = "tobool"; - private static final String VALUE = "value"; + private final ParameterDescriptor valueParam; + + public BooleanConversion() { + valueParam = object("value", Boolean.class).transform(o -> Boolean.parseBoolean(String.valueOf(o))).build(); + } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - return args.param(VALUE).evalRequired(args, context, Boolean.class); + return valueParam.required(args, context); } @Override @@ -39,7 +44,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(of(object(VALUE, Boolean.class).transform(o -> Boolean.parseBoolean(String.valueOf(o))).build())) + .params(of(valueParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java index 6bbf04893836..90b33681ee6e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java @@ -21,11 +21,12 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.floating; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; public class DoubleConversion extends AbstractFunction { @@ -33,11 +34,18 @@ public class DoubleConversion extends AbstractFunction { private static final String VALUE = "value"; private static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public DoubleConversion() { + valueParam = object(VALUE).build(); + defaultParam = floating(DEFAULT).optional().build(); + } @Override public Double evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); - final Double defaultValue = args.param(DEFAULT).eval(args, context, Double.class).orElse(0d); + final Object evaluated = valueParam.required(args, context); + final Double defaultValue = defaultParam.optional(args, context).orElse(0d); if (evaluated == null) { return defaultValue; } @@ -51,8 +59,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Double.class) .params(of( - object(VALUE).build(), - param().optional().name(DEFAULT).type(Double.class).build() + valueParam, + defaultParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java index b3074809b090..999b3b35523f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.collect.ImmutableList.of; @@ -34,12 +35,20 @@ public class LongConversion extends AbstractFunction { private static final String VALUE = "value"; private static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public LongConversion() { + valueParam = object(VALUE).build(); + defaultParam = integer(DEFAULT).optional().build(); + } + @Override public Long evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.param(VALUE).eval(args, context, Object.class).orElse(new Object()); - final Long defaultValue = args.param(DEFAULT).eval(args, context, Long.class).orElse(0L); + final Object evaluated = valueParam.required(args, context); + final Long defaultValue = defaultParam.optional(args, context).orElse(0L); - return firstNonNull(tryParse(evaluated.toString()), defaultValue); + return firstNonNull(tryParse(String.valueOf(evaluated)), defaultValue); } @Override @@ -48,8 +57,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Long.class) .params(of( - object(VALUE).build(), - integer(DEFAULT).optional().build() + valueParam, + defaultParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index c33059090676..4d98d4eea60f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddress; import org.joda.time.DateTime; @@ -34,11 +35,10 @@ public class StringConversion extends AbstractFunction { public static final String NAME = "tostring"; - private static final String VALUE = "value"; - private static final String DEFAULT = "default"; - // this is per-thread to save an expensive concurrent hashmap access private final ThreadLocal, Class>> declaringClassCache; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; public StringConversion() { declaringClassCache = new ThreadLocal, Class>>() { @@ -52,11 +52,13 @@ protected boolean removeEldestEntry(Map.Entry, Class> eldest) { }; } }; + valueParam = object("value").build(); + defaultParam = string("default").optional().build(); } @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object evaluated = args.param(VALUE).evalRequired(args, context, Object.class); + final Object evaluated = valueParam.required(args, context); // fast path for the most common targets if (evaluated instanceof String || evaluated instanceof Number @@ -79,11 +81,11 @@ public String evaluate(FunctionArgs args, EvaluationContext context) { if ((declaringClass != Object.class)) { return evaluated.toString(); } else { - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + return defaultParam.optional(args, context).orElse(""); } } catch (NoSuchMethodException ignored) { // should never happen because toString is always there - return args.param(DEFAULT).eval(args, context, String.class).orElse(""); + return defaultParam.optional(args, context).orElse(""); } } } @@ -94,8 +96,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of( - object(VALUE).build(), - string(DEFAULT).optional().build() + valueParam, + defaultParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java index 972652e33dc1..d37315a094d1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java @@ -33,22 +33,28 @@ public class FlexParseDate extends TimezoneAwareFunction { public static final String VALUE = "value"; public static final String NAME = "flex_parse_date"; public static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public FlexParseDate() { + valueParam = ParameterDescriptor.string(VALUE).build(); + defaultParam = ParameterDescriptor.type(DEFAULT, DateTime.class).optional().build(); + } @Override protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { - final String time = args.param(VALUE).evalRequired(args, context, String.class); - final DateTimeZone timeZone = args.param(TIMEZONE).evalRequired(args, context, DateTimeZone.class); + final String time = valueParam.required(args, context); - final List dates = new Parser(timeZone.toTimeZone()).parse(time); + final List dates = new Parser(timezone.toTimeZone()).parse(time); if (dates.size() == 0) { - final Optional defaultTime = args.param(DEFAULT).eval(args, context, DateTime.class); + final Optional defaultTime = defaultParam.optional(args, context); if (defaultTime.isPresent()) { return defaultTime.get(); } // TODO really? this should probably throw an exception of some sort to be handled in the interpreter return null; } - return new DateTime(dates.get(0).getDates().get(0), timeZone); + return new DateTime(dates.get(0).getDates().get(0), timezone); } @Override @@ -59,8 +65,8 @@ protected String getName() { @Override protected ImmutableList params() { return ImmutableList.of( - ParameterDescriptor.string(VALUE).build(), - ParameterDescriptor.type(DEFAULT, DateTime.class).optional().build() + valueParam, + defaultParam ); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java index 069feac57d7f..2fc4ca9076db 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java @@ -30,6 +30,13 @@ public class ParseDate extends TimezoneAwareFunction { public static final String NAME = "parse_date"; public static final String VALUE = "value"; public static final String PATTERN = "pattern"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor patternParam; + + public ParseDate() { + valueParam = ParameterDescriptor.string(VALUE).build(); + patternParam = ParameterDescriptor.string(PATTERN).build(); + } @Override protected String getName() { @@ -39,15 +46,15 @@ protected String getName() { @Override protected ImmutableList params() { return ImmutableList.of( - ParameterDescriptor.string(VALUE).build(), - ParameterDescriptor.string(PATTERN).build() + valueParam, + patternParam ); } @Override public DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { - final String dateString = args.param(VALUE).evalRequired(args, context, String.class); - final String pattern = args.param(PATTERN).evalRequired(args, context, String.class); + final String dateString = valueParam.required(args, context); + final String pattern = patternParam.required(args, context); final DateTimeFormatter formatter = DateTimeFormat.forPattern(pattern).withZone(timezone); return formatter.parseDateTime(dateString); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java index 267a5c10218a..9f4870613f54 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; @@ -29,22 +28,19 @@ public abstract class TimezoneAwareFunction extends AbstractFunction { public static final String TIMEZONE = "timezone"; + private final ParameterDescriptor timeZoneParam; - @Override - public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { - if (name.equals(TIMEZONE)) { - final String tz = (String) super.preComputeConstantArgument(args, name, arg); - return DateTimeZone.forID(tz); - } - return super.preComputeConstantArgument(args, name, arg); + public TimezoneAwareFunction() { + timeZoneParam = ParameterDescriptor + .string(TIMEZONE, DateTimeZone.class) + .transform(DateTimeZone::forID) + .optional() + .build(); } @Override public DateTime evaluate(FunctionArgs args, EvaluationContext context) { - final DateTimeZone timezone = - args.param(TIMEZONE) - .eval(args, context, DateTimeZone.class) - .orElse(DateTimeZone.UTC); + final DateTimeZone timezone = timeZoneParam.optional(args, context).orElse(DateTimeZone.UTC); return evaluate(args, context, timezone); } @@ -58,10 +54,7 @@ public FunctionDescriptor descriptor() { .returnType(DateTime.class) .params(ImmutableList.builder() .addAll(params()) - .add(ParameterDescriptor - .string(TIMEZONE, DateTimeZone.class) - .optional() - .build()) + .add(timeZoneParam) .build()) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java index a1889ba9ae4f..c7d20326d5b0 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java @@ -25,9 +25,16 @@ import static com.google.common.collect.ImmutableList.of; public abstract class SingleArgStringFunction extends AbstractFunction { + + private final ParameterDescriptor valueParam; + + public SingleArgStringFunction() { + valueParam = ParameterDescriptor.string("value").build(); + } + @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final String value = args.param("value").evalRequired(args, context, String.class); + final String value = valueParam.required(args, context); return getDigest(value); } @@ -41,7 +48,7 @@ public FunctionDescriptor descriptor() { .name(getName()) .returnType(String.class) .params(of( - ParameterDescriptor.string("value").build()) + valueParam) ) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java index e3e0cef6262c..8690440c06df 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java @@ -17,7 +17,6 @@ package org.graylog.plugins.pipelineprocessor.functions.ips; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; @@ -25,32 +24,34 @@ import org.jboss.netty.handler.ipfilter.CIDR; import java.net.UnknownHostException; -import java.util.function.Function; import static com.google.common.collect.ImmutableList.of; public class CidrMatch extends AbstractFunction { public static final String NAME = "cidr_match"; - public static final String CIDR_PARAM = "cidr"; - public static final StringToCIDR STRING_TO_CIDR = new StringToCIDR(); public static final String IP = "ip"; - @Override - public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { - final Object argument = super.preComputeConstantArgument(args, name, arg); - if (CIDR_PARAM.equals(name)) { - //noinspection unchecked - final Function name1 = (Function) args.param(CIDR_PARAM).transform(); - return name1.apply(argument.toString()); - } - return argument; + private final ParameterDescriptor cidrParam; + private final ParameterDescriptor ipParam; + + public CidrMatch() { + // a little ugly because newCIDR throws a checked exception :( + cidrParam = ParameterDescriptor.string("cidr", CIDR.class).transform(cidrString -> { + try { + return CIDR.newCIDR(cidrString); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + }).build(); + ipParam = ParameterDescriptor.type(IP, IpAddress.class).build(); } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final CIDR cidr = args.param(CIDR_PARAM).evalRequired(args, context, CIDR.class); - final IpAddress ipAddress = args.param(IP).evalRequired(args, context, IpAddress.class); + final CIDR cidr = cidrParam.required(args, context); + final IpAddress ipAddress = ipParam.required(args, context); + return cidr.contains(ipAddress.inetAddress()); } @@ -60,20 +61,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of( - ParameterDescriptor.string(CIDR_PARAM, CIDR.class).transform(STRING_TO_CIDR).build(), - ParameterDescriptor.type(IP, IpAddress.class).build() - )) + cidrParam, + ipParam)) .build(); } - - private static class StringToCIDR implements Function { - @Override - public CIDR apply(String s) { - try { - return CIDR.newCIDR(s); - } catch (UnknownHostException e) { - throw new IllegalArgumentException(e); - } - } - } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java index 86a5dc29dcdd..ddd7943b17b8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java @@ -46,6 +46,7 @@ public String toString() { return InetAddresses.toAddrString(address); } + @SuppressWarnings("unused") public IpAddress getAnonymized() { final byte[] address = this.address.getAddress(); address[address.length-1] = 0x00; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index 3d894283d4f8..89749810380b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -29,12 +29,16 @@ public class IpAddressConversion extends AbstractFunction { - public static final String IP = "ip"; public static final String NAME = "toip"; + private final ParameterDescriptor ipParam; + + public IpAddressConversion() { + ipParam = ParameterDescriptor.object("ip").build(); + } @Override public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { - final String ipString = args.param(IP).evalRequired(args, context, String.class); + final String ipString = String.valueOf(ipParam.required(args, context)); final InetAddress inetAddress = InetAddresses.forString(ipString); return new IpAddress(inetAddress); @@ -46,7 +50,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(IpAddress.class) .params(of( - ParameterDescriptor.string(IP).build() + ipParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java index c887a7951c16..17f1056f48c9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java @@ -37,15 +37,17 @@ public class JsonParse extends AbstractFunction { public static final String NAME = "parse_json"; private final ObjectMapper objectMapper; + private final ParameterDescriptor valueParam; @Inject public JsonParse(ObjectMapper objectMapper) { this.objectMapper = objectMapper; + valueParam = ParameterDescriptor.string("value").build(); } @Override public JsonNode evaluate(FunctionArgs args, EvaluationContext context) { - final String value = args.param("value").evalRequired(args, context, String.class); + final String value = valueParam.required(args, context); try { return objectMapper.readTree(value); } catch (IOException e) { @@ -60,7 +62,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(JsonNode.class) .params(of( - ParameterDescriptor.string("value").build() + valueParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java index 9d7e1dab7981..3e64916b5c64 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -24,23 +24,24 @@ import com.jayway.jsonpath.Option; import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import javax.inject.Inject; -import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import static com.google.common.collect.ImmutableList.of; +import static java.util.stream.Collectors.toMap; public class SelectJsonPath extends AbstractFunction> { public static final String NAME = "select_jsonpath"; + private final Configuration configuration; + private final ParameterDescriptor jsonParam; + private final ParameterDescriptor, Map> pathsParam; @Inject public SelectJsonPath(ObjectMapper objectMapper) { @@ -48,30 +49,27 @@ public SelectJsonPath(ObjectMapper objectMapper) { .options(Option.SUPPRESS_EXCEPTIONS) .jsonProvider(new JacksonJsonNodeJsonProvider(objectMapper)) .build(); - } - @Override - public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { - if ("paths".equals(name)) { - final Object o = super.preComputeConstantArgument(args, name, arg); - //noinspection unchecked - final HashMap map = (HashMap) o; - return map.entrySet().stream().collect(Collectors.toMap( - Map.Entry::getKey, - stringStringEntry -> JsonPath.compile(stringStringEntry.getValue()) - )); - } - return super.preComputeConstantArgument(args, name, arg); + jsonParam = ParameterDescriptor.type("json", JsonNode.class).build(); + // sigh generics and type erasure + //noinspection unchecked + pathsParam = ParameterDescriptor.type("paths", + (Class>) new TypeLiteral>() {}.getRawType(), + (Class>) new TypeLiteral>() {}.getRawType()) + .transform(inputMap -> inputMap + .entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> JsonPath.compile(e.getValue())))) + .build(); } @Override public Map evaluate(FunctionArgs args, EvaluationContext context) { - final JsonNode json = args.param("json").evalRequired(args, context, JsonNode.class); - //noinspection unchecked - final Map paths = args.param("paths").evalRequired(args, context, Map.class); + final JsonNode json = jsonParam.required(args, context); + final Map paths = pathsParam.required(args, context); - return paths.entrySet().stream() - .collect(Collectors.toMap( + return paths + .entrySet().stream() + .collect(toMap( Map.Entry::getKey, e -> e.getValue().read(json, configuration) )); @@ -82,12 +80,10 @@ public FunctionDescriptor> descriptor() { //noinspection unchecked return FunctionDescriptor.>builder() .name(NAME) - .returnType((Class>) new TypeLiteral>() {}.getRawType()) + .returnType((Class>) new TypeLiteral>() {}.getRawType()) .params(of( - ParameterDescriptor.type("json", JsonNode.class).build(), - ParameterDescriptor - .type("paths", Map.class, Map.class) - .build() + jsonParam, + pathsParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java index 553215b8a6c2..9ccfd4e4ab41 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog2.plugin.Message; import org.graylog2.plugin.Tools; import org.joda.time.DateTime; @@ -27,8 +28,8 @@ import java.util.Optional; import static com.google.common.collect.ImmutableList.of; -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class CreateMessage extends AbstractFunction { @@ -37,16 +38,25 @@ public class CreateMessage extends AbstractFunction { private static final String MESSAGE_ARG = "message"; private static final String SOURCE_ARG = "source"; private static final String TIMESTAMP_ARG = "timestamp"; + private final ParameterDescriptor messageParam; + private final ParameterDescriptor sourceParam; + private final ParameterDescriptor timestampParam; + + public CreateMessage() { + messageParam = string(MESSAGE_ARG).optional().build(); + sourceParam = string(SOURCE_ARG).optional().build(); + timestampParam = type(TIMESTAMP_ARG, DateTime.class).optional().build(); + } @Override public Message evaluate(FunctionArgs args, EvaluationContext context) { - final Optional optMessage = args.param(MESSAGE_ARG).eval(args, context, String.class); + final Optional optMessage = messageParam.optional(args, context); final String message = optMessage.isPresent() ? optMessage.get() : context.currentMessage().getMessage(); - final Optional optSource = args.param(SOURCE_ARG).eval(args, context, String.class); + final Optional optSource = sourceParam.optional(args, context); final String source = optSource.isPresent() ? optSource.get() : context.currentMessage().getSource(); - final Optional optTimestamp = args.param(TIMESTAMP_ARG).eval(args, context, DateTime.class); + final Optional optTimestamp = timestampParam.optional(args, context); final DateTime timestamp = optTimestamp.isPresent() ? optTimestamp.get() : Tools.nowUTC(); final Message newMessage = new Message(message, source, timestamp); @@ -62,9 +72,9 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Message.class) .params(of( - string(MESSAGE_ARG).optional().build(), - string(SOURCE_ARG).optional().build(), - param().name(TIMESTAMP_ARG).type(DateTime.class).optional().build() + messageParam, + sourceParam, + timestampParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java index 8f018536b86f..8373bc030450 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java @@ -21,26 +21,25 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog2.plugin.Message; -import java.util.Optional; - -import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.param; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class DropMessage extends AbstractFunction { public static final String NAME = "drop_message"; public static final String MESSAGE_ARG = "message"; + private final ParameterDescriptor messageParam; + + public DropMessage() { + messageParam = type(MESSAGE_ARG, Message.class).optional().build(); + } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final Optional message; - if (args.isPresent("message")) { - message = args.param("message").eval(args, context, Message.class); - } else { - message = Optional.of(context.currentMessage()); - } - message.get().setFilterOut(true); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + message.setFilterOut(true); return null; } @@ -51,7 +50,7 @@ public FunctionDescriptor descriptor() { .pure(true) .returnType(Void.class) .params(ImmutableList.of( - param().type(Message.class).optional().name(MESSAGE_ARG).build() + messageParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java index 25e868a5a701..6f394428dc2b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -27,10 +27,15 @@ public class HasField extends AbstractFunction { public static final String NAME = "has_field"; public static final String FIELD = "field"; + private final ParameterDescriptor fieldParam; + + public HasField() { + fieldParam = ParameterDescriptor.string(FIELD).build(); + } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final String field = args.param(FIELD).evalRequired(args, context, String.class); + final String field = fieldParam.required(args, context); return context.currentMessage().hasField(field); } @@ -39,7 +44,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(ImmutableList.of(ParameterDescriptor.string(FIELD).build())) + .params(ImmutableList.of(fieldParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java index f683b2d33df1..cab516d78f97 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -27,10 +27,15 @@ public class RemoveField extends AbstractFunction { public static final String NAME = "remove_field"; public static final String FIELD = "field"; + private final ParameterDescriptor fieldParam; + + public RemoveField() { + fieldParam = ParameterDescriptor.string(FIELD).build(); + } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final String field = args.param(FIELD).evalRequired(args, context, String.class); + final String field = fieldParam.required(args, context); context.currentMessage().removeField(field); return null; } @@ -40,7 +45,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(ImmutableList.of(ParameterDescriptor.string(FIELD).build())) + .params(ImmutableList.of(fieldParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java index c5108454b55d..3c86855833f4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -23,6 +23,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog2.database.NotFoundException; import org.graylog2.plugin.streams.Stream; import org.graylog2.streams.StreamService; @@ -36,20 +37,25 @@ public class RouteToStream extends AbstractFunction { private static final String ID_ARG = "id"; private static final String NAME_ARG = "name"; private final StreamService streamService; + private final ParameterDescriptor nameParam; + private final ParameterDescriptor idParam; @Inject public RouteToStream(StreamService streamService) { this.streamService = streamService; streamService.loadAllEnabled(); + + nameParam = string(NAME_ARG).optional().build(); + idParam = string(ID_ARG).optional().build(); } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - String id = args.param(ID_ARG).eval(args, context, String.class).orElse(""); + String id = idParam.optional(args, context).orElse(""); final Stream stream; if ("".equals(id)) { - final String name = args.param(NAME_ARG).eval(args, context, String.class).orElse(""); + final String name = nameParam.optional(args, context).orElse(""); if ("".equals(name)) { return null; } @@ -81,8 +87,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(of( - string(NAME_ARG).optional().build(), - string(ID_ARG).optional().build())) + nameParam, + idParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index dc3a28e076f3..e7a30e9da83a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -21,6 +21,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; @@ -32,10 +33,18 @@ public class SetField extends AbstractFunction { public static final String FIELD = "field"; public static final String VALUE = "value"; + private final ParameterDescriptor fieldParam; + private final ParameterDescriptor valueParam; + + public SetField() { + fieldParam = string(FIELD).build(); + valueParam = object(VALUE).build(); + } + @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final String field = args.param(FIELD).evalRequired(args, context, String.class); - final Object value = args.param(VALUE).evalRequired(args, context, Object.class); + final String field = fieldParam.required(args, context); + final Object value = valueParam.required(args, context); if (!Strings.isNullOrEmpty(field)) { context.currentMessage().addField(field, value); @@ -48,8 +57,8 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(of(string(FIELD).build(), - object(VALUE).build())) + .params(of(fieldParam, + valueParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java index 2229fc279bde..fa43e5774028 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -30,11 +30,16 @@ public class SetFields extends AbstractFunction { public static final String NAME = "set_fields"; public static final String FIELDS = "fields"; + private final ParameterDescriptor fieldsParam; + + public SetFields() { + fieldsParam = ParameterDescriptor.type(FIELDS, Map.class).build(); + } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { //noinspection unchecked - final Map fields = args.param(FIELDS).evalRequired(args, context, Map.class); + final Map fields = fieldsParam.required(args, context); context.currentMessage().addFields(fields); return null; } @@ -45,7 +50,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(of( - ParameterDescriptor.type(FIELDS, Map.class).build() + fieldsParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java index 55ce750b8c7c..5cf394d029f3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -31,11 +31,18 @@ public class Abbreviate extends AbstractFunction { public static final String NAME = "abbreviate"; private static final String VALUE = "value"; private static final String WIDTH = "width"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor widthParam; + + public Abbreviate() { + valueParam = ParameterDescriptor.string(VALUE).build(); + widthParam = ParameterDescriptor.integer(WIDTH).build(); + } @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final String value = args.param(VALUE).evalRequired(args, context, String.class); - final Long maxWidth = args.param(WIDTH).evalRequired(args, context, Long.class); + final String value = valueParam.required(args, context); + final Long maxWidth = Math.max(widthParam.required(args, context), 4L); return StringUtils.abbreviate(value, saturatedCast(maxWidth)); } @@ -49,8 +56,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(ImmutableList.of( - ParameterDescriptor.string(VALUE).build(), - ParameterDescriptor.integer(WIDTH).build() + valueParam, + widthParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java index 2d0618c011b9..b2e68062f76f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java @@ -28,12 +28,21 @@ public class Contains extends AbstractFunction { public static final String NAME = "contains"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor searchParam; + private final ParameterDescriptor ignoreCaseParam; + + public Contains() { + valueParam = ParameterDescriptor.string("value").build(); + searchParam = ParameterDescriptor.string("search").build(); + ignoreCaseParam = ParameterDescriptor.bool("ignore_case").optional().build(); + } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - final String value = args.param("value").evalRequired(args, context, String.class); - final String search = args.param("search").evalRequired(args, context, String.class); - final boolean ignoreCase = args.param("ignore_case").eval(args, context, Boolean.class).orElse(false); + final String value = valueParam.required(args, context); + final String search = searchParam.required(args, context); + final boolean ignoreCase = ignoreCaseParam.optional(args, context).orElse(false); if (ignoreCase) { return StringUtils.containsIgnoreCase(value, search); } else { @@ -47,9 +56,9 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of( - ParameterDescriptor.string("value").build(), - ParameterDescriptor.string("search").build(), - ParameterDescriptor.bool("ignore_case").optional().build() + valueParam, + searchParam, + ignoreCaseParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index 69fd31fc952e..4ffc83cd05b9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; @@ -37,32 +36,27 @@ public class RegexMatch extends AbstractFunction { - public static final String PATTERN_ARG = "pattern"; - public static final String VALUE_ARG = "value"; - public static final String GROUP_NAMES_ARG = "group_names"; public static final String NAME = "regex"; - - @Override - public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { - final String stringValue = (String) super.preComputeConstantArgument(args, name, arg); - switch (name) { - case PATTERN_ARG: - return Pattern.compile(stringValue); - } - return stringValue; + private final ParameterDescriptor pattern; + private final ParameterDescriptor value; + private final ParameterDescriptor optionalGroupNames; + + public RegexMatch() { + pattern = ParameterDescriptor.string("pattern", Pattern.class).transform(Pattern::compile).build(); + value = ParameterDescriptor.string("value").build(); + optionalGroupNames = ParameterDescriptor.type("group_names", List.class).optional().build(); } @Override public RegexMatchResult evaluate(FunctionArgs args, EvaluationContext context) { - final Pattern regex = args.param(PATTERN_ARG).evalRequired(args, context, Pattern.class); - final String value = args.param(VALUE_ARG).evalRequired(args, context, String.class); + final Pattern regex = pattern.required(args, context); + final String value = this.value.required(args, context); if (regex == null || value == null) { throw new IllegalArgumentException(); } //noinspection unchecked final List groupNames = - (List) args.param(GROUP_NAMES_ARG).eval(args, context, List.class) - .orElse(Collections.emptyList()); + (List) optionalGroupNames.optional(args, context).orElse(Collections.emptyList()); final Matcher matcher = regex.matcher(value); final boolean matches = matcher.matches(); @@ -78,9 +72,9 @@ public FunctionDescriptor descriptor() { .pure(true) .returnType(RegexMatchResult.class) .params(of( - ParameterDescriptor.string(PATTERN_ARG, Pattern.class).build(), - ParameterDescriptor.string(VALUE_ARG).build(), - ParameterDescriptor.type(GROUP_NAMES_ARG, List.class).optional().build() + pattern, + value, + optionalGroupNames )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java index ba4df872f882..582e30e4b418 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java @@ -29,13 +29,21 @@ public abstract class StringUtilsFunction extends AbstractFunction { private static final String VALUE = "value"; private static final String LOCALE = "locale"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor localeParam; + + public StringUtilsFunction() { + valueParam = ParameterDescriptor.string(VALUE).build(); + localeParam = ParameterDescriptor.string(LOCALE, + Locale.class).optional().transform(Locale::forLanguageTag).build(); + } @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final String value = args.param(VALUE).evalRequired(args, context, String.class); + final String value = valueParam.required(args, context); Locale locale = Locale.ENGLISH; if (isLocaleAware()) { - locale = args.param(LOCALE).eval(args, context, Locale.class).orElse(Locale.ENGLISH); + locale = localeParam.optional(args, context).orElse(Locale.ENGLISH); } return apply(value, locale); } @@ -43,10 +51,9 @@ public String evaluate(FunctionArgs args, EvaluationContext context) { @Override public FunctionDescriptor descriptor() { ImmutableList.Builder params = ImmutableList.builder(); - params.add(ParameterDescriptor.string(VALUE).build()); + params.add(valueParam); if (isLocaleAware()) { - params.add(ParameterDescriptor.string(LOCALE, - Locale.class).optional().transform(Locale::forLanguageTag).build()); + params.add(localeParam); } return FunctionDescriptor.builder() .name(getName()) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java index 0f550ca4b10c..a1c8b40703ce 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java @@ -29,12 +29,21 @@ public class Substring extends AbstractFunction { public static final String NAME = "substring"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor startParam; + private final ParameterDescriptor endParam; + + public Substring() { + valueParam = ParameterDescriptor.string("value").build(); + startParam = ParameterDescriptor.integer("start").build(); + endParam = ParameterDescriptor.integer("end").optional().build(); + } @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final String value = args.param("value").evalRequired(args, context, String.class); - final int start = Ints.saturatedCast(args.param("start").evalRequired(args, context, Long.class)); - final int end = Ints.saturatedCast(args.param("end").eval(args, context, Long.class).orElse((long) value.length())); + final String value = valueParam.required(args, context); + final int start = Ints.saturatedCast(startParam.required(args, context)); + final int end = Ints.saturatedCast(endParam.optional(args, context).orElse((long) value.length())); return StringUtils.substring(value, start, end); } @@ -45,9 +54,9 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of( - ParameterDescriptor.string("value").build(), - ParameterDescriptor.integer("start").build(), - ParameterDescriptor.integer("end").optional().build() + valueParam, + startParam, + endParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 8a77e458c1ff..b7c68b7cc7d9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -86,6 +86,7 @@ import java.util.Map; import java.util.Set; import java.util.SortedSet; +import java.util.Stack; import static java.util.stream.Collectors.toList; @@ -216,13 +217,14 @@ private class RuleAstBuilder extends RuleLangBaseListener { private final Set definedVars = Sets.newHashSet(); // this is true for nested field accesses - private boolean idIsFieldAccess = false; + private Stack isIdIsFieldAccess = new Stack<>(); public RuleAstBuilder(ParseContext parseContext) { this.parseContext = parseContext; args = parseContext.arguments(); argsList = parseContext.argumentLists(); exprs = parseContext.expressions(); + isIdIsFieldAccess.push(false); // top of stack } @Override @@ -373,12 +375,12 @@ public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { @Override public void enterNested(RuleLangParser.NestedContext ctx) { // nested field access is ok, these are not rule variables - idIsFieldAccess = true; + isIdIsFieldAccess.push(true); } @Override public void exitNested(RuleLangParser.NestedContext ctx) { - idIsFieldAccess = false; // reset for error checks + isIdIsFieldAccess.pop(); // reset for error checks final Expression object = exprs.get(ctx.fieldSet); final Expression field = exprs.get(ctx.field); final FieldAccessExpression expr = new FieldAccessExpression(object, field); @@ -513,12 +515,12 @@ public void exitParenExpr(RuleLangParser.ParenExprContext ctx) { @Override public void enterMessageRef(RuleLangParser.MessageRefContext ctx) { // nested field access is ok, these are not rule variables - idIsFieldAccess = true; + isIdIsFieldAccess.push(true); } @Override public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { - idIsFieldAccess = false; // reset for error checks + isIdIsFieldAccess.pop(); // reset for error checks final Expression fieldExpr = exprs.get(ctx.field); final MessageRefExpression expr = new MessageRefExpression(fieldExpr); log.info("$MSG: ctx {} => {}", ctx, expr); @@ -530,13 +532,13 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { // unquote identifier if necessary final String identifierName = unquote(ctx.Identifier().getText(), '`'); - if (!idIsFieldAccess && !definedVars.contains(identifierName)) { + if (!isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { parseContext.addError(new UndeclaredVariable(ctx)); } final Expression expr; String type; // if the identifier is also a declared variable name prefer the variable - if (idIsFieldAccess && !definedVars.contains(identifierName)) { + if (isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { expr = new FieldRefExpression(identifierName); type = "FIELDREF"; } else { @@ -661,7 +663,8 @@ public void enterEveryRule(ParserRuleContext ctx) { sb.append(" ( "); sb.append(expression.getClass().getSimpleName()); sb.append(":").append(ctx.getClass().getSimpleName()).append(" "); - sb.append(" <").append(expression.getType().getSimpleName()).append(">"); + sb.append(" <").append(expression.getType().getSimpleName()).append("> "); + sb.append(ctx.getText()); } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 70a6410bf581..2075d50579e7 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -246,7 +246,9 @@ public void strings() { @Test public void ipMatching() { final Rule rule = parser.parseRule(ruleForTest()); - final Message message = evaluateRule(rule); + final Message in = new Message("test", "test", Tools.nowUTC()); + in.addField("ip", "192.168.1.20"); + final Message message = evaluateRule(rule, in); assertThat(actionsTriggered.get()).isTrue(); assertThat(message).isNotNull(); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index ea318f9edc45..5527da94e51c 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -111,9 +111,12 @@ public FunctionDescriptor descriptor() { } }); functions.put("one_arg", new AbstractFunction() { + + private final ParameterDescriptor one = ParameterDescriptor.string("one").build(); + @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - return args.param("one").eval(args, context, String.class).orElse(""); + return one.optional(args, context).orElse(""); } @Override @@ -121,16 +124,21 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("one_arg") .returnType(String.class) - .params(of(ParameterDescriptor.string("one").build())) + .params(of(one)) .build(); } }); functions.put("concat", new AbstractFunction() { + + private final ParameterDescriptor three = ParameterDescriptor.object("three").build(); + private final ParameterDescriptor two = ParameterDescriptor.object("two").build(); + private final ParameterDescriptor one = ParameterDescriptor.string("one").build(); + @Override public String evaluate(FunctionArgs args, EvaluationContext context) { - final Object one = args.param("one").eval(args, context, Object.class).orElse(""); - final Object two = args.param("two").eval(args, context, Object.class).orElse(""); - final Object three = args.param("three").eval(args, context, Object.class).orElse(""); + final Object one = this.one.optional(args, context).orElse(""); + final Object two = this.two.optional(args, context).orElse(""); + final Object three = this.three.optional(args, context).orElse(""); return one.toString() + two.toString() + three.toString(); } @@ -140,9 +148,9 @@ public FunctionDescriptor descriptor() { .name("concat") .returnType(String.class) .params(of( - ParameterDescriptor.string("one").build(), - ParameterDescriptor.object("two").build(), - ParameterDescriptor.object("three").build() + one, + two, + three )) .build(); } @@ -184,9 +192,12 @@ public FunctionDescriptor descriptor() { } }); functions.put("customObject", new AbstractFunction() { + + private final ParameterDescriptor aDefault = ParameterDescriptor.string("default").build(); + @Override public CustomObject evaluate(FunctionArgs args, EvaluationContext context) { - return new CustomObject(args.param("default").eval(args, context, String.class).orElse("")); + return new CustomObject(aDefault.optional(args, context).orElse("")); } @Override @@ -194,14 +205,17 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("customObject") .returnType(CustomObject.class) - .params(of(ParameterDescriptor.string("default").build())) + .params(of(aDefault)) .build(); } }); functions.put("keys", new AbstractFunction() { + + private final ParameterDescriptor map = ParameterDescriptor.type("map", Map.class).build(); + @Override public List evaluate(FunctionArgs args, EvaluationContext context) { - final Optional map = args.param("map").eval(args, context, Map.class); + final Optional map = this.map.optional(args, context); return Lists.newArrayList(map.orElse(Collections.emptyMap()).keySet()); } @@ -210,16 +224,18 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("keys") .returnType(List.class) - .params(of(ParameterDescriptor.type("map", Map.class).build())) + .params(of(map)) .build(); } }); functions.put("sort", new AbstractFunction() { + + private final ParameterDescriptor collection = ParameterDescriptor.type("collection", + Collection.class).build(); + @Override public Collection evaluate(FunctionArgs args, EvaluationContext context) { - final Collection collection = args.param("collection").eval(args, - context, - Collection.class).orElse(Collections.emptyList()); + final Collection collection = this.collection.optional(args, context).orElse(Collections.emptyList()); return Ordering.natural().sortedCopy(collection); } @@ -228,7 +244,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name("sort") .returnType(Collection.class) - .params(of(ParameterDescriptor.type("collection", Collection.class).build())) + .params(of(collection)) .build(); } }); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt index 7c21ef8dec5f..216f22365b08 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt @@ -3,7 +3,7 @@ when cidr_match("192.0.0.0/8", toip("192.168.1.50")) && ! cidr_match("191.0.0.0/8", toip("192.168.1.50")) then - set_field("ip_anon", tostring(toip("192.168.1.20").anonymized)); + set_field("ip_anon", tostring(toip($message.ip).anonymized)); set_field("ipv6_anon", tostring(toip("2001:db8::1").anonymized)); trigger_test(); end \ No newline at end of file From 695f03a8257e93715c64898fff3f82092651a846 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 19 Feb 2016 10:31:52 +0100 Subject: [PATCH 099/521] Bump Graylog server dependency to 2.0.0-alpha.4-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e743e8597ad5..d75de839a2ad 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-alpha.3-SNAPSHOT + 2.0.0-alpha.4-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From ebaf7332586188a5c308a8fe4e81f22b31802a86 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 19 Feb 2016 10:48:42 +0100 Subject: [PATCH 100/521] Filename for Swapcase class --- .../functions/strings/{SwapCase.java => Swapcase.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/{SwapCase.java => Swapcase.java} (100%) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java similarity index 100% rename from src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/SwapCase.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java From fcc7f2a82f941aefcc2897ab323106a9560d9e3b Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 19 Feb 2016 15:00:43 +0100 Subject: [PATCH 101/521] Implement format_date function --- .../functions/ProcessorFunctionsModule.java | 2 + .../functions/dates/FormatDate.java | 68 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 2 + .../pipelineprocessor/functions/dates.txt | 3 +- 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index f52134a5cce9..3d18004782b3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -25,6 +25,7 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; @@ -96,6 +97,7 @@ protected void configure() { addMessageProcessorFunction(Now.NAME, Now.class); addMessageProcessorFunction(ParseDate.NAME, ParseDate.class); addMessageProcessorFunction(FlexParseDate.NAME, FlexParseDate.class); + addMessageProcessorFunction(FormatDate.NAME, FormatDate.class); // hash digest addMessageProcessorFunction(MD5.NAME, MD5.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java new file mode 100644 index 000000000000..b23d17b816b2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java @@ -0,0 +1,68 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import static com.google.common.collect.ImmutableList.of; + +public class FormatDate extends AbstractFunction { + + public static final String NAME = "format_date"; + + private final ParameterDescriptor value; + private final ParameterDescriptor format; + private final ParameterDescriptor timeZoneParam; + + public FormatDate() { + value = ParameterDescriptor.type("value", DateTime.class).build(); + format = ParameterDescriptor.string("format", DateTimeFormatter.class) + .transform(DateTimeFormat::forPattern) + .build(); + timeZoneParam = ParameterDescriptor.string("timezone", DateTimeZone.class) + .transform(DateTimeZone::forID) + .optional() + .build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final DateTime dateTime = value.required(args, context); + final DateTimeFormatter formatter = format.required(args, context); + final DateTimeZone timeZone = timeZoneParam.optional(args, context).orElse(DateTimeZone.UTC); + + return formatter.withZone(timeZone).print(dateTime); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of(value, format, timeZoneParam)) + .build(); + } + +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 2075d50579e7..dc7cf89e5334 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -25,6 +25,7 @@ import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; @@ -117,6 +118,7 @@ public static void registerFunctions() { functions.put(Now.NAME, new Now()); functions.put(FlexParseDate.NAME, new FlexParseDate()); functions.put(ParseDate.NAME, new ParseDate()); + functions.put(FormatDate.NAME, new FormatDate()); functions.put(MD5.NAME, new MD5()); functions.put(SHA1.NAME, new SHA1()); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt index ead44b739e6e..42a3bed59653 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt @@ -5,7 +5,8 @@ when parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") == parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && now("CET") == now("UTC") && now("CET") == now() && - flex_parse_date(value: "30th July 2010 18:03:25 ", timezone: "CET") == parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") + flex_parse_date(value: "30th July 2010 18:03:25 ", timezone: "CET") == parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") && + format_date(flex_parse_date("30th July 2010 18:03:25"), "yyyy-MM-dd") == "2010-07-30" then trigger_test(); let date = parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ"); From 5ab5e7c078945be72404841a7c2144a1bb79ffd7 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 26 Feb 2016 13:25:13 +0100 Subject: [PATCH 102/521] Sync node and npm version with the server --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d75de839a2ad..d22d08dbf8cf 100644 --- a/pom.xml +++ b/pom.xml @@ -393,8 +393,8 @@ install-node-and-npm - v0.12.5 - 2.11.2 + v4.3.1 + 3.7.3 From f70fd08ef968273bc0c871273fe20d888ab3cba4 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Mon, 29 Feb 2016 11:27:47 +0100 Subject: [PATCH 103/521] Bump Graylog server dependency to 2.0.0-alpha.5-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d22d08dbf8cf..b4e5ea1557fd 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-alpha.4-SNAPSHOT + 2.0.0-alpha.5-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From a778f036578cf0ae5f741d81f13260ce2c4e1006 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 29 Feb 2016 16:40:24 +0100 Subject: [PATCH 104/521] update version and remove obsolute dependency --- benchmarks/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 1bd04208e56c..7cc4b7ce8fbc 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -51,11 +51,6 @@ THE POSSIBILITY OF SUCH DAMAGE. graylog2-server ${graylog.version} - - org.graylog2 - graylog2-plugin - ${graylog.version} - org.graylog.plugins pipeline-processor @@ -92,7 +87,7 @@ THE POSSIBILITY OF SUCH DAMAGE. 1.11.3 1.8 pipeline-benchmarks - 2.0.0-alpha.3-SNAPSHOT + 2.0.0-alpha.5-SNAPSHOT From 0e1b22c382bad5e3878a9d636ce8c142a9884dd7 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 1 Mar 2016 14:20:47 +0100 Subject: [PATCH 105/521] UI restructuring --- src/web/PipelinesPage.jsx | 20 ++++++++++++++----- src/web/index.jsx | 2 ++ src/web/rules/RulesPage.jsx | 38 +++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 src/web/rules/RulesPage.jsx diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx index c2fc910f702d..8c19bb2e30ea 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/PipelinesPage.jsx @@ -1,8 +1,12 @@ import React, {PropTypes} from 'react'; import Reflux from 'reflux'; -import { Row, Col } from 'react-bootstrap'; +import { Button, Row, Col } from 'react-bootstrap'; + +import { LinkContainer } from 'react-router-bootstrap'; + import PageHeader from 'components/common/PageHeader'; +import DocumentationLink from 'components/support/DocumentationLink'; import Spinner from 'components/common/Spinner'; import PipelineStreamComponent from 'PipelineStreamComponent'; @@ -15,6 +19,7 @@ import RulesActions from 'RulesActions'; import RulesStore from 'RulesStore'; import RulesComponent from 'RulesComponent'; +import RulesPage from 'rules/RulesPage'; const PipelinesPage = React.createClass({ mixins: [ @@ -47,15 +52,20 @@ const PipelinesPage = React.createClass({ content = ; } else { content = [ - , - , - + ]; } return ( - Processing pipelines + Pipelines define how Graylog processes data by grouping rules into stages. Pipelines can apply to all incoming messages or only to messages on a certain stream. + + Read more about Graylog pipelines in the . + + + + + {content} ); diff --git a/src/web/index.jsx b/src/web/index.jsx index 8b5793ec9216..884c692e8cd1 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -1,10 +1,12 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; import PipelinesPage from './PipelinesPage'; +import RulesPage from './rules/RulesPage'; PluginStore.register(new PluginManifest(packageJson, { routes: [ {path: '/system/pipelines', component: PipelinesPage}, + {path: '/system/pipelines/rules', component: RulesPage} ], systemnavigation: [ diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx new file mode 100644 index 000000000000..22a085cba8e7 --- /dev/null +++ b/src/web/rules/RulesPage.jsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { Row, Col } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import PageHeader from 'components/common/PageHeader'; +import DocumentationLink from 'components/support/DocumentationLink'; + +import DocsHelper from 'util/DocsHelper'; + +const RulesPage = React.createClass({ + + render() { + return ( + + + + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list of actions. + Graylog evaluates the condition against a message and executes the actions if the condition is satisfied. + + + + Read more about Graylog pipeline rules in the . + + + + + + TODO + + + + ); + }, + +}); + +export default RulesPage; \ No newline at end of file From 466b83fe63e06c4ab0ed4c0baea6a0ca16a227af Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 1 Mar 2016 18:23:30 +0100 Subject: [PATCH 106/521] Add eslint configuration --- .eslintrc | 26 ++++++++++++++++++++++++++ package.json | 6 ++++++ 2 files changed, 32 insertions(+) create mode 100644 .eslintrc diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 000000000000..270c9ac50855 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,26 @@ +{ + "parser": "babel-eslint", + "ecmaFeatures": { + "classes": true, + "jsx": true, + }, + "extends": [ + "eslint:recommended", + "airbnb", + "import/warnings" + ], + "rules": { + "no-else-return": 1, + "no-nested-ternary": 1, + "new-cap": 0, + "indent": [2, 2, { "SwitchCase" : 1}], + "import/no-unresolved": 1, + }, + "settings": { + "import/resolve": { + "extensions": [".js", ".jsx", ".ts"], + "moduleDirectory": ["src", "node_modules", "public"] + } + } +} + diff --git a/package.json b/package.json index 8858ea9e44c7..a09e5c50f765 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,12 @@ "devDependencies": { "babel-core": "^5.8.25", "babel-loader": "^5.3.2", + "eslint": "^1.5.1", + "eslint-config-airbnb": "1.0.0", + "eslint-config-import": "^0.9.1", + "eslint-loader": "^1.0.0", + "eslint-plugin-import": "^0.10.x", + "eslint-plugin-react": "^3.4.2", "graylog-web-manifests": "^2.0.0-alpha.3", "graylog-web-plugin": "~0.0.18", "json-loader": "^0.5.4", From ef478c83ece42c698ce8daad85211af8062baf92 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 1 Mar 2016 18:32:01 +0100 Subject: [PATCH 107/521] fix various NPEs and out of bounds problems --- .../ast/expressions/FieldAccessExpression.java | 2 +- .../ast/statements/FunctionStatement.java | 11 ++++++++++- .../pipelineprocessor/parser/PipelineRuleParser.java | 7 ++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java index eb3420ead146..b766a4d35b16 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -52,7 +52,7 @@ public Object evaluate(EvaluationContext context) { log.debug("[field access] property {} of bean {}: {}", fieldName, bean.getClass().getTypeName(), property); return property; } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { - log.error("Unable to read property {} from {}", fieldName, bean); + log.debug("Unable to read property {} from {}", fieldName, bean); return null; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java index 74720ca4d39a..0dc18517b46b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java @@ -18,8 +18,12 @@ import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FunctionStatement implements Statement { + private static final Logger log = LoggerFactory.getLogger(FunctionStatement.class); + private final Expression functionExpression; public FunctionStatement(Expression functionExpression) { @@ -28,7 +32,12 @@ public FunctionStatement(Expression functionExpression) { @Override public Object evaluate(EvaluationContext context) { - return functionExpression.evaluate(context); + try { + return functionExpression.evaluate(context); + } catch (Exception e) { + log.debug("Exception during statement evaluation, skipping statement", e); + return null; + } } @Override diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index b7c68b7cc7d9..693bfe0a44cf 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -182,7 +182,8 @@ public Pipeline parsePipeline(PipelineSource pipelineSource) { } public static String unquote(String string, char quoteChar) { - if (string.charAt(0) == quoteChar && string.charAt(string.length() - 1) == quoteChar) { + if (string.length() >= 2 && + string.charAt(0) == quoteChar && string.charAt(string.length() - 1) == quoteChar) { return string.substring(1, string.length() - 1); } return string; @@ -230,13 +231,13 @@ public RuleAstBuilder(ParseContext parseContext) { @Override public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { final Rule.Builder ruleBuilder = Rule.builder(); - ruleBuilder.name(unquote(ctx.name.getText(), '"')); + ruleBuilder.name(unquote(ctx.name == null ? "" : ctx.name.getText(), '"')); final Expression expr = exprs.get(ctx.condition); LogicalExpression condition; if (expr instanceof LogicalExpression) { condition = (LogicalExpression) expr; - } else if (expr.getType().equals(Boolean.class)) { + } else if (expr != null && expr.getType().equals(Boolean.class)) { condition = new BooleanValuedFunctionWrapper(expr); } else { condition = new BooleanExpression(false); From ef1ce61c801922a5a3c5db46eaaa43a3d7cd3efb Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 1 Mar 2016 18:32:26 +0100 Subject: [PATCH 108/521] properly save title and description when updating rules and pipelines --- .../plugins/pipelineprocessor/rest/PipelineResource.java | 2 ++ .../graylog/plugins/pipelineprocessor/rest/RuleResource.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index a1fab01812e8..f150704771a6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -135,6 +135,8 @@ public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final PipelineSource toSave = pipelineSource.toBuilder() + .title(update.title()) + .description(update.description()) .source(update.source()) .modifiedAt(DateTime.now()) .build(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 32f64cf82cce..ce63ffb2f764 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -136,6 +136,8 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final RuleSource toSave = ruleSource.toBuilder() + .title(update.title()) + .description(update.description()) .source(update.source()) .modifiedAt(DateTime.now()) .build(); From 9ba19f5909f0b48f11e82875cbde4de02550a4be Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 1 Mar 2016 18:32:47 +0100 Subject: [PATCH 109/521] move rules page and update to reflect new structure --- src/web/PipelinesPage.jsx | 11 +--- src/web/Rule.jsx | 55 ------------------ src/web/RulesComponent.jsx | 75 ------------------------- src/web/{ => rules}/RuleForm.jsx | 6 +- src/web/rules/RuleList.jsx | 83 ++++++++++++++++++++++++++++ src/web/{ => rules}/RulesActions.jsx | 0 src/web/rules/RulesComponent.jsx | 33 +++++++++++ src/web/rules/RulesPage.jsx | 24 +++++++- src/web/{ => rules}/RulesStore.js | 10 +++- 9 files changed, 149 insertions(+), 148 deletions(-) delete mode 100644 src/web/Rule.jsx delete mode 100644 src/web/RulesComponent.jsx rename src/web/{ => rules}/RuleForm.jsx (96%) create mode 100644 src/web/rules/RuleList.jsx rename src/web/{ => rules}/RulesActions.jsx (100%) create mode 100644 src/web/rules/RulesComponent.jsx rename src/web/{ => rules}/RulesStore.js (93%) diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx index 8c19bb2e30ea..48a0d11a1b26 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/PipelinesPage.jsx @@ -15,16 +15,9 @@ import PipelinesActions from 'PipelinesActions'; import PipelinesStore from 'PipelinesStore'; import PipelinesComponent from 'PipelinesComponent'; -import RulesActions from 'RulesActions'; -import RulesStore from 'RulesStore'; -import RulesComponent from 'RulesComponent'; - -import RulesPage from 'rules/RulesPage'; - const PipelinesPage = React.createClass({ mixins: [ Reflux.connect(PipelinesStore), - Reflux.connect(RulesStore), ], contextTypes: { storeProvider: React.PropTypes.object, @@ -32,7 +25,6 @@ const PipelinesPage = React.createClass({ getInitialState() { return { pipelines: undefined, - rules: undefined, streams: undefined, assignments: [], } @@ -40,7 +32,6 @@ const PipelinesPage = React.createClass({ componentDidMount() { PipelinesActions.list(); - RulesActions.list(); var store = this.context.storeProvider.getStore('Streams'); store.listStreams().then((streams) => this.setState({streams: streams})); @@ -48,7 +39,7 @@ const PipelinesPage = React.createClass({ render() { let content; - if (!this.state.pipelines || !this.state.rules || !this.state.streams) { + if (!this.state.pipelines || !this.state.streams) { content = ; } else { content = [ diff --git a/src/web/Rule.jsx b/src/web/Rule.jsx deleted file mode 100644 index 5d632dcc28a4..000000000000 --- a/src/web/Rule.jsx +++ /dev/null @@ -1,55 +0,0 @@ -import React, {PropTypes} from 'react'; -import {Row, Col, Button} from 'react-bootstrap'; - -import AceEditor from 'react-ace'; -import brace from 'brace'; - -import 'brace/mode/text'; -import 'brace/theme/chrome'; - -import RuleForm from 'RuleForm'; - -const Rule = React.createClass({ - propTypes: { - rule: PropTypes.object.isRequired, - }, - - _delete() { - this.props.delete(this.props.rule); - }, - - render() { - return
  • -

    {this.props.rule.title}

    - - {this.props.rule.description} - - -
    - -
    - - - - - -     -
    • ; - } -}); - -export default Rule; \ No newline at end of file diff --git a/src/web/RulesComponent.jsx b/src/web/RulesComponent.jsx deleted file mode 100644 index d5bde2b2e96a..000000000000 --- a/src/web/RulesComponent.jsx +++ /dev/null @@ -1,75 +0,0 @@ -import React, {PropTypes} from 'react'; -import Reflux from 'reflux'; -import { Row, Col } from 'react-bootstrap'; -import { Input, Alert } from 'react-bootstrap'; - -import RulesActions from 'RulesActions'; -import RuleForm from 'RuleForm'; -import Rule from 'Rule'; - -const RulesComponent = React.createClass({ - propTypes: { - rules: PropTypes.array.isRequired, - }, - - _formatRule(rule) { - return ; - }, - - _sortByTitle(rule1, rule2) { - return rule1.title.localeCompare(rule2.title); - }, - - _save(rule, callback) { - console.log(rule); - if (rule.id) { - RulesActions.update(rule); - } else { - RulesActions.save(rule); - } - callback(); - }, - - _delete(rule) { - RulesActions.delete(rule.id); - }, - - _validateRule(rule, setErrorsCb) { - RulesActions.parse(rule, setErrorsCb); - }, - - render() { - let rules; - if (this.props.rules.length == 0) { - rules = - -   No rules configured. - - } else { - rules = this.props.rules.sort(this._sortByTitle).map(this._formatRule); - } - - return ( - - -

    Rules

    -
      - {rules} -
    - - -     - ); - }, -}); - -export default RulesComponent; \ No newline at end of file diff --git a/src/web/RuleForm.jsx b/src/web/rules/RuleForm.jsx similarity index 96% rename from src/web/RuleForm.jsx rename to src/web/rules/RuleForm.jsx index 9db18d58fe65..f585c0da9f4d 100644 --- a/src/web/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -124,7 +124,7 @@ const RuleForm = React.createClass({ render() { let triggerButtonContent; if (this.props.create) { - triggerButtonContent = 'Create rule'; + triggerButtonContent = 'Add new rule'; } else { triggerButtonContent = Edit; } @@ -135,13 +135,13 @@ const RuleForm = React.createClass({ {triggerButtonContent}
    {header}; + }, + _ruleInfoFormatter(rule) { + let actions = [ + , +  , + , + ]; + + return ( + + {rule.title} + {rule.description} + + + {actions} + + ); + }, + render() { + var filterKeys = ["title", "description"]; + var headers = ["Title", "Description", "Created at", "Last modified", "Actions"]; + + return ( +
    + +
    + +
    +     +
    + ); + } +}); + +export default RuleList; diff --git a/src/web/RulesActions.jsx b/src/web/rules/RulesActions.jsx similarity index 100% rename from src/web/RulesActions.jsx rename to src/web/rules/RulesActions.jsx diff --git a/src/web/rules/RulesComponent.jsx b/src/web/rules/RulesComponent.jsx new file mode 100644 index 000000000000..d14faf6ea99a --- /dev/null +++ b/src/web/rules/RulesComponent.jsx @@ -0,0 +1,33 @@ +import React, {PropTypes} from 'react'; +import Reflux from 'reflux'; +import { Row, Col } from 'react-bootstrap'; +import { Input, Alert } from 'react-bootstrap'; + +import RulesActions from './RulesActions'; +import RuleList from './RuleList'; + +const RulesComponent = React.createClass({ + propTypes: { + rules: PropTypes.array.isRequired, + }, + + render() { + let rules; + if (this.props.rules.length == 0) { + rules = + +   No rules configured. + + } else { + rules = ; + } + + return ( +
    + {rules} +
    + ); + }, +}); + +export default RulesComponent; \ No newline at end of file diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 22a085cba8e7..999acd7d98dd 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -2,12 +2,33 @@ import React from 'react'; import { Row, Col } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; +import Reflux from 'reflux'; + import PageHeader from 'components/common/PageHeader'; import DocumentationLink from 'components/support/DocumentationLink'; import DocsHelper from 'util/DocsHelper'; +import RulesComponent from './RulesComponent'; +import RulesStore from './RulesStore'; +import RulesActions from './RulesActions'; + const RulesPage = React.createClass({ + mixins: [ + Reflux.connect(RulesStore), + ], + contextTypes: { + storeProvider: React.PropTypes.object, + }, + componentDidMount() { + RulesActions.list(); + }, + + getInitialState() { + return { + rules: [], + }; + }, render() { return ( @@ -26,13 +47,12 @@ const RulesPage = React.createClass({ - TODO + ); }, - }); export default RulesPage; \ No newline at end of file diff --git a/src/web/RulesStore.js b/src/web/rules/RulesStore.js similarity index 93% rename from src/web/RulesStore.js rename to src/web/rules/RulesStore.js index 40238ceb8c3e..37a98a229db3 100644 --- a/src/web/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -1,6 +1,6 @@ import Reflux from 'reflux'; -import RulesActions from 'RulesActions'; +import RulesActions from './RulesActions'; import UserNotification from 'util/UserNotification'; import URLUtils from 'util/URLUtils'; @@ -41,8 +41,12 @@ const RulesStore = Reflux.createStore({ source: ruleSource.source }; return fetch('POST', url, rule).then((response) => { - this.rules = response; - this.trigger({rules: response}); + if (this.rules === undefined) { + this.rules = [response]; + } else { + this.rules.push(response); + } + this.trigger({rules: this.rules}); }, failCallback); }, From ca4d079996ff16f891823b946efa4dfde7f207f9 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 1 Mar 2016 19:06:09 +0100 Subject: [PATCH 110/521] always display the DataTable to allow creating rules when the list was empty --- src/web/rules/RulesComponent.jsx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/web/rules/RulesComponent.jsx b/src/web/rules/RulesComponent.jsx index d14faf6ea99a..c02068ae9153 100644 --- a/src/web/rules/RulesComponent.jsx +++ b/src/web/rules/RulesComponent.jsx @@ -12,19 +12,9 @@ const RulesComponent = React.createClass({ }, render() { - let rules; - if (this.props.rules.length == 0) { - rules = - -   No rules configured. - - } else { - rules = ; - } - return (
    - {rules} +
    ); }, From 0f5fe35ea7e8a42b81746d95b78d1d10052469ed Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 1 Mar 2016 19:20:00 +0100 Subject: [PATCH 111/521] Use different swagger paths for pipelines and rules resources --- .../plugins/pipelineprocessor/rest/PipelineResource.java | 2 +- .../graylog/plugins/pipelineprocessor/rest/RuleResource.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index f150704771a6..5d4ddd91056d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -48,7 +48,7 @@ import java.util.Collection; @Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") -@Path("/system/pipelines") +@Path("/system/pipelines/pipelines") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class PipelineResource extends RestResource implements PluginRestResource { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index ce63ffb2f764..4ef68851e9e6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -48,7 +48,7 @@ import java.util.Collection; @Api(value = "Pipeline/Rules", description = "Rules for the pipeline message processor") -@Path("/system/pipelines") +@Path("/system/pipelines/rules") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class RuleResource extends RestResource implements PluginRestResource { From 2de3ac704956df03187b1168710afd6830e98d04 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 1 Mar 2016 19:22:27 +0100 Subject: [PATCH 112/521] Revert "Use different swagger paths for pipelines and rules resources" This reverts commit 0f5fe35ea7e8a42b81746d95b78d1d10052469ed. --- .../plugins/pipelineprocessor/rest/PipelineResource.java | 2 +- .../graylog/plugins/pipelineprocessor/rest/RuleResource.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index 5d4ddd91056d..f150704771a6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -48,7 +48,7 @@ import java.util.Collection; @Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") -@Path("/system/pipelines/pipelines") +@Path("/system/pipelines") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class PipelineResource extends RestResource implements PluginRestResource { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 4ef68851e9e6..ce63ffb2f764 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -48,7 +48,7 @@ import java.util.Collection; @Api(value = "Pipeline/Rules", description = "Rules for the pipeline message processor") -@Path("/system/pipelines/rules") +@Path("/system/pipelines") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class RuleResource extends RestResource implements PluginRestResource { From 5851f14dca65568a39c17eabc25c27ddcb036133 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 1 Mar 2016 19:38:05 +0100 Subject: [PATCH 113/521] parse pipeline source and return the stages' numbers in the response --- .../rest/PipelineResource.java | 21 +++++++++++++++++-- .../rest/PipelineSource.java | 9 ++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index f150704771a6..f7a1e80c40f1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -16,10 +16,13 @@ */ package org.graylog.plugins.pipelineprocessor.rest; +import com.google.common.collect.Lists; import com.google.common.eventbus.EventBus; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; @@ -45,7 +48,10 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; @Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") @Path("/system/pipelines") @@ -113,14 +119,25 @@ public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNu @GET @Path("/pipeline") public Collection getAll() { - return pipelineSourceService.loadAll(); + final Collection pipelineSources = pipelineSourceService.loadAll(); + final ArrayList results = Lists.newArrayList(); + for (PipelineSource p : pipelineSources) { + final Pipeline pipeline = pipelineRuleParser.parsePipeline(p); + final List stages = pipeline.stages().stream().map(Stage::stage).collect(Collectors.toList()); + results.add(p.toBuilder().stages(stages).build()); + } + + return results; } @ApiOperation(value = "Get a processing pipeline", notes = "It can take up to a second until the change is applied") @Path("/pipeline/{id}") @GET public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { - return pipelineSourceService.load(id); + final PipelineSource pipelineSource = pipelineSourceService.load(id); + final Pipeline pipeline = pipelineRuleParser.parsePipeline(pipelineSource); + final List stages = pipeline.stages().stream().map(Stage::stage).collect(Collectors.toList()); + return pipelineSource.toBuilder().stages(stages).build(); } @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied") diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java index 6b3db0381b0a..98d05f34a0a1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -25,6 +25,8 @@ import org.mongojack.ObjectId; import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; @AutoValue @JsonAutoDetect @@ -54,6 +56,10 @@ public abstract class PipelineSource { @Nullable public abstract DateTime modifiedAt(); + @JsonProperty + @Nullable + public abstract List stages(); + public static Builder builder() { return new AutoValue_PipelineSource.Builder(); } @@ -74,6 +80,7 @@ public static PipelineSource create(@Id @ObjectId @JsonProperty("_id") @Nullable .source(source) .createdAt(createdAt) .modifiedAt(modifiedAt) + .stages(Collections.emptyList()) .build(); } @@ -92,5 +99,7 @@ public abstract static class Builder { public abstract Builder createdAt(DateTime createdAt); public abstract Builder modifiedAt(DateTime modifiedAt); + + public abstract Builder stages(List stages); } } From 3c48d9bb70d5316f4878d2e816e69d7c6a59a8d7 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 10:06:27 +0100 Subject: [PATCH 114/521] Fix case in Timestamp component --- src/web/rules/RuleList.jsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index 259e9c94d5e8..b8fe6fb62022 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -1,6 +1,6 @@ import React, {PropTypes} from 'react'; import DataTable from 'components/common/DataTable'; -import TimeStamp from 'components/common/TimeStamp'; +import Timestamp from 'components/common/Timestamp'; import { Button } from 'react-bootstrap'; @@ -46,8 +46,8 @@ const RuleList = React.createClass({ {rule.title} {rule.description} - - + + {actions} ); From e256bd092a6d63e5ac3a2822cfb111bbb4d2f0cf Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 11:42:50 +0100 Subject: [PATCH 115/521] Add missing node modules to package.json --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index a09e5c50f765..a713bba58520 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "license": "GPL-3.0", "dependencies": { "brace": "^0.7.0", + "history": "^1.17.0", "react": "0.14.x", "react-ace": "^3.1.0", "react-bootstrap": "^0.28.1", @@ -28,6 +29,7 @@ "devDependencies": { "babel-core": "^5.8.25", "babel-loader": "^5.3.2", + "css-loader": "^0.23.1", "eslint": "^1.5.1", "eslint-config-airbnb": "1.0.0", "eslint-config-import": "^0.9.1", @@ -38,6 +40,7 @@ "graylog-web-plugin": "~0.0.18", "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", + "style-loader": "^0.13.0", "ts-loader": "^0.8.0", "webpack": "^1.12.2" } From 1f42d9c365638efe570dff2240d3673f33d80da7 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 11:45:22 +0100 Subject: [PATCH 116/521] Add first version of pipelines overview page --- src/web/PipelinesPage.jsx | 14 ++- src/web/index.jsx | 4 +- src/web/pipelines/PipelinesOverviewPage.jsx | 42 ++++++++ .../pipelines/ProcessingTimelineComponent.css | 18 ++++ .../pipelines/ProcessingTimelineComponent.jsx | 95 +++++++++++++++++++ src/web/rules/RulesPage.jsx | 8 +- 6 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 src/web/pipelines/PipelinesOverviewPage.jsx create mode 100644 src/web/pipelines/ProcessingTimelineComponent.css create mode 100644 src/web/pipelines/ProcessingTimelineComponent.jsx diff --git a/src/web/PipelinesPage.jsx b/src/web/PipelinesPage.jsx index 48a0d11a1b26..84674d2b0db3 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/PipelinesPage.jsx @@ -48,15 +48,21 @@ const PipelinesPage = React.createClass({ } return ( - + Pipelines define how Graylog processes data by grouping rules into stages. Pipelines can apply to all incoming messages or only to messages on a certain stream. Read more about Graylog pipelines in the . - - - + + + + + {' '} + + + + {content} ); diff --git a/src/web/index.jsx b/src/web/index.jsx index 884c692e8cd1..b01ecde7472d 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -1,12 +1,14 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; +import PipelinesOverviewPage from './pipelines/PipelinesOverviewPage'; import PipelinesPage from './PipelinesPage'; import RulesPage from './rules/RulesPage'; PluginStore.register(new PluginManifest(packageJson, { routes: [ {path: '/system/pipelines', component: PipelinesPage}, - {path: '/system/pipelines/rules', component: RulesPage} + {path: '/system/pipelines/overview', component: PipelinesOverviewPage}, + {path: '/system/pipelines/rules', component: RulesPage}, ], systemnavigation: [ diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx new file mode 100644 index 000000000000..08cf348431a9 --- /dev/null +++ b/src/web/pipelines/PipelinesOverviewPage.jsx @@ -0,0 +1,42 @@ +import React from 'react'; +import { Row, Col, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { PageHeader } from 'components/common'; +import ProcessingTimelineComponent from './ProcessingTimelineComponent'; + +const PipelinesOverviewPage = React.createClass({ + render() { + return ( +
    + + + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} + rules are evaluated and applied. Messages can go through one or more stages. + + + Click on a pipeline name to view more about it and edit its stages. + + + + + + + {' '} + + + + + + + + + + + +
    + ); + }, +}); + +export default PipelinesOverviewPage; diff --git a/src/web/pipelines/ProcessingTimelineComponent.css b/src/web/pipelines/ProcessingTimelineComponent.css new file mode 100644 index 000000000000..d3bf83d421e2 --- /dev/null +++ b/src/web/pipelines/ProcessingTimelineComponent.css @@ -0,0 +1,18 @@ +.pipeline-stage { + border: 1px solid #666; + border-radius: 4px; + display: inline-block; + margin-right: 15px; + padding: 20px; + text-align: center; + width: 120px; +} + +.pipeline-stage.idle-stage { + background-color: #E3E5E5; + border-color: #D0D4D4; +} + +.pipeline-stage.used-stage { + background-color: #FFFFFF; +} \ No newline at end of file diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx new file mode 100644 index 000000000000..13859ad43b7f --- /dev/null +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -0,0 +1,95 @@ +import React from 'react'; +import Reflux from 'reflux'; +import { Alert } from 'react-bootstrap'; + +import { DataTable, Spinner } from 'components/common'; + +import PipelinesActions from 'PipelinesActions'; +import PipelinesStore from 'PipelinesStore'; + +import {} from './ProcessingTimelineComponent.css'; + +const ProcessingTimelineComponent = React.createClass({ + mixins: [Reflux.connect(PipelinesStore)], + + componentDidMount() { + PipelinesActions.list(); + }, + + _calculateUsedStages(pipelines) { + return pipelines + .map(pipeline => pipeline.stages) + .reduce((usedStages, pipelineStages) => { + // Concat stages in a single array removing duplicates + return usedStages.concat(pipelineStages.filter(stage => usedStages.indexOf(stage) === -1)); + }, []) + .sort(); + }, + + _headerCellFormatter(header) { + const style = {}; + if (header === 'Pipeline') { + style.width = 300; + } + + return {header}; + }, + + _formatStages(pipeline, stages) { + const formattedStages = []; + + this.usedStages.forEach(usedStage => { + if (stages.indexOf(usedStage) === -1) { + formattedStages.push( +
    Idle
    + ); + } else { + formattedStages.push( +
    Stage {usedStage}
    + ); + } + }, this); + + return formattedStages; + }, + + _pipelineFormatter(pipeline) { + return ( + + {pipeline.title} + {this._formatStages(pipeline, pipeline.stages)} + + ); + }, + + render() { + if (!this.state.pipelines) { + return ; + } + + if (this.state.pipelines.length === 0) { + return ( + + There are no pipelines configured in your system. Create one to start processing your messages. + + ); + } + + this.usedStages = this._calculateUsedStages(this.state.pipelines); + + const headers = ['Pipeline', 'ProcessingTimeline']; + return ( + + ); + }, +}); + +export default ProcessingTimelineComponent; diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 999acd7d98dd..602cef6dabdc 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Row, Col } from 'react-bootstrap'; +import { Row, Col, Button } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; import Reflux from 'reflux'; @@ -43,6 +43,12 @@ const RulesPage = React.createClass({ Read more about Graylog pipeline rules in the . + + + + + + From edc5f4d280bed3ea21d5dd472a17f2612989c220 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 2 Mar 2016 12:52:42 +0100 Subject: [PATCH 117/521] split rule and pipeline DAO and REST resource classes this allows us to return more details than we are storing in the database, such as which stage numbers are being used in a pipeline, as well as parse errors --- .../pipeline/PipelineBenchmark.java | 60 +++++++------- .../pipelineprocessor/db/PipelineDao.java | 77 ++++++++++++++++++ ...ourceService.java => PipelineService.java} | 27 +++---- .../db/PipelineStreamAssignmentService.java | 4 +- .../plugins/pipelineprocessor/db/RuleDao.java | 78 +++++++++++++++++++ ...uleSourceService.java => RuleService.java} | 29 +++---- .../parser/PipelineRuleParser.java | 9 +-- .../processors/PipelineInterpreter.java | 34 ++++---- .../rest/PipelineResource.java | 50 ++++++------ .../rest/PipelineSource.java | 34 ++++++++ .../rest/PipelineStreamResource.java | 10 +-- .../pipelineprocessor/rest/RuleResource.java | 35 +++++---- .../pipelineprocessor/rest/RuleSource.java | 31 ++++++++ .../processors/PipelineInterpreterTest.java | 54 ++++++------- 14 files changed, 377 insertions(+), 155 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java rename src/main/java/org/graylog/plugins/pipelineprocessor/db/{PipelineSourceService.java => PipelineService.java} (71%) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java rename src/main/java/org/graylog/plugins/pipelineprocessor/db/{RuleSourceService.java => RuleService.java} (71%) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index 5015082e9cc0..fc0795a9be36 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -24,17 +24,17 @@ import com.google.common.eventbus.EventBus; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; -import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; -import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; -import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; -import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.Configuration; import org.graylog2.database.NotFoundException; import org.graylog2.filters.ExtractorFilter; @@ -93,29 +93,29 @@ public static class InterpreterState { private final PipelineInterpreter interpreter; public InterpreterState() { - final RuleSourceService ruleSourceService = mock(RuleSourceService.class); - when(ruleSourceService.loadAll()).thenReturn(Collections.singleton( - RuleSource.create("abc", - "title", - "description", - "rule \"add\"\n" + - "when tostring($message.message) == \"original message\"\n" + - "then\n" + - " set_field(\"field\", \"derived message\");\n" + - "end", - Tools.nowUTC(), - null) + final RuleService ruleService = mock(RuleService.class); + when(ruleService.loadAll()).thenReturn(Collections.singleton( + RuleDao.create("abc", + "title", + "description", + "rule \"add\"\n" + + "when tostring($message.message) == \"original message\"\n" + + "then\n" + + " set_field(\"field\", \"derived message\");\n" + + "end", + Tools.nowUTC(), + null) )); - final PipelineSourceService pipelineSourceService = mock(PipelineSourceService.class); - when(pipelineSourceService.loadAll()).thenReturn(Collections.singleton( - PipelineSource.create("cde", "title", "description", - "pipeline \"pipeline\"\n" + - "stage 0 match all\n" + - " rule \"add\";\n" + - "end\n", - Tools.nowUTC(), - null) + final PipelineService pipelineService = mock(PipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("cde", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"add\";\n" + + "end\n", + Tools.nowUTC(), + null) )); final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); @@ -133,8 +133,8 @@ public InterpreterState() { final PipelineRuleParser parser = setupParser(functions); interpreter = new PipelineInterpreter( - ruleSourceService, - pipelineSourceService, + ruleService, + pipelineService, pipelineStreamAssignmentService, parser, mock(Journal.class), @@ -174,7 +174,8 @@ public void setup() { final ServerStatus serverStatus = mock(ServerStatus.class); when(serverStatus.getDetailedMessageRecordingStrategy()).thenReturn(ServerStatus.MessageDetailRecordingStrategy.NEVER); - final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setDaemon(true).build()); + final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setDaemon( + true).build()); final InputService inputService = mock(InputService.class); // extractors for the single input we are pretending to have @@ -196,7 +197,8 @@ public void setup() { mock(NotificationService.class), streamService ); - ExecutorService daemonExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).build()); + ExecutorService daemonExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon( + true).build()); when(engineFactory.create(any(), any())).thenReturn( new StreamRouterEngine(Collections.emptyList(), daemonExecutor, diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java new file mode 100644 index 000000000000..5d4809211577 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java @@ -0,0 +1,77 @@ +package org.graylog.plugins.pipelineprocessor.db; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.joda.time.DateTime; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; + +@AutoValue +public abstract class PipelineDao { + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String title(); + + @JsonProperty + @Nullable + public abstract String description(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + @Nullable + public abstract DateTime createdAt(); + + @JsonProperty + @Nullable + public abstract DateTime modifiedAt(); + + public static Builder builder() { + return new AutoValue_PipelineDao.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static PipelineDao create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @Nullable @JsonProperty("created_at") DateTime createdAt, + @Nullable @JsonProperty("modified_at") DateTime modifiedAt) { + return builder() + .id(id) + .title(title) + .description(description) + .source(source) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract PipelineDao build(); + + public abstract Builder id(String id); + + public abstract Builder title(String title); + + public abstract Builder description(String description); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java similarity index 71% rename from src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java index 6efb68febe5b..8456f2ec2d7a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineSourceService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java @@ -18,11 +18,11 @@ import com.google.common.collect.Sets; import com.mongodb.MongoException; -import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; import org.mongojack.DBCursor; +import org.mongojack.DBSort; import org.mongojack.JacksonDBCollection; import org.mongojack.WriteResult; import org.slf4j.Logger; @@ -32,39 +32,40 @@ import java.util.Collection; import java.util.Collections; -public class PipelineSourceService { - private static final Logger log = LoggerFactory.getLogger(PipelineSourceService.class); +public class PipelineService { + private static final Logger log = LoggerFactory.getLogger(PipelineService.class); public static final String COLLECTION = "pipeline_processor_pipelines"; - private final JacksonDBCollection dbCollection; + private final JacksonDBCollection dbCollection; @Inject - public PipelineSourceService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + public PipelineService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { dbCollection = JacksonDBCollection.wrap( mongoConnection.getDatabase().getCollection(COLLECTION), - PipelineSource.class, + PipelineDao.class, String.class, mapper.get()); + dbCollection.createIndex(DBSort.asc("title")); } - public PipelineSource save(PipelineSource pipeline) { - final WriteResult save = dbCollection.save(pipeline); + public PipelineDao save(PipelineDao pipeline) { + final WriteResult save = dbCollection.save(pipeline); return save.getSavedObject(); } - public PipelineSource load(String id) throws NotFoundException { - final PipelineSource pipeline = dbCollection.findOneById(id); + public PipelineDao load(String id) throws NotFoundException { + final PipelineDao pipeline = dbCollection.findOneById(id); if (pipeline == null) { throw new NotFoundException("No pipeline with id " + id); } return pipeline; } - public Collection loadAll() { + public Collection loadAll() { try { - final DBCursor pipelineSources = dbCollection.find(); - return Sets.newHashSet(pipelineSources.iterator()); + final DBCursor daos = dbCollection.find(); + return Sets.newHashSet(daos.iterator()); } catch (MongoException e) { log.error("Unable to load pipelines", e); return Collections.emptySet(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java index 9ec9325c9518..f698189e795f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java @@ -17,7 +17,6 @@ package org.graylog.plugins.pipelineprocessor.db; import com.google.common.collect.Sets; -import com.mongodb.BasicDBObject; import com.mongodb.MongoException; import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; @@ -25,6 +24,7 @@ import org.graylog2.database.NotFoundException; import org.mongojack.DBCursor; import org.mongojack.DBQuery; +import org.mongojack.DBSort; import org.mongojack.JacksonDBCollection; import org.mongojack.WriteResult; import org.slf4j.Logger; @@ -48,7 +48,7 @@ public PipelineStreamAssignmentService(MongoConnection mongoConnection, MongoJac PipelineStreamAssignment.class, String.class, mapper.get()); - dbCollection.createIndex(new BasicDBObject("stream_id", 1)); + dbCollection.createIndex(DBSort.asc("stream_id")); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java new file mode 100644 index 000000000000..3e4f5ac44b8c --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java @@ -0,0 +1,78 @@ +package org.graylog.plugins.pipelineprocessor.db; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.joda.time.DateTime; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; + +@AutoValue +public abstract class RuleDao { + + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String title(); + + @JsonProperty + @Nullable + public abstract String description(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + @Nullable + public abstract DateTime createdAt(); + + @JsonProperty + @Nullable + public abstract DateTime modifiedAt(); + + public static Builder builder() { + return new AutoValue_RuleDao.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static RuleDao create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @JsonProperty("created_at") @Nullable DateTime createdAt, + @JsonProperty("modified_at") @Nullable DateTime modifiedAt) { + return builder() + .id(id) + .source(source) + .title(title) + .description(description) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract RuleDao build(); + + public abstract Builder id(String id); + + public abstract Builder title(String title); + + public abstract Builder description(String description); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java similarity index 71% rename from src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java index 26f7d5033577..bf9ca320c294 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleSourceService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java @@ -18,11 +18,11 @@ import com.google.common.collect.Sets; import com.mongodb.MongoException; -import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; import org.mongojack.DBCursor; +import org.mongojack.DBSort; import org.mongojack.JacksonDBCollection; import org.mongojack.WriteResult; import org.slf4j.Logger; @@ -32,39 +32,40 @@ import java.util.Collection; import java.util.Collections; -public class RuleSourceService { - private static final Logger log = LoggerFactory.getLogger(RuleSourceService.class); +public class RuleService { + private static final Logger log = LoggerFactory.getLogger(RuleService.class); public static final String COLLECTION = "pipeline_processor_rules"; - private final JacksonDBCollection dbCollection; + private final JacksonDBCollection dbCollection; @Inject - public RuleSourceService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + public RuleService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { dbCollection = JacksonDBCollection.wrap( mongoConnection.getDatabase().getCollection(COLLECTION), - RuleSource.class, + RuleDao.class, String.class, mapper.get()); + dbCollection.createIndex(DBSort.asc("title")); } - public RuleSource save(RuleSource rule) { - final WriteResult save = dbCollection.save(rule); + public RuleDao save(RuleDao rule) { + final WriteResult save = dbCollection.save(rule); return save.getSavedObject(); } - public RuleSource load(String id) throws NotFoundException { - final RuleSource rule = dbCollection.findOneById(id); + public RuleDao load(String id) throws NotFoundException { + final RuleDao rule = dbCollection.findOneById(id); if (rule == null) { throw new NotFoundException("No rule with id " + id); } return rule; } - public Collection loadAll() { + public Collection loadAll() { try { - final DBCursor ruleSources = dbCollection.find(); - return Sets.newHashSet(ruleSources.iterator()); + final DBCursor ruleDaos = dbCollection.find(); + return Sets.newHashSet(ruleDaos.iterator()); } catch (MongoException e) { log.error("Unable to load processing rules", e); return Collections.emptySet(); @@ -72,7 +73,7 @@ public Collection loadAll() { } public void delete(String id) { - final WriteResult result = dbCollection.removeById(id); + final WriteResult result = dbCollection.removeById(id); if (result.getN() != 1) { log.error("Unable to delete rule {}", id); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 693bfe0a44cf..b4879f800dfb 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -36,7 +36,6 @@ import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; -import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; @@ -48,6 +47,7 @@ import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; @@ -75,7 +75,6 @@ import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; import org.graylog.plugins.pipelineprocessor.parser.errors.WrongNumberOfArgs; -import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,11 +156,11 @@ public List parsePipelines(String pipelines) throws ParseException { throw new ParseException(parseContext.getErrors()); } - public Pipeline parsePipeline(PipelineSource pipelineSource) { + public Pipeline parsePipeline(String id, String source) { final ParseContext parseContext = new ParseContext(); final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); - final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(pipelineSource.source())); + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(source)); lexer.removeErrorListeners(); lexer.addErrorListener(errorListener); @@ -176,7 +175,7 @@ public Pipeline parsePipeline(PipelineSource pipelineSource) { if (parseContext.getErrors().isEmpty()) { final Pipeline pipeline = parseContext.pipelines.get(0); - return pipeline.withId(pipelineSource.id()); + return pipeline.withId(id); } throw new ParseException(parseContext.getErrors()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 22f86fc13319..0364cff45de1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -33,16 +33,16 @@ import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; -import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; -import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; -import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; @@ -73,8 +73,8 @@ public class PipelineInterpreter implements MessageProcessor { private static final Logger log = LoggerFactory.getLogger(PipelineInterpreter.class); - private final RuleSourceService ruleSourceService; - private final PipelineSourceService pipelineSourceService; + private final RuleService ruleService; + private final PipelineService pipelineService; private final PipelineStreamAssignmentService pipelineStreamAssignmentService; private final PipelineRuleParser pipelineRuleParser; private final Journal journal; @@ -85,16 +85,16 @@ public class PipelineInterpreter implements MessageProcessor { private final AtomicReference> streamPipelineAssignments = new AtomicReference<>(ImmutableSetMultimap.of()); @Inject - public PipelineInterpreter(RuleSourceService ruleSourceService, - PipelineSourceService pipelineSourceService, + public PipelineInterpreter(RuleService ruleService, + PipelineService pipelineService, PipelineStreamAssignmentService pipelineStreamAssignmentService, PipelineRuleParser pipelineRuleParser, Journal journal, MetricRegistry metricRegistry, @Named("daemonScheduler") ScheduledExecutorService scheduler, @ClusterEventBus EventBus clusterBus) { - this.ruleSourceService = ruleSourceService; - this.pipelineSourceService = pipelineSourceService; + this.ruleService = ruleService; + this.pipelineService = pipelineService; this.pipelineStreamAssignmentService = pipelineStreamAssignmentService; this.pipelineRuleParser = pipelineRuleParser; @@ -112,26 +112,26 @@ public PipelineInterpreter(RuleSourceService ruleSourceService, private synchronized void reload() { // read all rules and compile them Map ruleNameMap = Maps.newHashMap(); - for (RuleSource ruleSource : ruleSourceService.loadAll()) { + for (RuleDao ruleDao : ruleService.loadAll()) { Rule rule; try { - rule = pipelineRuleParser.parseRule(ruleSource.source()); + rule = pipelineRuleParser.parseRule(ruleDao.source()); } catch (ParseException e) { - rule = Rule.alwaysFalse("Failed to parse rule: " + ruleSource.id()); + rule = Rule.alwaysFalse("Failed to parse rule: " + ruleDao.id()); } ruleNameMap.put(rule.name(), rule); } Map pipelineIdMap = Maps.newHashMap(); // read all pipelines and compile them - for (PipelineSource pipelineSource : pipelineSourceService.loadAll()) { + for (PipelineDao pipelineDao : pipelineService.loadAll()) { Pipeline pipeline; try { - pipeline = pipelineRuleParser.parsePipeline(pipelineSource); + pipeline = pipelineRuleParser.parsePipeline(pipelineDao.id(), pipelineDao.source()); } catch (ParseException e) { - pipeline = Pipeline.empty("Failed to parse pipeline" + pipelineSource.id()); + pipeline = Pipeline.empty("Failed to parse pipeline" + pipelineDao.id()); } - pipelineIdMap.put(pipelineSource.id(), pipeline); + pipelineIdMap.put(pipelineDao.id(), pipeline); } // resolve all rules in the stages diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index f7a1e80c40f1..e94f8598e010 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -22,8 +22,8 @@ import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; -import org.graylog.plugins.pipelineprocessor.ast.Stage; -import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; @@ -50,8 +50,6 @@ import javax.ws.rs.core.Response; import java.util.ArrayList; import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; @Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") @Path("/system/pipelines") @@ -61,15 +59,15 @@ public class PipelineResource extends RestResource implements PluginRestResource private static final Logger log = LoggerFactory.getLogger(PipelineResource.class); - private final PipelineSourceService pipelineSourceService; + private final PipelineService pipelineService; private final PipelineRuleParser pipelineRuleParser; private final EventBus clusterBus; @Inject - public PipelineResource(PipelineSourceService pipelineSourceService, + public PipelineResource(PipelineService pipelineService, PipelineRuleParser pipelineRuleParser, @ClusterEventBus EventBus clusterBus) { - this.pipelineSourceService = pipelineSourceService; + this.pipelineService = pipelineService; this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; } @@ -80,21 +78,21 @@ public PipelineResource(PipelineSourceService pipelineSourceService, @Path("/pipeline") public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { try { - pipelineRuleParser.parsePipeline(pipelineSource); + final Pipeline pipeline = pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } - final PipelineSource newPipelineSource = PipelineSource.builder() + final PipelineDao pipelineDao = PipelineDao.builder() .title(pipelineSource.title()) .description(pipelineSource.description()) .source(pipelineSource.source()) .createdAt(DateTime.now()) .modifiedAt(DateTime.now()) .build(); - final PipelineSource save = pipelineSourceService.save(newPipelineSource); + final PipelineDao save = pipelineService.save(pipelineDao); clusterBus.post(PipelinesChangedEvent.updatedPipelineId(save.id())); log.info("Created new pipeline {}", save); - return save; + return PipelineSource.fromDao(pipelineRuleParser, save); } @ApiOperation(value = "Parse a processing pipeline without saving it", notes = "") @@ -102,7 +100,7 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t @Path("/pipeline/parse") public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { try { - pipelineRuleParser.parsePipeline(pipelineSource); + pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -119,12 +117,10 @@ public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNu @GET @Path("/pipeline") public Collection getAll() { - final Collection pipelineSources = pipelineSourceService.loadAll(); + final Collection daos = pipelineService.loadAll(); final ArrayList results = Lists.newArrayList(); - for (PipelineSource p : pipelineSources) { - final Pipeline pipeline = pipelineRuleParser.parsePipeline(p); - final List stages = pipeline.stages().stream().map(Stage::stage).collect(Collectors.toList()); - results.add(p.toBuilder().stages(stages).build()); + for (PipelineDao dao : daos) { + results.add(PipelineSource.fromDao(pipelineRuleParser, dao)); } return results; @@ -134,10 +130,8 @@ public Collection getAll() { @Path("/pipeline/{id}") @GET public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { - final PipelineSource pipelineSource = pipelineSourceService.load(id); - final Pipeline pipeline = pipelineRuleParser.parsePipeline(pipelineSource); - final List stages = pipeline.stages().stream().map(Stage::stage).collect(Collectors.toList()); - return pipelineSource.toBuilder().stages(stages).build(); + final PipelineDao dao = pipelineService.load(id); + return PipelineSource.fromDao(pipelineRuleParser, dao); } @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied") @@ -145,30 +139,30 @@ public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) thr @PUT public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { - final PipelineSource pipelineSource = pipelineSourceService.load(id); + final PipelineDao dao = pipelineService.load(id); try { - pipelineRuleParser.parsePipeline(update); + pipelineRuleParser.parsePipeline(update.id(), update.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } - final PipelineSource toSave = pipelineSource.toBuilder() + final PipelineDao toSave = dao.toBuilder() .title(update.title()) .description(update.description()) .source(update.source()) .modifiedAt(DateTime.now()) .build(); - final PipelineSource savedPipeline = pipelineSourceService.save(toSave); + final PipelineDao savedPipeline = pipelineService.save(toSave); clusterBus.post(PipelinesChangedEvent.updatedPipelineId(savedPipeline.id())); - return savedPipeline; + return PipelineSource.fromDao(pipelineRuleParser, savedPipeline); } @ApiOperation(value = "Delete a processing pipeline", notes = "It can take up to a second until the change is applied") @Path("/pipeline/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { - pipelineSourceService.load(id); - pipelineSourceService.delete(id); + pipelineService.load(id); + pipelineService.delete(id); clusterBus.post(PipelinesChangedEvent.deletedPipelineId(id)); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java index 98d05f34a0a1..680869190d9b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -20,6 +20,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; import org.joda.time.DateTime; import org.mongojack.Id; import org.mongojack.ObjectId; @@ -27,6 +33,8 @@ import javax.annotation.Nullable; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; @AutoValue @JsonAutoDetect @@ -60,6 +68,10 @@ public abstract class PipelineSource { @Nullable public abstract List stages(); + @JsonProperty + @Nullable + public abstract Set errors(); + public static Builder builder() { return new AutoValue_PipelineSource.Builder(); } @@ -84,6 +96,26 @@ public static PipelineSource create(@Id @ObjectId @JsonProperty("_id") @Nullable .build(); } + public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao) { + Set errors = null; + Pipeline pipeline = null; + try { + pipeline = parser.parsePipeline(dao.id(), dao.source()); + } catch (ParseException e) { + errors = e.getErrors(); + } + return builder() + .id(dao.id()) + .title(dao.title()) + .description(dao.description()) + .source(dao.source()) + .createdAt(dao.createdAt()) + .modifiedAt(dao.modifiedAt()) + .stages(pipeline == null ? null : pipeline.stages().stream().map(Stage::stage).collect(Collectors.toList())) + .errors(errors) + .build(); + } + @AutoValue.Builder public abstract static class Builder { public abstract PipelineSource build(); @@ -101,5 +133,7 @@ public abstract static class Builder { public abstract Builder modifiedAt(DateTime modifiedAt); public abstract Builder stages(List stages); + + public abstract Builder errors(Set errors); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java index abfa6c112163..3157326e5e93 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java @@ -20,7 +20,7 @@ import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; -import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; import org.graylog2.database.NotFoundException; import org.graylog2.events.ClusterEventBus; @@ -46,17 +46,17 @@ public class PipelineStreamResource extends RestResource implements PluginRestResource { private final PipelineStreamAssignmentService assignmentService; - private final PipelineSourceService pipelineSourceService; + private final PipelineService pipelineService; private final StreamService streamService; private final EventBus clusterBus; @Inject public PipelineStreamResource(PipelineStreamAssignmentService assignmentService, - PipelineSourceService pipelineSourceService, + PipelineService pipelineService, StreamService streamService, @ClusterEventBus EventBus clusterBus) { this.assignmentService = assignmentService; - this.pipelineSourceService = pipelineSourceService; + this.pipelineService = pipelineService; this.streamService = streamService; this.clusterBus = clusterBus; } @@ -71,7 +71,7 @@ public PipelineStreamAssignment assignPipelines(@ApiParam(name = "Json body", re } // verify the pipelines exist for (String s : assignment.pipelineIds()) { - pipelineSourceService.load(s); + pipelineService.load(s); } final PipelineStreamAssignment save = assignmentService.save(assignment); clusterBus.post(save); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index ce63ffb2f764..33e9b16d05cb 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -20,7 +20,8 @@ import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; -import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; @@ -46,6 +47,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.Collection; +import java.util.stream.Collectors; @Api(value = "Pipeline/Rules", description = "Rules for the pipeline message processor") @Path("/system/pipelines") @@ -55,15 +57,15 @@ public class RuleResource extends RestResource implements PluginRestResource { private static final Logger log = LoggerFactory.getLogger(RuleResource.class); - private final RuleSourceService ruleSourceService; + private final RuleService ruleService; private final PipelineRuleParser pipelineRuleParser; private final EventBus clusterBus; @Inject - public RuleResource(RuleSourceService ruleSourceService, + public RuleResource(RuleService ruleService, PipelineRuleParser pipelineRuleParser, @ClusterEventBus EventBus clusterBus) { - this.ruleSourceService = ruleSourceService; + this.ruleService = ruleService; this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; } @@ -78,18 +80,18 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } - final RuleSource newRuleSource = RuleSource.builder() + final RuleDao newRuleSource = RuleDao.builder() .title(ruleSource.title()) .description(ruleSource.description()) .source(ruleSource.source()) .createdAt(DateTime.now()) .modifiedAt(DateTime.now()) .build(); - final RuleSource save = ruleSourceService.save(newRuleSource); + final RuleDao save = ruleService.save(newRuleSource); // TODO determine which pipelines could change because of this new rule (there could be pipelines referring to a previously unresolved rule) clusterBus.post(RulesChangedEvent.updatedRuleId(save.id())); log.info("Created new rule {}", save); - return save; + return RuleSource.fromDao(pipelineRuleParser, save); } @ApiOperation(value = "Parse a processing rule without saving it", notes = "") @@ -114,14 +116,17 @@ public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleS @GET @Path("/rule") public Collection getAll() { - return ruleSourceService.loadAll(); + final Collection ruleDaos = ruleService.loadAll(); + return ruleDaos.stream() + .map(ruleDao -> RuleSource.fromDao(pipelineRuleParser, ruleDao)) + .collect(Collectors.toList()); } @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") @Path("/rule/{id}") @GET public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { - return ruleSourceService.load(id); + return RuleSource.fromDao(pipelineRuleParser, ruleService.load(id)); } @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") @@ -129,32 +134,32 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws @PUT public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { - final RuleSource ruleSource = ruleSourceService.load(id); + final RuleDao ruleDao = ruleService.load(id); try { pipelineRuleParser.parseRule(update.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } - final RuleSource toSave = ruleSource.toBuilder() + final RuleDao toSave = ruleDao.toBuilder() .title(update.title()) .description(update.description()) .source(update.source()) .modifiedAt(DateTime.now()) .build(); - final RuleSource savedRule = ruleSourceService.save(toSave); + final RuleDao savedRule = ruleService.save(toSave); // TODO determine which pipelines could change because of this updated rule clusterBus.post(RulesChangedEvent.updatedRuleId(savedRule.id())); - return savedRule; + return RuleSource.fromDao(pipelineRuleParser, savedRule); } @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") @Path("/rule/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { - ruleSourceService.load(id); - ruleSourceService.delete(id); + ruleService.load(id); + ruleService.delete(id); // TODO determine which pipelines could change because of this deleted rule, causing them to recompile clusterBus.post(RulesChangedEvent.deletedRuleId(id)); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index 2b32a833ad7d..e42158ad1b34 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -20,11 +20,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; import org.joda.time.DateTime; import org.mongojack.Id; import org.mongojack.ObjectId; import javax.annotation.Nullable; +import java.util.Set; @AutoValue @JsonAutoDetect @@ -54,6 +59,10 @@ public abstract class RuleSource { @Nullable public abstract DateTime modifiedAt(); + @JsonProperty + @Nullable + public abstract Set errors(); + public static Builder builder() { return new AutoValue_RuleSource.Builder(); } @@ -77,6 +86,26 @@ public static RuleSource create(@Id @ObjectId @JsonProperty("_id") @Nullable Str .build(); } + public static RuleSource fromDao(PipelineRuleParser parser, RuleDao dao) { + Set errors = null; + try { + parser.parseRule(dao.source()); + + } catch (ParseException e) { + errors = e.getErrors(); + } + + return builder() + .id(dao.id()) + .source(dao.source()) + .title(dao.title()) + .description(dao.description()) + .createdAt(dao.createdAt()) + .modifiedAt(dao.modifiedAt()) + .errors(errors) + .build(); + } + @AutoValue.Builder public abstract static class Builder { public abstract RuleSource build(); @@ -92,5 +121,7 @@ public abstract static class Builder { public abstract Builder createdAt(DateTime createdAt); public abstract Builder modifiedAt(DateTime modifiedAt); + + public abstract Builder errors(Set errors); } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index fc6d2fe2f245..60d5d9d232ec 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -21,16 +21,16 @@ import com.google.common.collect.Maps; import com.google.common.eventbus.EventBus; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; -import org.graylog.plugins.pipelineprocessor.db.PipelineSourceService; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; -import org.graylog.plugins.pipelineprocessor.db.RuleSourceService; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.PipelineSource; import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; -import org.graylog.plugins.pipelineprocessor.rest.RuleSource; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; import org.graylog2.plugin.Tools; @@ -50,29 +50,29 @@ public class PipelineInterpreterTest { @Test public void testCreateMessage() { - final RuleSourceService ruleSourceService = mock(RuleSourceService.class); - when(ruleSourceService.loadAll()).thenReturn(Collections.singleton( - RuleSource.create("abc", - "title", - "description", - "rule \"creates message\"\n" + - "when string(message.`message`) == \"original message\"\n" + - "then\n" + - " create_message(\"derived message\");\n" + - "end", - Tools.nowUTC(), - null) + final RuleService ruleService = mock(RuleService.class); + when(ruleService.loadAll()).thenReturn(Collections.singleton( + RuleDao.create("abc", + "title", + "description", + "rule \"creates message\"\n" + + "when tostring($message.message) == \"original message\"\n" + + "then\n" + + " create_message(\"derived message\");\n" + + "end", + Tools.nowUTC(), + null) )); - final PipelineSourceService pipelineSourceService = mock(PipelineSourceService.class); - when(pipelineSourceService.loadAll()).thenReturn(Collections.singleton( - PipelineSource.create("cde", "title", "description", - "pipeline \"pipeline\"\n" + - "stage 0 match all\n" + - " rule \"creates message\";\n" + - "end\n", - Tools.nowUTC(), - null) + final PipelineService pipelineService = mock(PipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("cde", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"creates message\";\n" + + "end\n", + Tools.nowUTC(), + null) )); final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); @@ -90,8 +90,8 @@ public void testCreateMessage() { final PipelineRuleParser parser = setupParser(functions); final PipelineInterpreter interpreter = new PipelineInterpreter( - ruleSourceService, - pipelineSourceService, + ruleService, + pipelineService, pipelineStreamAssignmentService, parser, mock(Journal.class), From 18c311f13516b245ac494c64b220db6cf219cd02 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 2 Mar 2016 13:12:54 +0100 Subject: [PATCH 118/521] make the title indices unique --- .../graylog/plugins/pipelineprocessor/db/PipelineService.java | 3 ++- .../org/graylog/plugins/pipelineprocessor/db/RuleService.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java index 8456f2ec2d7a..07fa77aa9157 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java @@ -17,6 +17,7 @@ package org.graylog.plugins.pipelineprocessor.db; import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; import com.mongodb.MongoException; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; @@ -46,7 +47,7 @@ public PipelineService(MongoConnection mongoConnection, MongoJackObjectMapperPro PipelineDao.class, String.class, mapper.get()); - dbCollection.createIndex(DBSort.asc("title")); + dbCollection.createIndex(DBSort.asc("title"), new BasicDBObject("unique", true)); } public PipelineDao save(PipelineDao pipeline) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java index bf9ca320c294..0ab657ffeb0b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java @@ -17,6 +17,7 @@ package org.graylog.plugins.pipelineprocessor.db; import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; import com.mongodb.MongoException; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; @@ -46,7 +47,7 @@ public RuleService(MongoConnection mongoConnection, MongoJackObjectMapperProvide RuleDao.class, String.class, mapper.get()); - dbCollection.createIndex(DBSort.asc("title")); + dbCollection.createIndex(DBSort.asc("title"), new BasicDBObject("unique", true)); } public RuleDao save(RuleDao rule) { From 3d936185d5ae4f0f0a55e95365a3c12dd79d6c9e Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 2 Mar 2016 14:14:32 +0100 Subject: [PATCH 119/521] use the name of the parsed rule/pipeline instead of the separate title attribute --- .../pipelineprocessor/rest/PipelineResource.java | 16 +++++++++------- .../pipelineprocessor/rest/RuleResource.java | 16 ++++++++++------ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index e94f8598e010..90cc5a91c627 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -72,18 +72,18 @@ public PipelineResource(PipelineService pipelineService, this.clusterBus = clusterBus; } - @ApiOperation(value = "Create a processing pipeline from source", notes = "") @POST @Path("/pipeline") public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { + final Pipeline pipeline; try { - final Pipeline pipeline = pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); + pipeline = pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final PipelineDao pipelineDao = PipelineDao.builder() - .title(pipelineSource.title()) + .title(pipeline.name()) .description(pipelineSource.description()) .source(pipelineSource.source()) .createdAt(DateTime.now()) @@ -99,13 +99,14 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t @POST @Path("/pipeline/parse") public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { + final Pipeline pipeline; try { - pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); + pipeline = pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } return PipelineSource.builder() - .title(pipelineSource.title()) + .title(pipeline.name()) .description(pipelineSource.description()) .source(pipelineSource.source()) .createdAt(DateTime.now()) @@ -140,13 +141,14 @@ public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) thr public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { final PipelineDao dao = pipelineService.load(id); + final Pipeline pipeline; try { - pipelineRuleParser.parsePipeline(update.id(), update.source()); + pipeline = pipelineRuleParser.parsePipeline(update.id(), update.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final PipelineDao toSave = dao.toBuilder() - .title(update.title()) + .title(pipeline.name()) .description(update.description()) .source(update.source()) .modifiedAt(DateTime.now()) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 33e9b16d05cb..5a1ff6fa7bdc 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -20,6 +20,7 @@ import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; +import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; @@ -75,13 +76,14 @@ public RuleResource(RuleService ruleService, @POST @Path("/rule") public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { + final Rule rule; try { - pipelineRuleParser.parseRule(ruleSource.source()); + rule = pipelineRuleParser.parseRule(ruleSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final RuleDao newRuleSource = RuleDao.builder() - .title(ruleSource.title()) + .title(rule.name()) // use the name from the parsed rule source. .description(ruleSource.description()) .source(ruleSource.source()) .createdAt(DateTime.now()) @@ -98,13 +100,14 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No @POST @Path("/rule/parse") public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { + final Rule rule; try { - pipelineRuleParser.parseRule(ruleSource.source()); + rule = pipelineRuleParser.parseRule(ruleSource.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } return RuleSource.builder() - .title(ruleSource.title()) + .title(rule.name()) .description(ruleSource.description()) .source(ruleSource.source()) .createdAt(DateTime.now()) @@ -135,13 +138,14 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { final RuleDao ruleDao = ruleService.load(id); + final Rule rule; try { - pipelineRuleParser.parseRule(update.source()); + rule = pipelineRuleParser.parseRule(update.source()); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } final RuleDao toSave = ruleDao.toBuilder() - .title(update.title()) + .title(rule.name()) .description(update.description()) .source(update.source()) .modifiedAt(DateTime.now()) From a603fc0655867466a3ab7b7526d14cb9d1e6ca22 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 2 Mar 2016 14:25:14 +0100 Subject: [PATCH 120/521] remove title input field, the title is taken from the parsed rule source now --- src/web/rules/RuleForm.jsx | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index f585c0da9f4d..c31171e51015 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -139,15 +139,6 @@ const RuleForm = React.createClass({ onSubmitForm={this._save} submitButtonText="Save">
    - Date: Wed, 2 Mar 2016 14:47:06 +0100 Subject: [PATCH 121/521] return information about a pipeline's stages in the rest resource add license headers --- .../pipelineprocessor/db/PipelineDao.java | 16 ++++++ .../plugins/pipelineprocessor/db/RuleDao.java | 16 ++++++ .../rest/PipelineSource.java | 19 ++++--- .../pipelineprocessor/rest/StageSource.java | 52 +++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java index 5d4809211577..531e17b5a9ef 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.db; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java index 3e4f5ac44b8c..32ccd8522431 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.db; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java index 680869190d9b..5b0d6175db39 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; -import org.graylog.plugins.pipelineprocessor.ast.Stage; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; @@ -65,8 +64,7 @@ public abstract class PipelineSource { public abstract DateTime modifiedAt(); @JsonProperty - @Nullable - public abstract List stages(); + public abstract List stages(); @JsonProperty @Nullable @@ -80,7 +78,7 @@ public static Builder builder() { @JsonCreator public static PipelineSource create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, - @JsonProperty("title") String title, + @JsonProperty("title") String title, @JsonProperty("description") @Nullable String description, @JsonProperty("source") String source, @Nullable @JsonProperty("created_at") DateTime createdAt, @@ -104,6 +102,15 @@ public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao) } catch (ParseException e) { errors = e.getErrors(); } + final List stageSources = (pipeline == null) ? Collections.emptyList() : + pipeline.stages().stream() + .map(stage -> StageSource.builder() + .matchAll(stage.matchAll()) + .rules(stage.ruleReferences()) + .stage(stage.stage()) + .build()) + .collect(Collectors.toList()); + return builder() .id(dao.id()) .title(dao.title()) @@ -111,7 +118,7 @@ public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao) .source(dao.source()) .createdAt(dao.createdAt()) .modifiedAt(dao.modifiedAt()) - .stages(pipeline == null ? null : pipeline.stages().stream().map(Stage::stage).collect(Collectors.toList())) + .stages(stageSources) .errors(errors) .build(); } @@ -132,7 +139,7 @@ public abstract static class Builder { public abstract Builder modifiedAt(DateTime modifiedAt); - public abstract Builder stages(List stages); + public abstract Builder stages(List stages); public abstract Builder errors(Set errors); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java new file mode 100644 index 000000000000..9d146815b112 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java @@ -0,0 +1,52 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +public abstract class StageSource { + + @JsonProperty + public abstract int stage(); + + @JsonProperty + public abstract boolean matchAll(); + + @JsonProperty + public abstract List rules(); + + public static Builder builder() { + return new AutoValue_StageSource.Builder(); + } + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + public abstract StageSource build(); + + public abstract Builder stage(int stageNumber); + + public abstract Builder matchAll(boolean mustMatchAll); + + public abstract Builder rules(List ruleRefs); + } +} From aadfed89368a85848249d2b054ce117ce810e201 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 15:05:46 +0100 Subject: [PATCH 122/521] Adapt pipelines overview page to latest api changes --- src/web/pipelines/ProcessingTimelineComponent.jsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 13859ad43b7f..2adc24fea9c2 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -1,6 +1,7 @@ import React from 'react'; import Reflux from 'reflux'; import { Alert } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; import { DataTable, Spinner } from 'components/common'; @@ -21,7 +22,7 @@ const ProcessingTimelineComponent = React.createClass({ .map(pipeline => pipeline.stages) .reduce((usedStages, pipelineStages) => { // Concat stages in a single array removing duplicates - return usedStages.concat(pipelineStages.filter(stage => usedStages.indexOf(stage) === -1)); + return usedStages.concat(pipelineStages.map(stage => stage.stage).filter(stage => usedStages.indexOf(stage) === -1)); }, []) .sort(); }, @@ -37,9 +38,10 @@ const ProcessingTimelineComponent = React.createClass({ _formatStages(pipeline, stages) { const formattedStages = []; + const stageNumbers = stages.map(stage => stage.stage); this.usedStages.forEach(usedStage => { - if (stages.indexOf(usedStage) === -1) { + if (stageNumbers.indexOf(usedStage) === -1) { formattedStages.push(
    Idle
    ); @@ -56,7 +58,9 @@ const ProcessingTimelineComponent = React.createClass({ _pipelineFormatter(pipeline) { return ( - {pipeline.title} + + {pipeline.title} + {this._formatStages(pipeline, pipeline.stages)} ); From c505702b5d335ba79ef842dcdeed384cc455da4d Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 2 Mar 2016 15:21:19 +0100 Subject: [PATCH 123/521] api endpoint to retrieve named rules in bulk --- .../pipelineprocessor/db/RuleService.java | 11 ++++++++++ .../rest/BulkRuleRequest.java | 20 +++++++++++++++++++ .../pipelineprocessor/rest/RuleResource.java | 11 ++++++++++ 3 files changed, 42 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java index 0ab657ffeb0b..21883745cfba 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java @@ -23,6 +23,7 @@ import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; import org.mongojack.DBCursor; +import org.mongojack.DBQuery; import org.mongojack.DBSort; import org.mongojack.JacksonDBCollection; import org.mongojack.WriteResult; @@ -79,4 +80,14 @@ public void delete(String id) { log.error("Unable to delete rule {}", id); } } + + public Collection loadNamed(Collection ruleNames) { + try { + final DBCursor ruleDaos = dbCollection.find(DBQuery.in("title", ruleNames)); + return Sets.newHashSet(ruleDaos.iterator()); + } catch (MongoException e) { + log.error("Unable to bulk load rules", e); + return Collections.emptySet(); + } + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java new file mode 100644 index 000000000000..1dc31e2e7d0a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java @@ -0,0 +1,20 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +@JsonAutoDetect +public abstract class BulkRuleRequest { + @JsonProperty + public abstract List rules(); + + @JsonCreator + public static BulkRuleRequest create(@JsonProperty("rules") List rules) { + return new AutoValue_BulkRuleRequest(rules); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 5a1ff6fa7bdc..60169650ff1f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -132,6 +132,17 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws return RuleSource.fromDao(pipelineRuleParser, ruleService.load(id)); } + @ApiOperation("Retrieve the named processing rules in bulk") + @Path("/rule/multiple") + @POST + public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) { + Collection ruleDaos = ruleService.loadNamed(rules.rules()); + + return ruleDaos.stream() + .map(ruleDao -> RuleSource.fromDao(pipelineRuleParser, ruleDao)) + .collect(Collectors.toList()); + } + @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") @Path("/rule/{id}") @PUT From 37e45be11fe6e078bfaba4e9487d6bfae66f8a26 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 2 Mar 2016 15:29:58 +0100 Subject: [PATCH 124/521] fix api browser naming for rest resources --- .../rest/PipelineResource.java | 14 +++++++------- .../rest/PipelineStreamResource.java | 2 +- .../pipelineprocessor/rest/RuleResource.java | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index 90cc5a91c627..f66fea2d14a0 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -52,7 +52,7 @@ import java.util.Collection; @Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") -@Path("/system/pipelines") +@Path("/system/pipelines/pipeline") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class PipelineResource extends RestResource implements PluginRestResource { @@ -74,7 +74,7 @@ public PipelineResource(PipelineService pipelineService, @ApiOperation(value = "Create a processing pipeline from source", notes = "") @POST - @Path("/pipeline") + @Path("/") public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { final Pipeline pipeline; try { @@ -97,7 +97,7 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t @ApiOperation(value = "Parse a processing pipeline without saving it", notes = "") @POST - @Path("/pipeline/parse") + @Path("/parse") public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { final Pipeline pipeline; try { @@ -116,7 +116,7 @@ public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNu @ApiOperation(value = "Get all processing pipelines") @GET - @Path("/pipeline") + @Path("/") public Collection getAll() { final Collection daos = pipelineService.loadAll(); final ArrayList results = Lists.newArrayList(); @@ -128,7 +128,7 @@ public Collection getAll() { } @ApiOperation(value = "Get a processing pipeline", notes = "It can take up to a second until the change is applied") - @Path("/pipeline/{id}") + @Path("/{id}") @GET public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { final PipelineDao dao = pipelineService.load(id); @@ -136,7 +136,7 @@ public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) thr } @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied") - @Path("/pipeline/{id}") + @Path("/{id}") @PUT public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { @@ -160,7 +160,7 @@ public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, } @ApiOperation(value = "Delete a processing pipeline", notes = "It can take up to a second until the change is applied") - @Path("/pipeline/{id}") + @Path("/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { pipelineService.load(id); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java index 3157326e5e93..7f9a45da43fd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java @@ -39,7 +39,7 @@ import javax.ws.rs.core.MediaType; import java.util.Set; -@Api(value = "Pipeline/Streams", description = "Stream assignment of processing pipelines") +@Api(value = "Pipelines/Streams", description = "Stream assignment of processing pipelines") @Path("/system/pipelines/streams") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 60169650ff1f..8438ae2609cf 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -50,8 +50,8 @@ import java.util.Collection; import java.util.stream.Collectors; -@Api(value = "Pipeline/Rules", description = "Rules for the pipeline message processor") -@Path("/system/pipelines") +@Api(value = "Pipelines/Rules", description = "Rules for the pipeline message processor") +@Path("/system/pipelines/rule") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public class RuleResource extends RestResource implements PluginRestResource { @@ -74,7 +74,7 @@ public RuleResource(RuleService ruleService, @ApiOperation(value = "Create a processing rule from source", notes = "") @POST - @Path("/rule") + @Path("/") public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { final Rule rule; try { @@ -98,7 +98,7 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No @ApiOperation(value = "Parse a processing rule without saving it", notes = "") @POST - @Path("/rule/parse") + @Path("/parse") public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { final Rule rule; try { @@ -117,7 +117,7 @@ public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleS @ApiOperation(value = "Get all processing rules") @GET - @Path("/rule") + @Path("/") public Collection getAll() { final Collection ruleDaos = ruleService.loadAll(); return ruleDaos.stream() @@ -126,14 +126,14 @@ public Collection getAll() { } @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") - @Path("/rule/{id}") + @Path("/{id}") @GET public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { return RuleSource.fromDao(pipelineRuleParser, ruleService.load(id)); } @ApiOperation("Retrieve the named processing rules in bulk") - @Path("/rule/multiple") + @Path("/multiple") @POST public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) { Collection ruleDaos = ruleService.loadNamed(rules.rules()); @@ -144,7 +144,7 @@ public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) } @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") - @Path("/rule/{id}") + @Path("/{id}") @PUT public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { @@ -170,7 +170,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, } @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") - @Path("/rule/{id}") + @Path("/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { ruleService.load(id); From ebd5067da09e6335b94a0e25192edfff7a5e4c4b Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 19:37:41 +0100 Subject: [PATCH 125/521] Make titles nullables --- .../graylog/plugins/pipelineprocessor/rest/PipelineSource.java | 1 + .../org/graylog/plugins/pipelineprocessor/rest/RuleSource.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java index 5b0d6175db39..e7b36e4e0057 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -46,6 +46,7 @@ public abstract class PipelineSource { public abstract String id(); @JsonProperty + @Nullable public abstract String title(); @JsonProperty diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index e42158ad1b34..0bacc9726a68 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -42,6 +42,7 @@ public abstract class RuleSource { public abstract String id(); @JsonProperty + @Nullable public abstract String title(); @JsonProperty From 38c58f3af5c8e50b9036cb8eef6858e9c6befb8a Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 19:38:11 +0100 Subject: [PATCH 126/521] Add multiple action in RulesStore --- src/web/rules/RulesActions.jsx | 13 +++++++------ src/web/rules/RulesStore.js | 11 +++++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/web/rules/RulesActions.jsx b/src/web/rules/RulesActions.jsx index 433843e23fd5..f79106e1dc28 100644 --- a/src/web/rules/RulesActions.jsx +++ b/src/web/rules/RulesActions.jsx @@ -1,12 +1,13 @@ import Reflux from 'reflux'; const RulesActions = Reflux.createActions({ - 'delete': { asyncResult: true }, - 'list': { asyncResult: true }, - 'get': { asyncResult: true }, - 'save': { asyncResult: true }, - 'update': { asyncResult: true }, - 'parse' : { asyncResult: true }, + 'delete': {asyncResult: true}, + 'list': {asyncResult: true}, + 'get': {asyncResult: true}, + 'save': {asyncResult: true}, + 'update': {asyncResult: true}, + 'parse': {asyncResult: true}, + 'multiple': {asyncResult: true}, }); export default RulesActions; \ No newline at end of file diff --git a/src/web/rules/RulesStore.js b/src/web/rules/RulesStore.js index 37a98a229db3..d7cd67ea5aac 100644 --- a/src/web/rules/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -98,7 +98,14 @@ const RulesStore = Reflux.createStore({ } } ); - } + }, + multiple(ruleNames, callback) { + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/multiple'); + const promise = fetch('POST', url, {rules: ruleNames}); + promise.then(callback); + + return promise; + }, }); -export default RulesStore; \ No newline at end of file +export default RulesStore; From 789430a077743c072d421a97c60eb1fb67bd702f Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 19:38:32 +0100 Subject: [PATCH 127/521] Use natural sort to compare stages Fixes an issue comparing negative values :grumpy: --- package.json | 1 + src/web/pipelines/ProcessingTimelineComponent.jsx | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a713bba58520..b9785b41b47f 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "dependencies": { "brace": "^0.7.0", "history": "^1.17.0", + "javascript-natural-sort": "^0.7.1", "react": "0.14.x", "react-ace": "^3.1.0", "react-bootstrap": "^0.28.1", diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 2adc24fea9c2..df5d7e75a913 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -2,6 +2,7 @@ import React from 'react'; import Reflux from 'reflux'; import { Alert } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; +import naturalSort from 'javascript-natural-sort'; import { DataTable, Spinner } from 'components/common'; @@ -24,7 +25,7 @@ const ProcessingTimelineComponent = React.createClass({ // Concat stages in a single array removing duplicates return usedStages.concat(pipelineStages.map(stage => stage.stage).filter(stage => usedStages.indexOf(stage) === -1)); }, []) - .sort(); + .sort(naturalSort); }, _headerCellFormatter(header) { From 6d6510c1898ee55506c6a36def504687fb124dda Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 19:39:01 +0100 Subject: [PATCH 128/521] Add get single pipeline action in PipelinesStore Also fix the way we update the store state. --- src/web/PipelinesStore.js | 41 +++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/web/PipelinesStore.js b/src/web/PipelinesStore.js index 0b04e47632e4..ab81fc042551 100644 --- a/src/web/PipelinesStore.js +++ b/src/web/PipelinesStore.js @@ -12,6 +12,15 @@ const PipelinesStore = Reflux.createStore({ listenables: [PipelinesActions], pipelines: undefined, + _updatePipelinesState(pipeline) { + if (!this.pipelines) { + this.pipelines = [pipeline]; + } else { + this.pipelines = this.pipelines.map((p) => p.id === pipeline.id ? pipeline : p); + } + this.trigger({pipelines: this.pipelines}); + }, + list() { const failCallback = (error) => { UserNotification.error('Fetching pipelines failed with status: ' + error.message, @@ -26,7 +35,14 @@ const PipelinesStore = Reflux.createStore({ }, get(pipelineId) { + const failCallback = (error) => { + UserNotification.error('Fetching pipeline failed with status: ' + error.message, + `Could not retrieve processing pipeline "${pipelineId}"`); + }; + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/pipeline/${pipelineId}`); + const promise = fetch('GET', url); + promise.then(this._updatePipelinesState, failCallback); }, save(pipelineSource) { @@ -38,12 +54,9 @@ const PipelinesStore = Reflux.createStore({ const pipeline = { title: pipelineSource.title, description: pipelineSource.description, - source: pipelineSource.source + source: pipelineSource.source, }; - return fetch('POST', url, pipeline).then((response) => { - this.pipelines = response; - this.trigger({pipelines: response}); - }, failCallback); + return fetch('POST', url, pipeline).then(this._updatePipelinesState, failCallback); }, update(pipelineSource) { @@ -56,12 +69,9 @@ const PipelinesStore = Reflux.createStore({ id: pipelineSource.id, title: pipelineSource.title, description: pipelineSource.description, - source: pipelineSource.source + source: pipelineSource.source, }; - return fetch('PUT', url, pipeline).then((response) => { - this.pipelines = this.pipelines.map((e) => e.id === response.id ? response : e); - this.trigger({pipelines: this.pipelines}); - }, failCallback); + return fetch('PUT', url, pipeline).then(this._updatePipelinesState, failCallback); }, delete(pipelineId) { const failCallback = (error) => { @@ -70,7 +80,8 @@ const PipelinesStore = Reflux.createStore({ }; const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline/' + pipelineId); return fetch('DELETE', url).then(() => { - this.pipelines = this.pipelines.filter((el) => el.id !== pipelineId); + const updatedPipelines = this.pipelines || []; + this.pipelines = updatedPipelines.filter((el) => el.id !== pipelineId); this.trigger({pipelines: this.pipelines}); }, failCallback); }, @@ -79,10 +90,10 @@ const PipelinesStore = Reflux.createStore({ const pipeline = { title: pipelineSource.title, description: pipelineSource.description, - source: pipelineSource.source + source: pipelineSource.source, }; return fetch('POST', url, pipeline).then( - (response) => { + () => { // call to clear the errors, the parsing was successful callback([]); }, @@ -94,7 +105,7 @@ const PipelinesStore = Reflux.createStore({ } } ); - } + }, }); -export default PipelinesStore; \ No newline at end of file +export default PipelinesStore; From 820d3655366b95fc3f9721d2531dc514edb80c97 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 2 Mar 2016 19:39:43 +0100 Subject: [PATCH 129/521] Add first version of pipeline details page - Show pipeline description - Show all stages and their rules - Let user add new stages with their corresponding rules --- src/web/index.jsx | 2 + src/web/logic/SourceGenerator.js | 16 +++ src/web/pipelines/Pipeline.jsx | 47 +++++++ src/web/pipelines/PipelineDetailsPage.jsx | 75 ++++++++++++ src/web/pipelines/Stage.jsx | 91 ++++++++++++++ src/web/pipelines/StageForm.jsx | 142 ++++++++++++++++++++++ 6 files changed, 373 insertions(+) create mode 100644 src/web/logic/SourceGenerator.js create mode 100644 src/web/pipelines/Pipeline.jsx create mode 100644 src/web/pipelines/PipelineDetailsPage.jsx create mode 100644 src/web/pipelines/Stage.jsx create mode 100644 src/web/pipelines/StageForm.jsx diff --git a/src/web/index.jsx b/src/web/index.jsx index b01ecde7472d..40008210e31c 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -1,6 +1,7 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; import PipelinesOverviewPage from './pipelines/PipelinesOverviewPage'; +import PipelineDetailsPage from './pipelines/PipelineDetailsPage'; import PipelinesPage from './PipelinesPage'; import RulesPage from './rules/RulesPage'; @@ -9,6 +10,7 @@ PluginStore.register(new PluginManifest(packageJson, { {path: '/system/pipelines', component: PipelinesPage}, {path: '/system/pipelines/overview', component: PipelinesOverviewPage}, {path: '/system/pipelines/rules', component: RulesPage}, + {path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage}, ], systemnavigation: [ diff --git a/src/web/logic/SourceGenerator.js b/src/web/logic/SourceGenerator.js new file mode 100644 index 000000000000..bc1a1a7e2c65 --- /dev/null +++ b/src/web/logic/SourceGenerator.js @@ -0,0 +1,16 @@ +const SourceGenerator = { + generatePipeline(pipeline) { + let source = `pipeline "${pipeline.title}"\n`; + pipeline.stages.forEach(stage => { + source += `stage ${stage.stage} match ${stage.match_all ? 'all' : 'either'}\n`; + stage.rules.forEach(rule => { + source += `rule "${rule}"\n`; + }); + }); + source += 'end'; + + return source; + }, +}; + +export default SourceGenerator; diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx new file mode 100644 index 000000000000..16018389d254 --- /dev/null +++ b/src/web/pipelines/Pipeline.jsx @@ -0,0 +1,47 @@ +import React, {PropTypes} from 'react'; +import { Row, Col } from 'react-bootstrap'; + +import { EntityList } from 'components/common'; +import Stage from './Stage'; +import StageForm from './StageForm'; + +const Pipeline = React.createClass({ + propTypes: { + pipeline: PropTypes.object.isRequired, + onStagesChange: PropTypes.func.isRequired, + }, + + _saveStage(stage, callback) { + const newStages = this.props.pipeline.stages.slice(); + newStages.push(stage); + this.props.onStagesChange(newStages, callback); + }, + + _formatStage(stage, maxStage) { + return ; + }, + + render() { + const maxStage = this.props.pipeline.stages.reduce((max, currentStage) => Math.max(max, currentStage.stage), -Infinity); + const formattedStages = this.props.pipeline.stages + .sort((s1, s2) => s1.stage - s2.stage) + .map(stage => this._formatStage(stage, maxStage)); + + return ( +
    + + +
    + +
    +

    Description

    +

    {this.props.pipeline.description}

    + +     + +
    + ); + }, +}); + +export default Pipeline; diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx new file mode 100644 index 000000000000..ef2087789ef6 --- /dev/null +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -0,0 +1,75 @@ +import React, {PropTypes} from 'react'; +import Reflux from 'reflux'; +import { Row, Col, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { PageHeader, Spinner } from 'components/common'; +import Pipeline from './Pipeline'; + +import SourceGenerator from 'logic/SourceGenerator'; +import ObjectUtils from 'util/ObjectUtils'; +import PipelinesActions from 'PipelinesActions'; +import PipelinesStore from 'PipelinesStore'; + +function filterPipeline(state) { + return state.pipelines ? state.pipelines.filter(p => p.id === this.props.params.pipelineId)[0] : undefined; +} + +const PipelineDetailsPage = React.createClass({ + propTypes: { + params: PropTypes.object.isRequired, + }, + + mixins: [Reflux.connectFilter(PipelinesStore, 'pipeline', filterPipeline)], + + componentDidMount() { + PipelinesActions.get(this.props.params.pipelineId); + }, + + _onStagesChange(newStages, callback) { + const newPipeline = ObjectUtils.clone(this.state.pipeline); + newPipeline.stages = newStages; + const pipelineSource = SourceGenerator.generatePipeline(newPipeline); + newPipeline.source = pipelineSource; + PipelinesActions.update(newPipeline); + callback(); + }, + + render() { + if (!this.state.pipeline) { + return ; + } + + return ( +
    + Pipeline "{this.state.pipeline.title}"} titleSize={9} buttonSize={3}> + + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} + rules are evaluated and applied. Messages can go through one or more stages. + + + After each stage is completed, you can decide if messages matching all or one of the rules continue to the next stage. + + + + + + + {' '} + + + + + + + + + + + +
    + ); + }, +}); + +export default PipelineDetailsPage; diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx new file mode 100644 index 000000000000..1ed10b351d6e --- /dev/null +++ b/src/web/pipelines/Stage.jsx @@ -0,0 +1,91 @@ +import React, {PropTypes} from 'react'; +import { Col } from 'react-bootstrap'; + +import { DataTable, EntityListItem, Spinner } from 'components/common'; +import RulesActions from 'rules/RulesActions'; + +const Stage = React.createClass({ + propTypes: { + stage: PropTypes.object.isRequired, + isLastStage: PropTypes.bool, + }, + + getInitialState() { + return { + rules: undefined, + }; + }, + + + componentDidMount() { + RulesActions.multiple(this.props.stage.rules, (rules) => { + const newRules = this.props.stage.rules.map(ruleName => rules.filter(rule => rule.title === ruleName)[0]); + this.setState({rules: newRules}); + }); + }, + + _ruleHeaderFormatter(header) { + return {header}; + }, + + _ruleRowFormatter(rule) { + return ( + + {rule.title} + {rule.description} + + + ); + }, + + _formatRules(rules) { + const headers = ['Title', 'Description', 'Actions']; + + return ( + + ); + }, + + render() { + const stage = this.props.stage; + + const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules` )}`; + + let description; + if (this.props.isLastStage) { + description = 'There are no further stages in this pipeline. Once rules in this stage are applied, the pipeline will have finished processing.'; + } else { + description = ( + + Messages satisfying {stage.match_all ? 'all rules' : 'at least one rule'}{' '} + in this stage, will continue to the next stage. + + ); + } + + let content; + if (this.state.rules) { + content = ( + {this._formatRules(this.state.rules)} + ); + } else { + content = ; + } + + return ( + + ); + }, +}); + +export default Stage; diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx new file mode 100644 index 000000000000..86683fde8a35 --- /dev/null +++ b/src/web/pipelines/StageForm.jsx @@ -0,0 +1,142 @@ +import React, { PropTypes } from 'react'; +import Reflux from 'reflux'; +import { Input, Button } from 'react-bootstrap'; + +import { SelectableList, Spinner } from 'components/common'; +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; +import ObjectUtils from 'util/ObjectUtils'; +import FormsUtils from 'util/FormsUtils'; + +import RulesActions from 'rules/RulesActions'; +import RulesStore from 'rules/RulesStore'; + +const StageForm = React.createClass({ + propTypes: { + stage: PropTypes.object, + create: React.PropTypes.bool, + save: React.PropTypes.func.isRequired, + validateStage: React.PropTypes.func.isRequired, + }, + mixins: [Reflux.connect(RulesStore)], + + getDefaultProps() { + return { + stage: { + stage: 0, + match_all: false, + rules: [], + }, + }; + }, + + getInitialState() { + const stage = ObjectUtils.clone(this.props.stage); + return { + // when editing, take the stage that's been passed in + stage: { + stage: stage.stage, + match_all: stage.match_all, + rules: stage.rules, + }, + }; + }, + + componentDidMount() { + RulesActions.list(); + }, + + openModal() { + this.refs.modal.open(); + }, + + _onChange(event) { + const stage = ObjectUtils.clone(this.state.stage); + stage[event.target.name] = FormsUtils.getValueFromInput(event.target); + this.setState({stage: stage}); + }, + + _onRulesChange(newRules) { + const stage = ObjectUtils.clone(this.state.stage); + stage.rules = newRules; + this.setState({stage: stage}); + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _saved() { + this._closeModal(); + if (this.props.create) { + this.setState({stage: this.getInitialState()}); + } + }, + + _save() { + this.props.save(this.state.stage, this._saved); + }, + + _getFormattedOptions(rules) { + return rules ? rules.map(rule => { + return {value: rule.title, label: rule.title}; + }) : []; + }, + + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Add new stage'; + } else { + triggerButtonContent = Edit; + } + + return ( + + + +
    + + + + + + + + + + + +
    +     +     + ); + }, +}); + +export default StageForm; From 214947ba9b608f94c92ccd282b4165e509fbb36e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 12:57:20 +0100 Subject: [PATCH 130/521] Add edit stages functionality --- src/web/pipelines/Pipeline.jsx | 10 +++++++- src/web/pipelines/PipelineDetailsPage.jsx | 4 +++- src/web/pipelines/Stage.jsx | 28 ++++++++++------------- src/web/pipelines/StageForm.jsx | 14 ++++-------- src/web/rules/RulesStore.js | 10 +++++--- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index 16018389d254..f223f4ad7675 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -17,8 +17,16 @@ const Pipeline = React.createClass({ this.props.onStagesChange(newStages, callback); }, + _updateStage(prevStage) { + return (stage, callback) => { + const newStages = this.props.pipeline.stages.filter(s => s.stage !== prevStage.stage); + newStages.push(stage); + this.props.onStagesChange(newStages, callback); + }; + }, + _formatStage(stage, maxStage) { - return ; + return ; }, render() { diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index ef2087789ef6..cec3004fcefc 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -10,6 +10,7 @@ import SourceGenerator from 'logic/SourceGenerator'; import ObjectUtils from 'util/ObjectUtils'; import PipelinesActions from 'PipelinesActions'; import PipelinesStore from 'PipelinesStore'; +import RulesStore from 'rules/RulesStore'; function filterPipeline(state) { return state.pipelines ? state.pipelines.filter(p => p.id === this.props.params.pipelineId)[0] : undefined; @@ -24,6 +25,7 @@ const PipelineDetailsPage = React.createClass({ componentDidMount() { PipelinesActions.get(this.props.params.pipelineId); + RulesStore.list(); }, _onStagesChange(newStages, callback) { @@ -64,7 +66,7 @@ const PipelineDetailsPage = React.createClass({ - + diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 1ed10b351d6e..2259a38b0707 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -1,28 +1,18 @@ import React, {PropTypes} from 'react'; +import Reflux from 'reflux'; import { Col } from 'react-bootstrap'; import { DataTable, EntityListItem, Spinner } from 'components/common'; -import RulesActions from 'rules/RulesActions'; +import RulesStore from 'rules/RulesStore'; +import StageForm from './StageForm'; const Stage = React.createClass({ propTypes: { stage: PropTypes.object.isRequired, isLastStage: PropTypes.bool, + onSave: PropTypes.func.isRequired, }, - - getInitialState() { - return { - rules: undefined, - }; - }, - - - componentDidMount() { - RulesActions.multiple(this.props.stage.rules, (rules) => { - const newRules = this.props.stage.rules.map(ruleName => rules.filter(rule => rule.title === ruleName)[0]); - this.setState({rules: newRules}); - }); - }, + mixins: [Reflux.connect(RulesStore)], _ruleHeaderFormatter(header) { return {header}; @@ -58,6 +48,10 @@ const Stage = React.createClass({ const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules` )}`; + const actions = [ + , + ]; + let description; if (this.props.isLastStage) { description = 'There are no further stages in this pipeline. Once rules in this stage are applied, the pipeline will have finished processing.'; @@ -71,9 +65,10 @@ const Stage = React.createClass({ } let content; + // We check if we have the rules details before trying to render them if (this.state.rules) { content = ( - {this._formatRules(this.state.rules)} + {this._formatRules(this.props.stage.rules.map(name => this.state.rules.filter(r => r.title === name)[0]))} ); } else { content = ; @@ -82,6 +77,7 @@ const Stage = React.createClass({ return ( ); diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx index 86683fde8a35..3d1db6f09af7 100644 --- a/src/web/pipelines/StageForm.jsx +++ b/src/web/pipelines/StageForm.jsx @@ -7,7 +7,6 @@ import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; import ObjectUtils from 'util/ObjectUtils'; import FormsUtils from 'util/FormsUtils'; -import RulesActions from 'rules/RulesActions'; import RulesStore from 'rules/RulesStore'; const StageForm = React.createClass({ @@ -41,10 +40,6 @@ const StageForm = React.createClass({ }; }, - componentDidMount() { - RulesActions.list(); - }, - openModal() { this.refs.modal.open(); }, @@ -68,7 +63,7 @@ const StageForm = React.createClass({ _saved() { this._closeModal(); if (this.props.create) { - this.setState({stage: this.getInitialState()}); + this.setState(this.getInitialState()); } }, @@ -107,6 +102,7 @@ const StageForm = React.createClass({ label="Stage" autoFocus onChange={this._onChange} + help="Stage priority. The lower the number, the earlier it will execute." value={this.state.stage.stage}/> @@ -126,10 +122,10 @@ const StageForm = React.createClass({ onChange={this._onChange} checked={!this.state.stage.match_all}/> - + + onChange={this._onRulesChange} selectedOptions={this.state.stage.rules}/>
    diff --git a/src/web/rules/RulesStore.js b/src/web/rules/RulesStore.js index d7cd67ea5aac..88a41312e5aa 100644 --- a/src/web/rules/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -12,6 +12,10 @@ const RulesStore = Reflux.createStore({ listenables: [RulesActions], rules: undefined, + getInitialState() { + return {rules: this.rules}; + }, + list() { const failCallback = (error) => { UserNotification.error('Fetching rules failed with status: ' + error.message, @@ -38,7 +42,7 @@ const RulesStore = Reflux.createStore({ const rule = { title: ruleSource.title, description: ruleSource.description, - source: ruleSource.source + source: ruleSource.source, }; return fetch('POST', url, rule).then((response) => { if (this.rules === undefined) { @@ -60,7 +64,7 @@ const RulesStore = Reflux.createStore({ id: ruleSource.id, title: ruleSource.title, description: ruleSource.description, - source: ruleSource.source + source: ruleSource.source, }; return fetch('PUT', url, rule).then((response) => { this.rules = this.rules.map((e) => e.id === response.id ? response : e); @@ -83,7 +87,7 @@ const RulesStore = Reflux.createStore({ const rule = { title: ruleSource.title, description: ruleSource.description, - source: ruleSource.source + source: ruleSource.source, }; return fetch('POST', url, rule).then( (response) => { From 7620c10e3d932b4b7e80ca0465ce52460e1f8fe3 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 13:06:14 +0100 Subject: [PATCH 131/521] Add delete stage functionality to pipeline details page --- src/web/pipelines/Pipeline.jsx | 14 +++++++++++++- src/web/pipelines/PipelineDetailsPage.jsx | 4 +++- src/web/pipelines/Stage.jsx | 8 +++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index f223f4ad7675..dae8d8f39cd5 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -25,8 +25,20 @@ const Pipeline = React.createClass({ }; }, + _deleteStage(stage) { + return () => { + if (confirm(`You are about to delete stage ${stage.stage}, are you sure you want to proceed?`)) { + const newStages = this.props.pipeline.stages.filter(s => s.stage !== stage.stage); + this.props.onStagesChange(newStages); + } + }; + }, + _formatStage(stage, maxStage) { - return ; + return ( + + ); }, render() { diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index cec3004fcefc..343b402a61be 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -34,7 +34,9 @@ const PipelineDetailsPage = React.createClass({ const pipelineSource = SourceGenerator.generatePipeline(newPipeline); newPipeline.source = pipelineSource; PipelinesActions.update(newPipeline); - callback(); + if (typeof callback === 'function') { + callback(); + } }, render() { diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 2259a38b0707..7af0b1c4e2db 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -1,6 +1,6 @@ import React, {PropTypes} from 'react'; import Reflux from 'reflux'; -import { Col } from 'react-bootstrap'; +import { Col, Button } from 'react-bootstrap'; import { DataTable, EntityListItem, Spinner } from 'components/common'; import RulesStore from 'rules/RulesStore'; @@ -10,7 +10,8 @@ const Stage = React.createClass({ propTypes: { stage: PropTypes.object.isRequired, isLastStage: PropTypes.bool, - onSave: PropTypes.func.isRequired, + onUpdate: PropTypes.func.isRequired, + onDelete: PropTypes.func.isRequired, }, mixins: [Reflux.connect(RulesStore)], @@ -49,7 +50,8 @@ const Stage = React.createClass({ const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules` )}`; const actions = [ - , + , + , ]; let description; From b8d6b3bb5c2ec447fcd72b9f187a3633f3355c2e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 13:09:49 +0100 Subject: [PATCH 132/521] Remove duplicated state from rules page --- src/web/rules/RulesPage.jsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 602cef6dabdc..e1fbe30a385d 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -24,12 +24,6 @@ const RulesPage = React.createClass({ RulesActions.list(); }, - getInitialState() { - return { - rules: [], - }; - }, - render() { return ( From e31c58e7fca1da31bcefd23120f24bb380806e3c Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 14:23:54 +0100 Subject: [PATCH 133/521] Check if rules are there before rendering :spinning: --- src/web/rules/RuleList.jsx | 1 - src/web/rules/RulesComponent.jsx | 11 ++++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index b8fe6fb62022..bd586f12fb5b 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -13,7 +13,6 @@ const RuleList = React.createClass({ }, _save(rule, callback) { - console.log(rule); if (rule.id) { RulesActions.update(rule); } else { diff --git a/src/web/rules/RulesComponent.jsx b/src/web/rules/RulesComponent.jsx index c02068ae9153..9c1975959901 100644 --- a/src/web/rules/RulesComponent.jsx +++ b/src/web/rules/RulesComponent.jsx @@ -1,17 +1,18 @@ import React, {PropTypes} from 'react'; -import Reflux from 'reflux'; -import { Row, Col } from 'react-bootstrap'; -import { Input, Alert } from 'react-bootstrap'; -import RulesActions from './RulesActions'; +import { Spinner } from 'components/common'; import RuleList from './RuleList'; const RulesComponent = React.createClass({ propTypes: { - rules: PropTypes.array.isRequired, + rules: PropTypes.array, }, render() { + if (!this.props.rules) { + return ; + } + return (
    From 87cfd72451d7b20bda18cdca3c50b1a6e4d8946e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 14:28:06 +0100 Subject: [PATCH 134/521] Remove actions from rules in stages --- src/web/pipelines/Stage.jsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 7af0b1c4e2db..f11415c3c7ef 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -22,15 +22,14 @@ const Stage = React.createClass({ _ruleRowFormatter(rule) { return ( - {rule.title} + {rule.title} {rule.description} - ); }, _formatRules(rules) { - const headers = ['Title', 'Description', 'Actions']; + const headers = ['Title', 'Description']; return ( {this._formatRules(this.props.stage.rules.map(name => this.state.rules.filter(r => r.title === name)[0]))} - ); + content = this._formatRules(this.props.stage.rules.map(name => this.state.rules.filter(r => r.title === name)[0])); } else { - content = ; + content = ; } return ( @@ -81,7 +78,7 @@ const Stage = React.createClass({ titleSuffix={suffix} actions={actions} description={description} - contentRow={content}/> + contentRow={{content}}/> ); }, }); From 76e709e22789ceeba99205fcb8121020397466a3 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 15:40:40 +0100 Subject: [PATCH 135/521] Make grammar accept stages without rules --- .../org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 index 276da69f2523..197e9ebaa1d2 100644 --- a/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 +++ b/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 @@ -56,7 +56,7 @@ pipelineDeclaration stageDeclaration : Stage stage=Integer Match modifier=(All|Either) - ruleRef+ + ruleRef* ; ruleRef From 1b80ae6b666b43630f258e7018058fa0dc28fab4 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 15:40:56 +0100 Subject: [PATCH 136/521] Add, update, and delete pipelines --- src/web/PipelinesStore.js | 26 ++++- src/web/pipelines/PipelineForm.jsx | 109 ++++++++++++++++++ .../pipelines/ProcessingTimelineComponent.jsx | 66 ++++++++--- src/web/pipelines/StageForm.jsx | 2 +- 4 files changed, 183 insertions(+), 20 deletions(-) create mode 100644 src/web/pipelines/PipelineForm.jsx diff --git a/src/web/PipelinesStore.js b/src/web/PipelinesStore.js index ab81fc042551..ccbbbb3cafe1 100644 --- a/src/web/PipelinesStore.js +++ b/src/web/PipelinesStore.js @@ -16,7 +16,12 @@ const PipelinesStore = Reflux.createStore({ if (!this.pipelines) { this.pipelines = [pipeline]; } else { - this.pipelines = this.pipelines.map((p) => p.id === pipeline.id ? pipeline : p); + const doesPipelineExist = this.pipelines.some(p => p.id === pipeline.id); + if (doesPipelineExist) { + this.pipelines = this.pipelines.map((p) => p.id === pipeline.id ? pipeline : p); + } else { + this.pipelines.push(pipeline); + } } this.trigger({pipelines: this.pipelines}); }, @@ -56,7 +61,12 @@ const PipelinesStore = Reflux.createStore({ description: pipelineSource.description, source: pipelineSource.source, }; - return fetch('POST', url, pipeline).then(this._updatePipelinesState, failCallback); + return fetch('POST', url, pipeline).then( + response => { + this._updatePipelinesState(response); + UserNotification.success(`Pipeline "${pipeline.title}" created successfully`); + }, + failCallback); }, update(pipelineSource) { @@ -71,18 +81,24 @@ const PipelinesStore = Reflux.createStore({ description: pipelineSource.description, source: pipelineSource.source, }; - return fetch('PUT', url, pipeline).then(this._updatePipelinesState, failCallback); + return fetch('PUT', url, pipeline).then( + response => { + this._updatePipelinesState(response); + UserNotification.success(`Pipeline "${pipeline.title}" updated successfully`); + }, + failCallback); }, delete(pipelineId) { const failCallback = (error) => { - UserNotification.error('Updating pipeline failed with status: ' + error.message, - 'Could not update processing pipeline'); + UserNotification.error('Deleting pipeline failed with status: ' + error.message, + `Could not delete processing pipeline "${pipelineId}"`); }; const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/pipeline/' + pipelineId); return fetch('DELETE', url).then(() => { const updatedPipelines = this.pipelines || []; this.pipelines = updatedPipelines.filter((el) => el.id !== pipelineId); this.trigger({pipelines: this.pipelines}); + UserNotification.success(`Pipeline "${pipelineId}" deleted successfully`); }, failCallback); }, parse(pipelineSource, callback) { diff --git a/src/web/pipelines/PipelineForm.jsx b/src/web/pipelines/PipelineForm.jsx new file mode 100644 index 000000000000..6b2023cafe6d --- /dev/null +++ b/src/web/pipelines/PipelineForm.jsx @@ -0,0 +1,109 @@ +import React, { PropTypes } from 'react'; +import { Input, Button } from 'react-bootstrap'; + +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; +import ObjectUtils from 'util/ObjectUtils'; +import FormsUtils from 'util/FormsUtils'; + +const PipelineForm = React.createClass({ + propTypes: { + pipeline: PropTypes.object, + create: React.PropTypes.bool, + save: React.PropTypes.func.isRequired, + validatePipeline: React.PropTypes.func.isRequired, + }, + + getDefaultProps() { + return { + pipeline: { + id: undefined, + title: '', + description: '', + stages: [{ stage: 0, rules: [] }], + }, + }; + }, + + getInitialState() { + const pipeline = ObjectUtils.clone(this.props.pipeline); + return { + // when editing, take the pipeline that's been passed in + pipeline: { + id: pipeline.id, + title: pipeline.title, + description: pipeline.description, + stages: pipeline.stages, + }, + }; + }, + + openModal() { + this.refs.modal.open(); + }, + + _onChange(event) { + const pipeline = ObjectUtils.clone(this.state.pipeline); + pipeline[event.target.name] = FormsUtils.getValueFromInput(event.target); + this.setState({pipeline: pipeline}); + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _saved() { + this._closeModal(); + if (this.props.create) { + this.setState(this.getInitialState()); + } + }, + + _save() { + this.props.save(this.state.pipeline, this._saved); + }, + + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Add new pipeline'; + } else { + triggerButtonContent = Edit; + } + + return ( + + + +
    + + + +
    +     +     + ); + }, +}); + +export default PipelineForm; diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index df5d7e75a913..b7e65dbdf3e2 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -1,14 +1,18 @@ import React from 'react'; import Reflux from 'reflux'; -import { Alert } from 'react-bootstrap'; +import { Alert, Button } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; import naturalSort from 'javascript-natural-sort'; import { DataTable, Spinner } from 'components/common'; +import PipelineForm from './PipelineForm'; import PipelinesActions from 'PipelinesActions'; import PipelinesStore from 'PipelinesStore'; +import ObjectUtils from 'util/ObjectUtils'; +import SourceGenerator from 'logic/SourceGenerator'; + import {} from './ProcessingTimelineComponent.css'; const ProcessingTimelineComponent = React.createClass({ @@ -30,8 +34,15 @@ const ProcessingTimelineComponent = React.createClass({ _headerCellFormatter(header) { const style = {}; - if (header === 'Pipeline') { - style.width = 300; + switch (header) { + case 'Pipeline': + style.width = 300; + break; + case 'Actions': + style.width = 120; + break; + default: + // Nothing to see here } return {header}; @@ -58,15 +69,39 @@ const ProcessingTimelineComponent = React.createClass({ _pipelineFormatter(pipeline) { return ( - + {pipeline.title} {this._formatStages(pipeline, pipeline.stages)} + + +   + + ); }, + _savePipeline(pipeline, callback) { + const requestPipeline = ObjectUtils.clone(pipeline); + requestPipeline.source = SourceGenerator.generatePipeline(pipeline); + if (requestPipeline.id) { + PipelinesActions.update(requestPipeline); + } else { + PipelinesActions.save(requestPipeline); + } + callback(); + }, + + _deletePipeline(pipeline) { + return () => { + if (confirm(`Do you really want to delete pipeline "${pipeline.title}"? This action cannot be undone.`)) { + PipelinesActions.delete(pipeline.id); + } + }; + }, + render() { if (!this.state.pipelines) { return ; @@ -82,17 +117,20 @@ const ProcessingTimelineComponent = React.createClass({ this.usedStages = this._calculateUsedStages(this.state.pipelines); - const headers = ['Pipeline', 'ProcessingTimeline']; + const headers = ['Pipeline', 'ProcessingTimeline', 'Actions']; return ( - +
    +
    + +
    ); }, }); diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx index 3d1db6f09af7..00b354763406 100644 --- a/src/web/pipelines/StageForm.jsx +++ b/src/web/pipelines/StageForm.jsx @@ -2,7 +2,7 @@ import React, { PropTypes } from 'react'; import Reflux from 'reflux'; import { Input, Button } from 'react-bootstrap'; -import { SelectableList, Spinner } from 'components/common'; +import { SelectableList } from 'components/common'; import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; import ObjectUtils from 'util/ObjectUtils'; import FormsUtils from 'util/FormsUtils'; From 611349aa532b2e62bc1b58f02e6c80f9afe2f7ab Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 16:05:27 +0100 Subject: [PATCH 137/521] Reorganize pipelines - Move actions and store to the right folder - Remove unused components --- src/web/Pipeline.jsx | 55 ------ src/web/PipelineForm.jsx | 179 ------------------ src/web/PipelineStreamComponent.jsx | 26 --- src/web/PipelinesComponent.jsx | 75 -------- src/web/index.jsx | 10 +- src/web/pipelines/PipelineDetailsPage.jsx | 4 +- src/web/{ => pipelines}/PipelinesActions.jsx | 0 .../PipelinesInputsPage.jsx} | 30 +-- src/web/{ => pipelines}/PipelinesStore.js | 2 +- .../pipelines/ProcessingTimelineComponent.jsx | 4 +- 10 files changed, 26 insertions(+), 359 deletions(-) delete mode 100644 src/web/Pipeline.jsx delete mode 100644 src/web/PipelineForm.jsx delete mode 100644 src/web/PipelineStreamComponent.jsx delete mode 100644 src/web/PipelinesComponent.jsx rename src/web/{ => pipelines}/PipelinesActions.jsx (100%) rename src/web/{PipelinesPage.jsx => pipelines/PipelinesInputsPage.jsx} (79%) rename src/web/{ => pipelines}/PipelinesStore.js (98%) diff --git a/src/web/Pipeline.jsx b/src/web/Pipeline.jsx deleted file mode 100644 index 8032dc3eb2cc..000000000000 --- a/src/web/Pipeline.jsx +++ /dev/null @@ -1,55 +0,0 @@ -import React, {PropTypes} from 'react'; -import {Row, Col, Button} from 'react-bootstrap'; - -import AceEditor from 'react-ace'; -import brace from 'brace'; - -import 'brace/mode/text'; -import 'brace/theme/chrome'; - -import PipelineForm from 'PipelineForm'; - -const Pipeline = React.createClass({ - propTypes: { - pipeline: PropTypes.object.isRequired, - }, - - _delete() { - this.props.delete(this.props.pipeline); - }, - - render() { - return
  • -

    {this.props.pipeline.title}

    - - {this.props.pipeline.description} - - -
    - -
    - - - - - -     -
    • ; - } -}); - -export default Pipeline; \ No newline at end of file diff --git a/src/web/PipelineForm.jsx b/src/web/PipelineForm.jsx deleted file mode 100644 index 6ff910ab906d..000000000000 --- a/src/web/PipelineForm.jsx +++ /dev/null @@ -1,179 +0,0 @@ -import React, { PropTypes } from 'react'; - -import { Row, Col } from 'react-bootstrap'; -import { Input } from 'react-bootstrap'; - -import AceEditor from 'react-ace'; -import brace from 'brace'; - -import 'brace/mode/text'; -import 'brace/theme/chrome'; - -import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; - -const PipelineForm = React.createClass({ - parseTimer: undefined, - propTypes: { - pipeline: PropTypes.object, - create: React.PropTypes.bool, - save: React.PropTypes.func.isRequired, - validatePipeline: React.PropTypes.func.isRequired, - }, - - getDefaultProps() { - return { - pipeline: { - id: '', - title: '', - description: '', - source: '', - }, - } - }, - - getInitialState() { - const pipeline = this.props.pipeline; - return { - // when editing, take the pipeline that's been passed in - pipeline: { - id: pipeline.id, - title: pipeline.title, - description: pipeline.description, - source: pipeline.source - }, - editor: undefined, - parseErrors: [], - } - }, - - componentWillUnmount() { - if (this.parseTimer !== undefined) { - clearTimeout(this.parseTimer); - this.parseTimer = undefined; - } - }, - - openModal() { - this.refs.modal.open(); - }, - - _updateEditor() { - const session = this.state.editor.session; - const annotations = this.state.parseErrors.map(e => { - return {row: e.line - 1, column: e.position_in_line - 1, text: e.reason, type: "error"} - }); - session.setAnnotations(annotations); - }, - - _setParseErrors(errors) { - this.setState({parseErrors: errors}, this._updateEditor); - }, - - _onSourceChange(value) { - // don't try to parse the previous value, gets reset below - if (this.parseTimer !== undefined) { - clearTimeout(this.parseTimer); - } - const pipeline = this.state.pipeline; - pipeline.source = value; - this.setState({pipeline: pipeline}); - - if (this.props.validatePipeline) { - // have the caller validate the pipeline after typing stopped for a while. usually this will mean send to server to parse - this.parseTimer = setTimeout(() => this.props.validatePipeline(pipeline, this._setParseErrors), 500); - } - }, - - _onDescriptionChange(event) { - const pipeline = this.state.pipeline; - pipeline.description = event.target.value; - this.setState({pipeline: pipeline}); - }, - - _onTitleChange(event) { - const pipeline = this.state.pipeline; - pipeline.title = event.target.value; - this.setState({pipeline: pipeline}); - }, - - _onLoad(editor) { - this.setState({editor: editor}); - }, - - _getId(prefixIdName) { - return this.state.name !== undefined ? prefixIdName + this.state.name : prefixIdName; - }, - - _closeModal() { - this.refs.modal.close(); - }, - - _saved() { - this._closeModal(); - if (this.props.create) { - this.setState({pipeline: {}}); - } - }, - - _save() { - if (this.state.parseErrors.length === 0) { - this.props.save(this.state.pipeline, this._saved); - } - }, - - render() { - let triggerButtonContent; - if (this.props.create) { - triggerButtonContent = 'Create pipeline'; - } else { - triggerButtonContent = Edit; - } - return ( - - - -
    - - - - -
    - -
    -
    -     -     - ); - }, -}); - -export default PipelineForm; \ No newline at end of file diff --git a/src/web/PipelineStreamComponent.jsx b/src/web/PipelineStreamComponent.jsx deleted file mode 100644 index e11306c7e5eb..000000000000 --- a/src/web/PipelineStreamComponent.jsx +++ /dev/null @@ -1,26 +0,0 @@ -import React, {PropTypes} from 'react'; -import {Row, Col, Button} from 'react-bootstrap'; - -const PipelineStreamComponent = React.createClass({ - propTypes: { - pipelines: PropTypes.array.isRequired, - streams: PropTypes.array.isRequired, - assignments: PropTypes.array.isRequired, - }, - render() { - let streamAssignments = [ -
  • Todo
    • - ]; - return ( - - -

    Stream binding

    -
      - {streamAssignments} -
    - -     ); - }, -}); - -export default PipelineStreamComponent; \ No newline at end of file diff --git a/src/web/PipelinesComponent.jsx b/src/web/PipelinesComponent.jsx deleted file mode 100644 index 91b7a0ebcd7c..000000000000 --- a/src/web/PipelinesComponent.jsx +++ /dev/null @@ -1,75 +0,0 @@ -import React, {PropTypes} from 'react'; -import Reflux from 'reflux'; -import { Row, Col } from 'react-bootstrap'; -import { Input, Alert } from 'react-bootstrap'; - -import PipelinesActions from 'PipelinesActions'; -import PipelineForm from 'PipelineForm'; -import Pipeline from 'Pipeline'; - -const PipelinesComponent = React.createClass({ - propTypes: { - pipelines: PropTypes.array.isRequired, - }, - - _formatPipeline(pipeline) { - return ; - }, - - _sortByTitle(pipeline1, pipeline2) { - return pipeline1.title.localeCompare(pipeline2.title); - }, - - _save(pipeline, callback) { - console.log(pipeline); - if (pipeline.id) { - PipelinesActions.update(pipeline); - } else { - PipelinesActions.save(pipeline); - } - callback(); - }, - - _delete(pipeline) { - PipelinesActions.delete(pipeline.id); - }, - - _validatePipeline(pipeline, setErrorsCb) { - PipelinesActions.parse(pipeline, setErrorsCb); - }, - - render() { - let pipelines; - if (this.props.pipelines.length == 0) { - pipelines = - -   No pipelines configured. - - } else { - pipelines = this.props.pipelines.sort(this._sortByTitle).map(this._formatPipeline); - } - - return ( - - -

    Pipelines

    -
      - {pipelines} -
    - - -     - ); - }, -}); - -export default PipelinesComponent; \ No newline at end of file diff --git a/src/web/index.jsx b/src/web/index.jsx index 40008210e31c..68846ea25ff2 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -1,13 +1,13 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; -import PipelinesOverviewPage from './pipelines/PipelinesOverviewPage'; -import PipelineDetailsPage from './pipelines/PipelineDetailsPage'; -import PipelinesPage from './PipelinesPage'; -import RulesPage from './rules/RulesPage'; +import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage'; +import PipelineDetailsPage from 'pipelines/PipelineDetailsPage'; +import PipelinesInputsPage from 'pipelines/PipelinesInputsPage'; +import RulesPage from 'rules/RulesPage'; PluginStore.register(new PluginManifest(packageJson, { routes: [ - {path: '/system/pipelines', component: PipelinesPage}, + {path: '/system/pipelines', component: PipelinesInputsPage}, {path: '/system/pipelines/overview', component: PipelinesOverviewPage}, {path: '/system/pipelines/rules', component: RulesPage}, {path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage}, diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index 343b402a61be..710cfdd1dd14 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -8,8 +8,8 @@ import Pipeline from './Pipeline'; import SourceGenerator from 'logic/SourceGenerator'; import ObjectUtils from 'util/ObjectUtils'; -import PipelinesActions from 'PipelinesActions'; -import PipelinesStore from 'PipelinesStore'; +import PipelinesActions from 'pipelines/PipelinesActions'; +import PipelinesStore from 'pipelines/PipelinesStore'; import RulesStore from 'rules/RulesStore'; function filterPipeline(state) { diff --git a/src/web/PipelinesActions.jsx b/src/web/pipelines/PipelinesActions.jsx similarity index 100% rename from src/web/PipelinesActions.jsx rename to src/web/pipelines/PipelinesActions.jsx diff --git a/src/web/PipelinesPage.jsx b/src/web/pipelines/PipelinesInputsPage.jsx similarity index 79% rename from src/web/PipelinesPage.jsx rename to src/web/pipelines/PipelinesInputsPage.jsx index 84674d2b0db3..2540e850231c 100644 --- a/src/web/PipelinesPage.jsx +++ b/src/web/pipelines/PipelinesInputsPage.jsx @@ -9,19 +9,18 @@ import PageHeader from 'components/common/PageHeader'; import DocumentationLink from 'components/support/DocumentationLink'; import Spinner from 'components/common/Spinner'; -import PipelineStreamComponent from 'PipelineStreamComponent'; +import PipelinesActions from 'pipelines/PipelinesActions'; +import PipelinesStore from 'pipelines/PipelinesStore'; -import PipelinesActions from 'PipelinesActions'; -import PipelinesStore from 'PipelinesStore'; -import PipelinesComponent from 'PipelinesComponent'; +const PipelinesInputsPage = React.createClass({ + contextTypes: { + storeProvider: PropTypes.object, + }, -const PipelinesPage = React.createClass({ mixins: [ Reflux.connect(PipelinesStore), ], - contextTypes: { - storeProvider: React.PropTypes.object, - }, + getInitialState() { return { pipelines: undefined, @@ -33,7 +32,7 @@ const PipelinesPage = React.createClass({ componentDidMount() { PipelinesActions.list(); - var store = this.context.storeProvider.getStore('Streams'); + const store = this.context.storeProvider.getStore('Streams'); store.listStreams().then((streams) => this.setState({streams: streams})); }, @@ -42,9 +41,7 @@ const PipelinesPage = React.createClass({ if (!this.state.pipelines || !this.state.streams) { content = ; } else { - content = [ - - ]; + content = []; } return ( @@ -64,9 +61,14 @@ const PipelinesPage = React.createClass({ - {content} + + + + {content} + + ); }, }); -export default PipelinesPage; +export default PipelinesInputsPage; diff --git a/src/web/PipelinesStore.js b/src/web/pipelines/PipelinesStore.js similarity index 98% rename from src/web/PipelinesStore.js rename to src/web/pipelines/PipelinesStore.js index ccbbbb3cafe1..76580f20272e 100644 --- a/src/web/PipelinesStore.js +++ b/src/web/pipelines/PipelinesStore.js @@ -1,6 +1,6 @@ import Reflux from 'reflux'; -import PipelinesActions from 'PipelinesActions'; +import PipelinesActions from 'pipelines/PipelinesActions'; import UserNotification from 'util/UserNotification'; import URLUtils from 'util/URLUtils'; diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index b7e65dbdf3e2..a6ece1aa479e 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -7,8 +7,8 @@ import naturalSort from 'javascript-natural-sort'; import { DataTable, Spinner } from 'components/common'; import PipelineForm from './PipelineForm'; -import PipelinesActions from 'PipelinesActions'; -import PipelinesStore from 'PipelinesStore'; +import PipelinesActions from 'pipelines/PipelinesActions'; +import PipelinesStore from 'pipelines/PipelinesStore'; import ObjectUtils from 'util/ObjectUtils'; import SourceGenerator from 'logic/SourceGenerator'; From 151f9e66f0b67cd5fa2c423241f50a37e00fcd60 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 16:37:56 +0100 Subject: [PATCH 138/521] Rename pipeline assignments to connections --- .../PipelineProcessorModule.java | 4 +- ... => PipelineStreamConnectionsService.java} | 43 ++++++++++--------- .../processors/PipelineInterpreter.java | 40 ++++++++--------- ....java => PipelineConnectionsResource.java} | 42 +++++++++--------- ...ent.java => PipelineStreamConnection.java} | 8 ++-- .../processors/PipelineInterpreterTest.java | 14 +++--- 6 files changed, 76 insertions(+), 75 deletions(-) rename src/main/java/org/graylog/plugins/pipelineprocessor/db/{PipelineStreamAssignmentService.java => PipelineStreamConnectionsService.java} (59%) rename src/main/java/org/graylog/plugins/pipelineprocessor/rest/{PipelineStreamResource.java => PipelineConnectionsResource.java} (61%) rename src/main/java/org/graylog/plugins/pipelineprocessor/rest/{PipelineStreamAssignment.java => PipelineStreamConnection.java} (90%) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 0d67b8279371..ec8c5493c60a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -19,7 +19,7 @@ import org.graylog.plugins.pipelineprocessor.functions.ProcessorFunctionsModule; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamResource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnectionsResource; import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; @@ -40,7 +40,7 @@ protected void configure() { addMessageProcessor(PipelineInterpreter.class); addRestResource(RuleResource.class); addRestResource(PipelineResource.class); - addRestResource(PipelineStreamResource.class); + addRestResource(PipelineConnectionsResource.class); install(new ProcessorFunctionsModule()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java similarity index 59% rename from src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java index f698189e795f..c8e66b5834d2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamAssignmentService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java @@ -17,8 +17,9 @@ package org.graylog.plugins.pipelineprocessor.db; import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; import com.mongodb.MongoException; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; @@ -34,50 +35,50 @@ import java.util.Collections; import java.util.Set; -public class PipelineStreamAssignmentService { - private static final Logger log = LoggerFactory.getLogger(PipelineStreamAssignmentService.class); +public class PipelineStreamConnectionsService { + private static final Logger log = LoggerFactory.getLogger(PipelineStreamConnectionsService.class); public static final String COLLECTION = "pipeline_processor_pipelines_streams"; - private final JacksonDBCollection dbCollection; + private final JacksonDBCollection dbCollection; @Inject - public PipelineStreamAssignmentService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + public PipelineStreamConnectionsService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { dbCollection = JacksonDBCollection.wrap( mongoConnection.getDatabase().getCollection(COLLECTION), - PipelineStreamAssignment.class, + PipelineStreamConnection.class, String.class, mapper.get()); - dbCollection.createIndex(DBSort.asc("stream_id")); + dbCollection.createIndex(DBSort.asc("stream_id"), new BasicDBObject("unique", true)); } - public PipelineStreamAssignment save(PipelineStreamAssignment assignment) { - PipelineStreamAssignment existingAssignment = dbCollection.findOne(DBQuery.is("stream_id", assignment.streamId())); - if (existingAssignment == null) { - existingAssignment = PipelineStreamAssignment.create(null, assignment.streamId(), Collections.emptySet()); + public PipelineStreamConnection save(PipelineStreamConnection connections) { + PipelineStreamConnection existingConnections = dbCollection.findOne(DBQuery.is("stream_id", connections.streamId())); + if (existingConnections == null) { + existingConnections = PipelineStreamConnection.create(null, connections.streamId(), Collections.emptySet()); } - final PipelineStreamAssignment toSave = existingAssignment.toBuilder() - .pipelineIds(assignment.pipelineIds()).build(); - final WriteResult save = dbCollection.save(toSave); + final PipelineStreamConnection toSave = existingConnections.toBuilder() + .pipelineIds(connections.pipelineIds()).build(); + final WriteResult save = dbCollection.save(toSave); return save.getSavedObject(); } - public PipelineStreamAssignment load(String streamId) throws NotFoundException { - final PipelineStreamAssignment oneById = dbCollection.findOne(DBQuery.is("stream_id", streamId)); + public PipelineStreamConnection load(String streamId) throws NotFoundException { + final PipelineStreamConnection oneById = dbCollection.findOne(DBQuery.is("stream_id", streamId)); if (oneById == null) { - throw new NotFoundException("No pipeline assignments with for stream " + streamId); + throw new NotFoundException("No pipeline connections with for stream " + streamId); } return oneById; } - public Set loadAll() { + public Set loadAll() { try { - final DBCursor assignments = dbCollection.find(); - return Sets.newHashSet(assignments.iterator()); + final DBCursor connections = dbCollection.find(); + return Sets.newHashSet(connections.iterator()); } catch (MongoException e) { - log.error("Unable to load pipelines", e); + log.error("Unable to load pipeline connections", e); return Collections.emptySet(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 0364cff45de1..91db60221e4b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -35,14 +35,14 @@ import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.db.PipelineService; -import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; @@ -75,19 +75,19 @@ public class PipelineInterpreter implements MessageProcessor { private final RuleService ruleService; private final PipelineService pipelineService; - private final PipelineStreamAssignmentService pipelineStreamAssignmentService; + private final PipelineStreamConnectionsService pipelineStreamConnectionsService; private final PipelineRuleParser pipelineRuleParser; private final Journal journal; private final ScheduledExecutorService scheduler; private final Meter filteredOutMessages; private final AtomicReference> currentPipelines = new AtomicReference<>(ImmutableMap.of()); - private final AtomicReference> streamPipelineAssignments = new AtomicReference<>(ImmutableSetMultimap.of()); + private final AtomicReference> streamPipelineConnections = new AtomicReference<>(ImmutableSetMultimap.of()); @Inject public PipelineInterpreter(RuleService ruleService, PipelineService pipelineService, - PipelineStreamAssignmentService pipelineStreamAssignmentService, + PipelineStreamConnectionsService pipelineStreamConnectionsService, PipelineRuleParser pipelineRuleParser, Journal journal, MetricRegistry metricRegistry, @@ -95,14 +95,14 @@ public PipelineInterpreter(RuleService ruleService, @ClusterEventBus EventBus clusterBus) { this.ruleService = ruleService; this.pipelineService = pipelineService; - this.pipelineStreamAssignmentService = pipelineStreamAssignmentService; + this.pipelineStreamConnectionsService = pipelineStreamConnectionsService; this.pipelineRuleParser = pipelineRuleParser; this.journal = journal; this.scheduler = scheduler; this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); - // listens to cluster wide Rule, Pipeline and pipeline stream assignment changes + // listens to cluster wide Rule, Pipeline and pipeline stream connection changes clusterBus.register(this); reload(); @@ -155,15 +155,15 @@ private synchronized void reload() { }); currentPipelines.set(ImmutableMap.copyOf(pipelineIdMap)); - // read all stream assignments of those pipelines to allow processing messages through them - final HashMultimap assignments = HashMultimap.create(); - for (PipelineStreamAssignment streamAssignment : pipelineStreamAssignmentService.loadAll()) { - streamAssignment.pipelineIds().stream() + // read all stream connections of those pipelines to allow processing messages through them + final HashMultimap connections = HashMultimap.create(); + for (PipelineStreamConnection streamConnection : pipelineStreamConnectionsService.loadAll()) { + streamConnection.pipelineIds().stream() .map(pipelineIdMap::get) .filter(Objects::nonNull) - .forEach(pipeline -> assignments.put(streamAssignment.streamId(), pipeline)); + .forEach(pipeline -> connections.put(streamConnection.streamId(), pipeline)); } - streamPipelineAssignments.set(ImmutableSetMultimap.copyOf(assignments)); + streamPipelineConnections.set(ImmutableSetMultimap.copyOf(connections)); } @@ -194,7 +194,7 @@ public Messages process(Messages messages) { // this makes a copy of the list! final Set initialStreamIds = message.getStreams().stream().map(Stream::getId).collect(Collectors.toSet()); - final ImmutableSetMultimap streamAssignment = streamPipelineAssignments.get(); + final ImmutableSetMultimap streamConnection = streamPipelineConnections.get(); if (initialStreamIds.isEmpty()) { if (processingBlacklist.contains(tuple(msgId, "default"))) { @@ -202,8 +202,8 @@ public Messages process(Messages messages) { pipelinesToRun = ImmutableSet.of(); log.debug("[{}] already processed default stream, skipping", msgId); } else { - // get the default stream pipeline assignments for this message - pipelinesToRun = streamAssignment.get("default"); + // get the default stream pipeline connections for this message + pipelinesToRun = streamConnection.get("default"); log.debug("[{}] running default stream pipelines: [{}]", msgId, pipelinesToRun.stream().map(Pipeline::name).toArray()); @@ -212,10 +212,10 @@ public Messages process(Messages messages) { // 2. if a message-stream combination has already been processed (is in the set), skip that execution final Set streamsIds = initialStreamIds.stream() .filter(streamId -> !processingBlacklist.contains(tuple(msgId, streamId))) - .filter(streamAssignment::containsKey) + .filter(streamConnection::containsKey) .collect(Collectors.toSet()); pipelinesToRun = ImmutableSet.copyOf(streamsIds.stream() - .flatMap(streamId -> streamAssignment.get(streamId).stream()) + .flatMap(streamId -> streamConnection.get(streamId).stream()) .collect(Collectors.toSet())); log.debug("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); } @@ -337,8 +337,8 @@ public void handlePipelineChanges(PipelinesChangedEvent event) { } @Subscribe - public void handlePipelineAssignmentChanges(PipelineStreamAssignment assignment) { - log.debug("Pipeline stream assignment changed: {}", assignment); + public void handlePipelineConnectionChanges(PipelineStreamConnection connection) { + log.debug("Pipeline stream connection changed: {}", connection); scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java similarity index 61% rename from src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index 7f9a45da43fd..1a8a0bba6289 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -21,7 +21,7 @@ import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; import org.graylog.plugins.pipelineprocessor.db.PipelineService; -import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog2.database.NotFoundException; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.rest.PluginRestResource; @@ -39,56 +39,56 @@ import javax.ws.rs.core.MediaType; import java.util.Set; -@Api(value = "Pipelines/Streams", description = "Stream assignment of processing pipelines") -@Path("/system/pipelines/streams") +@Api(value = "Pipelines/Connections", description = "Stream connections of processing pipelines") +@Path("/system/pipelines/connections") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) -public class PipelineStreamResource extends RestResource implements PluginRestResource { +public class PipelineConnectionsResource extends RestResource implements PluginRestResource { - private final PipelineStreamAssignmentService assignmentService; + private final PipelineStreamConnectionsService connectionsService; private final PipelineService pipelineService; private final StreamService streamService; private final EventBus clusterBus; @Inject - public PipelineStreamResource(PipelineStreamAssignmentService assignmentService, - PipelineService pipelineService, - StreamService streamService, - @ClusterEventBus EventBus clusterBus) { - this.assignmentService = assignmentService; + public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsService, + PipelineService pipelineService, + StreamService streamService, + @ClusterEventBus EventBus clusterBus) { + this.connectionsService = connectionsService; this.pipelineService = pipelineService; this.streamService = streamService; this.clusterBus = clusterBus; } - @ApiOperation(value = "Attach a processing pipeline to a stream", notes = "") + @ApiOperation(value = "Connect processing pipelines to a stream", notes = "") @POST - public PipelineStreamAssignment assignPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineStreamAssignment assignment) throws NotFoundException { - final String streamId = assignment.streamId(); + public PipelineStreamConnection connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineStreamConnection connection) throws NotFoundException { + final String streamId = connection.streamId(); // the default stream doesn't exist as an entity if (!streamId.equalsIgnoreCase("default")) { streamService.load(streamId); } // verify the pipelines exist - for (String s : assignment.pipelineIds()) { + for (String s : connection.pipelineIds()) { pipelineService.load(s); } - final PipelineStreamAssignment save = assignmentService.save(assignment); + final PipelineStreamConnection save = connectionsService.save(connection); clusterBus.post(save); return save; } - @ApiOperation("Get pipeline attachments for the given stream") + @ApiOperation("Get pipeline connections for the given stream") @GET @Path("/{streamId}") - public PipelineStreamAssignment getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { - return assignmentService.load(streamId); + public PipelineStreamConnection getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { + return connectionsService.load(streamId); } - @ApiOperation("Get all pipeline attachments") + @ApiOperation("Get all pipeline connections") @GET - public Set getAllAttachments() throws NotFoundException { - return assignmentService.loadAll(); + public Set getAll() throws NotFoundException { + return connectionsService.loadAll(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamConnection.java similarity index 90% rename from src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamConnection.java index 8b74bee4c21e..5d82b54bc6d3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamAssignment.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamConnection.java @@ -28,7 +28,7 @@ @AutoValue @JsonAutoDetect -public abstract class PipelineStreamAssignment { +public abstract class PipelineStreamConnection { @JsonProperty("id") @Nullable @@ -43,7 +43,7 @@ public abstract class PipelineStreamAssignment { public abstract Set pipelineIds(); @JsonCreator - public static PipelineStreamAssignment create(@Id @ObjectId @JsonProperty("id") @Nullable String _id, + public static PipelineStreamConnection create(@Id @ObjectId @JsonProperty("id") @Nullable String _id, @JsonProperty("stream_id") String streamId, @JsonProperty("pipeline_ids") Set pipelineIds) { return builder() @@ -54,14 +54,14 @@ public static PipelineStreamAssignment create(@Id @ObjectId @JsonProperty("id") } public static Builder builder() { - return new AutoValue_PipelineStreamAssignment.Builder(); + return new AutoValue_PipelineStreamConnection.Builder(); } public abstract Builder toBuilder(); @AutoValue.Builder public abstract static class Builder { - public abstract PipelineStreamAssignment build(); + public abstract PipelineStreamConnection build(); public abstract Builder _id(String _id); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index 60d5d9d232ec..1e2b59330532 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -23,14 +23,14 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.db.PipelineService; -import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; import org.graylog2.plugin.Tools; @@ -75,12 +75,12 @@ public void testCreateMessage() { null) )); - final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); - final PipelineStreamAssignment pipelineStreamAssignment = PipelineStreamAssignment.create(null, + final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(PipelineStreamConnectionsService.class); + final PipelineStreamConnection pipelineStreamConnection = PipelineStreamConnection.create(null, "default", newHashSet("cde")); - when(pipelineStreamAssignmentService.loadAll()).thenReturn( - newHashSet(pipelineStreamAssignment) + when(pipelineStreamConnectionsService.loadAll()).thenReturn( + newHashSet(pipelineStreamConnection) ); final Map> functions = Maps.newHashMap(); @@ -92,7 +92,7 @@ public void testCreateMessage() { final PipelineInterpreter interpreter = new PipelineInterpreter( ruleService, pipelineService, - pipelineStreamAssignmentService, + pipelineStreamConnectionsService, parser, mock(Journal.class), mock(MetricRegistry.class), From 4ebcc8e03ff9de21d7789aa1c7b5726e0bd71621 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 16:38:29 +0100 Subject: [PATCH 139/521] Remove unused get from rules store --- src/web/rules/RulesActions.jsx | 1 - src/web/rules/RulesStore.js | 4 ---- 2 files changed, 5 deletions(-) diff --git a/src/web/rules/RulesActions.jsx b/src/web/rules/RulesActions.jsx index f79106e1dc28..29935a45c1fa 100644 --- a/src/web/rules/RulesActions.jsx +++ b/src/web/rules/RulesActions.jsx @@ -3,7 +3,6 @@ import Reflux from 'reflux'; const RulesActions = Reflux.createActions({ 'delete': {asyncResult: true}, 'list': {asyncResult: true}, - 'get': {asyncResult: true}, 'save': {asyncResult: true}, 'update': {asyncResult: true}, 'parse': {asyncResult: true}, diff --git a/src/web/rules/RulesStore.js b/src/web/rules/RulesStore.js index 88a41312e5aa..90d87ff8bf99 100644 --- a/src/web/rules/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -29,10 +29,6 @@ const RulesStore = Reflux.createStore({ }, failCallback); }, - get(ruleId) { - - }, - save(ruleSource) { const failCallback = (error) => { UserNotification.error('Saving rule failed with status: ' + error.message, From 34e9565e4b10d43ccbe3edec503939bc01d0943e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 19:11:25 +0100 Subject: [PATCH 140/521] Add basic pipeline connections handling - List pipeline connections - Update connections - Connect pipelines to an stream without any --- src/web/index.jsx | 4 +- src/web/pipeline-connections/Connection.jsx | 62 ++++++++ .../pipeline-connections/ConnectionForm.jsx | 150 ++++++++++++++++++ .../PipelineConnections.jsx | 56 +++++++ .../PipelineConnectionsActions.js | 8 + .../PipelineConnectionsPage.jsx} | 44 +++-- .../PipelineConnectionsStore.js | 59 +++++++ src/web/pipelines/PipelinesStore.js | 4 + 8 files changed, 369 insertions(+), 18 deletions(-) create mode 100644 src/web/pipeline-connections/Connection.jsx create mode 100644 src/web/pipeline-connections/ConnectionForm.jsx create mode 100644 src/web/pipeline-connections/PipelineConnections.jsx create mode 100644 src/web/pipeline-connections/PipelineConnectionsActions.js rename src/web/{pipelines/PipelinesInputsPage.jsx => pipeline-connections/PipelineConnectionsPage.jsx} (53%) create mode 100644 src/web/pipeline-connections/PipelineConnectionsStore.js diff --git a/src/web/index.jsx b/src/web/index.jsx index 68846ea25ff2..a53b456f9bc2 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -2,12 +2,12 @@ import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage'; import PipelineDetailsPage from 'pipelines/PipelineDetailsPage'; -import PipelinesInputsPage from 'pipelines/PipelinesInputsPage'; +import PipelineConnectionsPage from 'pipeline-connections/PipelineConnectionsPage'; import RulesPage from 'rules/RulesPage'; PluginStore.register(new PluginManifest(packageJson, { routes: [ - {path: '/system/pipelines', component: PipelinesInputsPage}, + {path: '/system/pipelines', component: PipelineConnectionsPage}, {path: '/system/pipelines/overview', component: PipelinesOverviewPage}, {path: '/system/pipelines/rules', component: RulesPage}, {path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage}, diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx new file mode 100644 index 000000000000..59639cb14121 --- /dev/null +++ b/src/web/pipeline-connections/Connection.jsx @@ -0,0 +1,62 @@ +import React from 'react'; +import { Col } from 'react-bootstrap'; + +import { DataTable, EntityListItem } from 'components/common'; +import ConnectionForm from './ConnectionForm'; + +const Connection = React.createClass({ + propTypes: { + stream: React.PropTypes.object.isRequired, + pipelines: React.PropTypes.array.isRequired, + onUpdate: React.PropTypes.func.isRequired, + }, + + _pipelineHeaderFormatter(header) { + return {header}; + }, + + _pipelineRowFormatter(pipeline) { + return ( + + {pipeline.title} + {pipeline.description} + + ); + }, + + _formatPipelines(pipelines) { + const headers = ['Title', 'Description']; + + return ( + + ); + }, + + render() { + const actions = ( + + ); + + const content = ( + + {this._formatPipelines(this.props.pipelines)} + + ); + + return ( + + ); + }, +}); + +export default Connection; diff --git a/src/web/pipeline-connections/ConnectionForm.jsx b/src/web/pipeline-connections/ConnectionForm.jsx new file mode 100644 index 000000000000..bdb8cdb30012 --- /dev/null +++ b/src/web/pipeline-connections/ConnectionForm.jsx @@ -0,0 +1,150 @@ +import React, { PropTypes } from 'react'; +import Reflux from 'reflux'; +import { Input, Button } from 'react-bootstrap'; + +import { Select, SelectableList } from 'components/common'; +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; +import ObjectUtils from 'util/ObjectUtils'; + +import PipelinesStore from 'pipelines/PipelinesStore'; + +const ConnectionForm = React.createClass({ + propTypes: { + stream: PropTypes.object, + connection: PropTypes.object, + streams: PropTypes.array, + create: PropTypes.bool, + save: PropTypes.func.isRequired, + validateConnection: PropTypes.func.isRequired, + }, + + mixins: [Reflux.connect(PipelinesStore)], + + getDefaultProps() { + return { + streams: [], + connection: { + stream: undefined, + pipelines: [], + }, + }; + }, + + getInitialState() { + const connection = ObjectUtils.clone(this.props.connection); + return { + // when editing, take the connection that's been passed in + connection: { + stream: connection.stream, + pipelines: this._getFormattedOptions(connection.pipelines), + }, + }; + }, + + openModal() { + this.refs.modal.open(); + }, + + _onStreamChange(newStream) { + const connection = ObjectUtils.clone(this.state.connection); + connection.stream = this.props.streams.filter(s => s.id === newStream)[0]; + this.setState({connection: connection}); + }, + + _onConnectionsChange(newRules) { + const connection = ObjectUtils.clone(this.state.connection); + connection.pipelines = newRules; + this.setState({connection: connection}); + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _saved() { + this._closeModal(); + if (this.props.create) { + this.setState(this.getInitialState()); + } + }, + + _save() { + const connection = this.state.connection; + const filteredConnection = {}; + filteredConnection.stream = connection.stream.id; + filteredConnection.pipelines = connection.pipelines.map(p => p.value); + this.props.save(filteredConnection, this._saved); + }, + + _getFormattedStreams(streams) { + if (!streams) { + return []; + } + + return streams.map(s => { + return {value: s.id, label: s.title}; + }); + }, + + _getFormattedOptions(pipelines) { + if (!pipelines) { + return []; + } + + return pipelines + .map(pipeline => { + return {value: pipeline.id, label: pipeline.title}; + }); + }, + + _getFilteredFormattedOptions(pipelines) { + return this._getFormattedOptions(pipelines) + // Remove already selected options + .filter(pipeline => !this.state.connection.pipelines.some(p => p.value === pipeline.value)); + }, + + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Add new connection'; + } else { + triggerButtonContent = Edit connections; + } + + let streamSelector; + if (this.props.create) { + streamSelector = ( + + + + +
    +     + + ); + }, +}); + +export default ConnectionForm; diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx new file mode 100644 index 000000000000..dd6903b9b0b0 --- /dev/null +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -0,0 +1,56 @@ +import React from 'react'; +import { Row, Col } from 'react-bootstrap'; + +import { EntityList, TypeAheadDataFilter } from 'components/common'; +import Connection from './Connection'; +import ConnectionForm from './ConnectionForm'; + +const PipelineConnections = React.createClass({ + propTypes: { + pipelines: React.PropTypes.array.isRequired, + streams: React.PropTypes.array.isRequired, + connections: React.PropTypes.array.isRequired, + onConnectionsChange: React.PropTypes.func.isRequired, + }, + + _updateConnection(newConnection, callback) { + this.props.onConnectionsChange(newConnection, callback); + }, + + render() { + // TODO: Sort this list by stream title + const formattedConnections = this.props.connections.map(c => { + return ( + s.id === c.stream_id)[0]} + pipelines={this.props.pipelines.filter(p => c.pipeline_ids.indexOf(p.id) !== -1)} + onUpdate={this._updateConnection}/> + ); + }); + + const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id)); + + return ( +
    + + + {}}/> + + +
    + +
    + +     + +
    + ); + }, +}); + +export default PipelineConnections; diff --git a/src/web/pipeline-connections/PipelineConnectionsActions.js b/src/web/pipeline-connections/PipelineConnectionsActions.js new file mode 100644 index 000000000000..9bd43ed44269 --- /dev/null +++ b/src/web/pipeline-connections/PipelineConnectionsActions.js @@ -0,0 +1,8 @@ +import Reflux from 'reflux'; + +const PipelineConnectionsActions = Reflux.createActions({ + 'list': {asyncResult: true}, + 'update': {asyncResult: true}, +}); + +export default PipelineConnectionsActions; diff --git a/src/web/pipelines/PipelinesInputsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx similarity index 53% rename from src/web/pipelines/PipelinesInputsPage.jsx rename to src/web/pipeline-connections/PipelineConnectionsPage.jsx index 2540e850231c..44ad56d79854 100644 --- a/src/web/pipelines/PipelinesInputsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -1,52 +1,64 @@ import React, {PropTypes} from 'react'; import Reflux from 'reflux'; - import { Button, Row, Col } from 'react-bootstrap'; - import { LinkContainer } from 'react-router-bootstrap'; -import PageHeader from 'components/common/PageHeader'; import DocumentationLink from 'components/support/DocumentationLink'; -import Spinner from 'components/common/Spinner'; +import { PageHeader, Spinner } from 'components/common'; +import PipelineConnections from './PipelineConnections'; import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; +import PipelineConnectionsActions from 'pipeline-connections/PipelineConnectionsActions'; +import PipelineConnectionsStore from 'pipeline-connections/PipelineConnectionsStore'; -const PipelinesInputsPage = React.createClass({ +const PipelineConnectionsPage = React.createClass({ contextTypes: { storeProvider: PropTypes.object, }, - mixins: [ - Reflux.connect(PipelinesStore), - ], + mixins: [Reflux.connect(PipelinesStore), Reflux.connect(PipelineConnectionsStore)], getInitialState() { return { - pipelines: undefined, streams: undefined, - assignments: [], - } + }; }, componentDidMount() { PipelinesActions.list(); + PipelineConnectionsActions.list(); const store = this.context.storeProvider.getStore('Streams'); store.listStreams().then((streams) => this.setState({streams: streams})); }, + _updateConnections(connections, callback) { + PipelineConnectionsActions.update(connections); + callback(); + }, + + _isLoading() { + return !this.state.pipelines || !this.state.streams || !this.state.connections; + }, + render() { let content; - if (!this.state.pipelines || !this.state.streams) { + if (this._isLoading()) { content = ; } else { - content = []; + content = ( + + ); } return ( - - Pipelines define how Graylog processes data by grouping rules into stages. Pipelines can apply to all incoming messages or only to messages on a certain stream. + + + Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} + as input for the different pipelines you configure. + Read more about Graylog pipelines in the . @@ -71,4 +83,4 @@ const PipelinesInputsPage = React.createClass({ }, }); -export default PipelinesInputsPage; +export default PipelineConnectionsPage; diff --git a/src/web/pipeline-connections/PipelineConnectionsStore.js b/src/web/pipeline-connections/PipelineConnectionsStore.js new file mode 100644 index 000000000000..ac4163ce49f5 --- /dev/null +++ b/src/web/pipeline-connections/PipelineConnectionsStore.js @@ -0,0 +1,59 @@ +import Reflux from 'reflux'; + +import UserNotification from 'util/UserNotification'; +import URLUtils from 'util/URLUtils'; +import fetch from 'logic/rest/FetchProvider'; + +import PipelineConnectionsActions from './PipelineConnectionsActions'; + +const urlPrefix = '/plugins/org.graylog.plugins.pipelineprocessor'; + +const PipelineConnectionsStore = Reflux.createStore({ + listenables: [PipelineConnectionsActions], + connections: undefined, + + getInitialState() { + return {connections: this.connections}; + }, + + list() { + const failCallback = (error) => { + UserNotification.error('Fetching pipeline connections failed with status: ' + error.message, + 'Could not retrieve pipeline connections'); + }; + + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/connections'); + const promise = fetch('GET', url); + promise.then((response) => { + this.connections = response; + this.trigger({connections: response}); + }, failCallback); + }, + + update(connection) { + const failCallback = (error) => { + UserNotification.error('Updating pipeline connections failed with status: ' + error.message, + 'Could not update pipeline connections'); + }; + const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/connections'); + const updatedConnection = { + stream_id: connection.stream, + pipeline_ids: connection.pipelines, + }; + const promise = fetch('POST', url, updatedConnection); + promise.then( + response => { + if (this.connections.filter(c => c.stream_id === response.stream_id)[0]) { + this.connections = this.connections.map(c => (c.stream_id === response.stream_id ? response : c)); + } else { + this.connections.push(response); + } + + this.trigger({connections: this.connections}); + UserNotification.success(`Pipeline connections updated successfully`); + }, + failCallback); + }, +}); + +export default PipelineConnectionsStore; diff --git a/src/web/pipelines/PipelinesStore.js b/src/web/pipelines/PipelinesStore.js index 76580f20272e..79ded6de7f18 100644 --- a/src/web/pipelines/PipelinesStore.js +++ b/src/web/pipelines/PipelinesStore.js @@ -12,6 +12,10 @@ const PipelinesStore = Reflux.createStore({ listenables: [PipelinesActions], pipelines: undefined, + getInitialState() { + return {pipelines: this.pipelines}; + }, + _updatePipelinesState(pipeline) { if (!this.pipelines) { this.pipelines = [pipeline]; From 465f4f33f53b10034f89a7226b11d6443de0529e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 19:22:32 +0100 Subject: [PATCH 141/521] Change order of edit delete buttons on stages --- src/web/pipeline-connections/PipelineConnections.jsx | 2 ++ src/web/pipelines/Stage.jsx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index dd6903b9b0b0..6923f2d526fa 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -19,6 +19,7 @@ const PipelineConnections = React.createClass({ render() { // TODO: Sort this list by stream title + // TODO: Filter this list using the data filter const formattedConnections = this.props.connections.map(c => { return ( s.id === c.stream_id)[0]} @@ -29,6 +30,7 @@ const PipelineConnections = React.createClass({ const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id)); + // TODO: Add default stream return (
    diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index f11415c3c7ef..ce22e0abc6b3 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -49,8 +49,8 @@ const Stage = React.createClass({ const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules` )}`; const actions = [ + , , - , ]; let description; From d7e569aeac777719e22823eeeb929894aeba24f4 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 19:31:59 +0100 Subject: [PATCH 142/521] Make stream filter work --- .../PipelineConnections.jsx | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index 6923f2d526fa..2720d26f22c2 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -13,20 +13,31 @@ const PipelineConnections = React.createClass({ onConnectionsChange: React.PropTypes.func.isRequired, }, + getInitialState() { + return { + filteredStreams: this.props.streams.slice(), + }; + }, + + _updateFilteredStreams(filteredStreams) { + this.setState({filteredStreams: filteredStreams}); + }, + _updateConnection(newConnection, callback) { this.props.onConnectionsChange(newConnection, callback); }, render() { // TODO: Sort this list by stream title - // TODO: Filter this list using the data filter - const formattedConnections = this.props.connections.map(c => { - return ( - s.id === c.stream_id)[0]} - pipelines={this.props.pipelines.filter(p => c.pipeline_ids.indexOf(p.id) !== -1)} - onUpdate={this._updateConnection}/> - ); - }); + const formattedConnections = this.props.connections + .filter(c => this.state.filteredStreams.some(s => s.id === c.stream_id)) + .map(c => { + return ( + s.id === c.stream_id)[0]} + pipelines={this.props.pipelines.filter(p => c.pipeline_ids.indexOf(p.id) !== -1)} + onUpdate={this._updateConnection}/> + ); + }); const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id)); @@ -40,7 +51,7 @@ const PipelineConnections = React.createClass({ displayKey={'title'} filterSuggestions={[]} searchInKeys={['title', 'description']} - onDataFiltered={() => {}}/> + onDataFiltered={this._updateFilteredStreams}/>
    From df1545c62458573d4b11f1411c1eada931ec220b Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 19:32:07 +0100 Subject: [PATCH 143/521] Add button to streams on pipeline connections --- .../pipeline-connections/PipelineConnectionsPage.jsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 44ad56d79854..9afc29887211 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -7,6 +7,8 @@ import DocumentationLink from 'components/support/DocumentationLink'; import { PageHeader, Spinner } from 'components/common'; import PipelineConnections from './PipelineConnections'; +import Routes from 'routing/Routes'; + import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; import PipelineConnectionsActions from 'pipeline-connections/PipelineConnectionsActions'; @@ -54,7 +56,7 @@ const PipelineConnectionsPage = React.createClass({ } return ( - + Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} as input for the different pipelines you configure. @@ -64,10 +66,14 @@ const PipelineConnectionsPage = React.createClass({ + + + +   - {' '} +   From e432701f28e581817667bcef2f0b1906768d968e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 3 Mar 2016 19:35:03 +0100 Subject: [PATCH 144/521] Add more details to stream connection --- src/web/pipeline-connections/Connection.jsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx index 59639cb14121..2129b6a6be74 100644 --- a/src/web/pipeline-connections/Connection.jsx +++ b/src/web/pipeline-connections/Connection.jsx @@ -53,6 +53,8 @@ const Connection = React.createClass({ return ( ); From f6011c4d914a6faeaeec8a6f524692399b404e9b Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 12:39:03 +0100 Subject: [PATCH 145/521] fix benchmark after renames and server changes --- .../benchmarks/pipeline/PipelineBenchmark.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index fc0795a9be36..262ebb5d5992 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -26,7 +26,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.db.PipelineService; -import org.graylog.plugins.pipelineprocessor.db.PipelineStreamAssignmentService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; @@ -34,7 +34,7 @@ import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamAssignment; +import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; import org.graylog2.Configuration; import org.graylog2.database.NotFoundException; import org.graylog2.filters.ExtractorFilter; @@ -53,7 +53,6 @@ import org.graylog2.plugin.filters.MessageFilter; import org.graylog2.rules.DroolsEngine; import org.graylog2.shared.journal.Journal; -import org.graylog2.shared.stats.ThroughputStats; import org.graylog2.streams.StreamFaultManager; import org.graylog2.streams.StreamMetrics; import org.graylog2.streams.StreamRouter; @@ -118,12 +117,12 @@ public InterpreterState() { null) )); - final PipelineStreamAssignmentService pipelineStreamAssignmentService = mock(PipelineStreamAssignmentService.class); - final PipelineStreamAssignment pipelineStreamAssignment = PipelineStreamAssignment.create(null, + final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(PipelineStreamConnectionsService.class); + final PipelineStreamConnection pipelineStreamConnection = PipelineStreamConnection.create(null, "default", newHashSet("cde")); - when(pipelineStreamAssignmentService.loadAll()).thenReturn( - newHashSet(pipelineStreamAssignment) + when(pipelineStreamConnectionsService.loadAll()).thenReturn( + newHashSet(pipelineStreamConnection) ); final Map> functions = Maps.newHashMap(); @@ -135,7 +134,7 @@ public InterpreterState() { interpreter = new PipelineInterpreter( ruleService, pipelineService, - pipelineStreamAssignmentService, + pipelineStreamConnectionsService, parser, mock(Journal.class), mock(MetricRegistry.class), @@ -217,7 +216,7 @@ public void setup() { filters.add(new ExtractorFilter(inputService)); filters.add(new StaticFieldFilter(inputService)); - filters.add(new StreamMatcherFilter(streamRouter, mock(ThroughputStats.class))); + filters.add(new StreamMatcherFilter(streamRouter)); if (!noDrools) { filters.add(new RulesFilter(droolsEngine, filterService)); } From 5a628039b5ffbbf564a79cf44ad40cee9235938f Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 13:13:59 +0100 Subject: [PATCH 146/521] Correct stage modal title --- src/web/pipelines/Stage.jsx | 4 ++-- src/web/pipelines/StageForm.jsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index ce22e0abc6b3..e3e8b939e49a 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -46,11 +46,11 @@ const Stage = React.createClass({ render() { const stage = this.props.stage; - const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules` )}`; + const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules`)}`; const actions = [ , - , + , ]; let description; diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx index 00b354763406..8de11074054b 100644 --- a/src/web/pipelines/StageForm.jsx +++ b/src/web/pipelines/StageForm.jsx @@ -92,7 +92,7 @@ const StageForm = React.createClass({ {triggerButtonContent}
    From f0381a87addf168ff8418559d0e0eae20ab544ca Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 14:06:05 +0100 Subject: [PATCH 147/521] Rename PipelineStreamsConnection to PipelineConnections --- .../db/PipelineStreamConnectionsService.java | 24 +++++++++---------- .../processors/PipelineInterpreter.java | 6 ++--- ...nnection.java => PipelineConnections.java} | 12 +++++----- .../rest/PipelineConnectionsResource.java | 9 +++---- .../processors/PipelineInterpreterTest.java | 6 ++--- 5 files changed, 29 insertions(+), 28 deletions(-) rename src/main/java/org/graylog/plugins/pipelineprocessor/rest/{PipelineStreamConnection.java => PipelineConnections.java} (79%) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java index c8e66b5834d2..da4cbdd4fbb8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java @@ -19,7 +19,7 @@ import com.google.common.collect.Sets; import com.mongodb.BasicDBObject; import com.mongodb.MongoException; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; @@ -40,42 +40,42 @@ public class PipelineStreamConnectionsService { public static final String COLLECTION = "pipeline_processor_pipelines_streams"; - private final JacksonDBCollection dbCollection; + private final JacksonDBCollection dbCollection; @Inject public PipelineStreamConnectionsService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { dbCollection = JacksonDBCollection.wrap( mongoConnection.getDatabase().getCollection(COLLECTION), - PipelineStreamConnection.class, + PipelineConnections.class, String.class, mapper.get()); dbCollection.createIndex(DBSort.asc("stream_id"), new BasicDBObject("unique", true)); } - public PipelineStreamConnection save(PipelineStreamConnection connections) { - PipelineStreamConnection existingConnections = dbCollection.findOne(DBQuery.is("stream_id", connections.streamId())); + public PipelineConnections save(PipelineConnections connections) { + PipelineConnections existingConnections = dbCollection.findOne(DBQuery.is("stream_id", connections.streamId())); if (existingConnections == null) { - existingConnections = PipelineStreamConnection.create(null, connections.streamId(), Collections.emptySet()); + existingConnections = PipelineConnections.create(null, connections.streamId(), Collections.emptySet()); } - final PipelineStreamConnection toSave = existingConnections.toBuilder() + final PipelineConnections toSave = existingConnections.toBuilder() .pipelineIds(connections.pipelineIds()).build(); - final WriteResult save = dbCollection.save(toSave); + final WriteResult save = dbCollection.save(toSave); return save.getSavedObject(); } - public PipelineStreamConnection load(String streamId) throws NotFoundException { - final PipelineStreamConnection oneById = dbCollection.findOne(DBQuery.is("stream_id", streamId)); + public PipelineConnections load(String streamId) throws NotFoundException { + final PipelineConnections oneById = dbCollection.findOne(DBQuery.is("stream_id", streamId)); if (oneById == null) { throw new NotFoundException("No pipeline connections with for stream " + streamId); } return oneById; } - public Set loadAll() { + public Set loadAll() { try { - final DBCursor connections = dbCollection.find(); + final DBCursor connections = dbCollection.find(); return Sets.newHashSet(connections.iterator()); } catch (MongoException e) { log.error("Unable to load pipeline connections", e); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 91db60221e4b..69c424d228dc 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -42,7 +42,7 @@ import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; @@ -157,7 +157,7 @@ private synchronized void reload() { // read all stream connections of those pipelines to allow processing messages through them final HashMultimap connections = HashMultimap.create(); - for (PipelineStreamConnection streamConnection : pipelineStreamConnectionsService.loadAll()) { + for (PipelineConnections streamConnection : pipelineStreamConnectionsService.loadAll()) { streamConnection.pipelineIds().stream() .map(pipelineIdMap::get) .filter(Objects::nonNull) @@ -337,7 +337,7 @@ public void handlePipelineChanges(PipelinesChangedEvent event) { } @Subscribe - public void handlePipelineConnectionChanges(PipelineStreamConnection connection) { + public void handlePipelineConnectionChanges(PipelineConnections connection) { log.debug("Pipeline stream connection changed: {}", connection); scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamConnection.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java similarity index 79% rename from src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamConnection.java rename to src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java index 5d82b54bc6d3..b49b4ee95197 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineStreamConnection.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java @@ -28,7 +28,7 @@ @AutoValue @JsonAutoDetect -public abstract class PipelineStreamConnection { +public abstract class PipelineConnections { @JsonProperty("id") @Nullable @@ -43,9 +43,9 @@ public abstract class PipelineStreamConnection { public abstract Set pipelineIds(); @JsonCreator - public static PipelineStreamConnection create(@Id @ObjectId @JsonProperty("id") @Nullable String _id, - @JsonProperty("stream_id") String streamId, - @JsonProperty("pipeline_ids") Set pipelineIds) { + public static PipelineConnections create(@Id @ObjectId @JsonProperty("id") @Nullable String _id, + @JsonProperty("stream_id") String streamId, + @JsonProperty("pipeline_ids") Set pipelineIds) { return builder() ._id(_id) .streamId(streamId) @@ -54,14 +54,14 @@ public static PipelineStreamConnection create(@Id @ObjectId @JsonProperty("id") } public static Builder builder() { - return new AutoValue_PipelineStreamConnection.Builder(); + return new AutoValue_PipelineConnections.Builder(); } public abstract Builder toBuilder(); @AutoValue.Builder public abstract static class Builder { - public abstract PipelineStreamConnection build(); + public abstract PipelineConnections build(); public abstract Builder _id(String _id); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index 1a8a0bba6289..b11f3cbb4a4c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -37,6 +37,7 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; +import java.util.Collections; import java.util.Set; @Api(value = "Pipelines/Connections", description = "Stream connections of processing pipelines") @@ -63,7 +64,7 @@ public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsS @ApiOperation(value = "Connect processing pipelines to a stream", notes = "") @POST - public PipelineStreamConnection connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineStreamConnection connection) throws NotFoundException { + public PipelineConnections connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineConnections connection) throws NotFoundException { final String streamId = connection.streamId(); // the default stream doesn't exist as an entity if (!streamId.equalsIgnoreCase("default")) { @@ -73,7 +74,7 @@ public PipelineStreamConnection connectPipelines(@ApiParam(name = "Json body", r for (String s : connection.pipelineIds()) { pipelineService.load(s); } - final PipelineStreamConnection save = connectionsService.save(connection); + final PipelineConnections save = connectionsService.save(connection); clusterBus.post(save); return save; } @@ -81,13 +82,13 @@ public PipelineStreamConnection connectPipelines(@ApiParam(name = "Json body", r @ApiOperation("Get pipeline connections for the given stream") @GET @Path("/{streamId}") - public PipelineStreamConnection getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { + public PipelineConnections getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { return connectionsService.load(streamId); } @ApiOperation("Get all pipeline connections") @GET - public Set getAll() throws NotFoundException { + public Set getAll() throws NotFoundException { return connectionsService.loadAll(); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index 1e2b59330532..b165d82c75d7 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -30,7 +30,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; import org.graylog2.plugin.Message; import org.graylog2.plugin.Messages; import org.graylog2.plugin.Tools; @@ -76,11 +76,11 @@ public void testCreateMessage() { )); final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(PipelineStreamConnectionsService.class); - final PipelineStreamConnection pipelineStreamConnection = PipelineStreamConnection.create(null, + final PipelineConnections pipelineConnections = PipelineConnections.create(null, "default", newHashSet("cde")); when(pipelineStreamConnectionsService.loadAll()).thenReturn( - newHashSet(pipelineStreamConnection) + newHashSet(pipelineConnections) ); final Map> functions = Maps.newHashMap(); From 8c9ea22d94a3cc24f31d7502466a86e86c744785 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 14:06:21 +0100 Subject: [PATCH 148/521] Always include default stream in pipeline connections --- .../rest/PipelineConnectionsResource.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index b11f3cbb4a4c..ac62fc4a1468 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -89,7 +89,12 @@ public PipelineConnections getPipelinesForStream(@ApiParam(name = "streamId") @P @ApiOperation("Get all pipeline connections") @GET public Set getAll() throws NotFoundException { - return connectionsService.loadAll(); + Set pipelineConnections = connectionsService.loadAll(); + // to simplify clients, we always return the default stream, until we have it as a true entity + if (!pipelineConnections.stream().anyMatch(pc -> pc.streamId().equals("default"))) { + pipelineConnections.add(PipelineConnections.create(null, "default", Collections.emptySet())); + } + return pipelineConnections; } } From f7b190610479d8b83f40efa97c5d73b7bcdd2b46 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 14:07:47 +0100 Subject: [PATCH 149/521] Show create pipeline button when there are none --- src/web/pipelines/ProcessingTimelineComponent.jsx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index a6ece1aa479e..c7eb7c22a569 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -42,7 +42,7 @@ const ProcessingTimelineComponent = React.createClass({ style.width = 120; break; default: - // Nothing to see here + // Nothing to see here } return {header}; @@ -109,9 +109,12 @@ const ProcessingTimelineComponent = React.createClass({ if (this.state.pipelines.length === 0) { return ( - - There are no pipelines configured in your system. Create one to start processing your messages. - +
    +
    + + There are no pipelines configured in your system. Create one to start processing your messages. + +
    ); } From b421c545927af0af3d194d77e3283d0fb4863c5c Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 14:08:16 +0100 Subject: [PATCH 150/521] Include "dummy" data for default stream in frontend --- src/web/pipeline-connections/ConnectionForm.jsx | 3 ++- src/web/pipeline-connections/PipelineConnections.jsx | 1 - src/web/pipeline-connections/PipelineConnectionsPage.jsx | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/web/pipeline-connections/ConnectionForm.jsx b/src/web/pipeline-connections/ConnectionForm.jsx index bdb8cdb30012..3ed01376fcb6 100644 --- a/src/web/pipeline-connections/ConnectionForm.jsx +++ b/src/web/pipeline-connections/ConnectionForm.jsx @@ -128,7 +128,7 @@ const ConnectionForm = React.createClass({ {triggerButtonContent}
    @@ -138,6 +138,7 @@ const ConnectionForm = React.createClass({
    diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index 2720d26f22c2..90e1f49e935f 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -41,7 +41,6 @@ const PipelineConnections = React.createClass({ const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id)); - // TODO: Add default stream return (
    diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 9afc29887211..eb81d81ffb05 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -32,7 +32,10 @@ const PipelineConnectionsPage = React.createClass({ PipelineConnectionsActions.list(); const store = this.context.storeProvider.getStore('Streams'); - store.listStreams().then((streams) => this.setState({streams: streams})); + store.listStreams().then((streams) => { + streams.push({id: 'default', title: 'Incoming messages', description: 'Default stream of all incoming messages.'}); + this.setState({streams: streams}); + }); }, _updateConnections(connections, callback) { From 3458821698bce73d8dcbcd19511decedc57f2ba0 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 14:58:41 +0100 Subject: [PATCH 151/521] fix benchmark after renames and server changes --- .../org/graylog/benchmarks/pipeline/PipelineBenchmark.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java index 262ebb5d5992..494992472009 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/PipelineBenchmark.java @@ -34,7 +34,7 @@ import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; -import org.graylog.plugins.pipelineprocessor.rest.PipelineStreamConnection; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; import org.graylog2.Configuration; import org.graylog2.database.NotFoundException; import org.graylog2.filters.ExtractorFilter; @@ -118,7 +118,7 @@ public InterpreterState() { )); final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(PipelineStreamConnectionsService.class); - final PipelineStreamConnection pipelineStreamConnection = PipelineStreamConnection.create(null, + final PipelineConnections pipelineStreamConnection = PipelineConnections.create(null, "default", newHashSet("cde")); when(pipelineStreamConnectionsService.loadAll()).thenReturn( From f1eb7a2ace41cca4a6958f38b3d7c5b0b8ffcaaf Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 15:20:22 +0100 Subject: [PATCH 152/521] Update eslint options --- .eslintrc | 12 ++++-------- package.json | 8 +++----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.eslintrc b/.eslintrc index 270c9ac50855..12bf5d0b6b40 100644 --- a/.eslintrc +++ b/.eslintrc @@ -7,20 +7,16 @@ "extends": [ "eslint:recommended", "airbnb", - "import/warnings" ], "rules": { "no-else-return": 1, "no-nested-ternary": 1, "new-cap": 0, "indent": [2, 2, { "SwitchCase" : 1}], - "import/no-unresolved": 1, + "max-len" : 0, + "react/prefer-es6-class": 0, + "arrow-body-style": 0, + "react/jsx-indent-props": 0, }, - "settings": { - "import/resolve": { - "extensions": [".js", ".jsx", ".ts"], - "moduleDirectory": ["src", "node_modules", "public"] - } - } } diff --git a/package.json b/package.json index b9785b41b47f..371a64e2c039 100644 --- a/package.json +++ b/package.json @@ -31,12 +31,10 @@ "babel-core": "^5.8.25", "babel-loader": "^5.3.2", "css-loader": "^0.23.1", - "eslint": "^1.5.1", - "eslint-config-airbnb": "1.0.0", - "eslint-config-import": "^0.9.1", + "eslint": "^2.2.0", + "eslint-config-airbnb": "6.0.2", "eslint-loader": "^1.0.0", - "eslint-plugin-import": "^0.10.x", - "eslint-plugin-react": "^3.4.2", + "eslint-plugin-react": "^4.1.0", "graylog-web-manifests": "^2.0.0-alpha.3", "graylog-web-plugin": "~0.0.18", "json-loader": "^0.5.4", From cf37355296beb2b941355ae86ce22c4d064b1f74 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 15:51:44 +0100 Subject: [PATCH 153/521] added MessageProcessor.Descriptor to PipelineInterpreter --- .../pipelineprocessor/PipelineProcessorModule.java | 5 ++--- .../processors/PipelineInterpreter.java | 12 ++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index ec8c5493c60a..0043f838a7ca 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -18,8 +18,8 @@ import org.graylog.plugins.pipelineprocessor.functions.ProcessorFunctionsModule; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; -import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; import org.graylog.plugins.pipelineprocessor.rest.PipelineConnectionsResource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; @@ -36,8 +36,7 @@ public Set getConfigBeans() { @Override protected void configure() { - //addMessageProcessor(NaiveRuleProcessor.class); - addMessageProcessor(PipelineInterpreter.class); + addMessageProcessor(PipelineInterpreter.class, PipelineInterpreter.Descriptor.class); addRestResource(RuleResource.class); addRestResource(PipelineResource.class); addRestResource(PipelineConnectionsResource.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 69c424d228dc..71df1f85269d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -342,4 +342,16 @@ public void handlePipelineConnectionChanges(PipelineConnections connection) { scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } + public static class Descriptor implements MessageProcessor.Descriptor { + @Override + public String name() { + return "Pipeline Processor"; + } + + @Override + public String className() { + return PipelineInterpreter.class.getCanonicalName(); + } + } + } From e5c3df345d8345b07d2c337b4101c42c9d3d7b64 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 15:56:12 +0100 Subject: [PATCH 154/521] lower log levels, no need to be so noisy --- .../parser/PipelineRuleParser.java | 38 +++++++++---------- .../rest/PipelineResource.java | 2 +- .../pipelineprocessor/rest/RuleResource.java | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index b4879f800dfb..c386858a3ca6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -246,7 +246,7 @@ public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { ruleBuilder.then(parseContext.statements); final Rule rule = ruleBuilder.build(); parseContext.addRule(rule); - log.info("Declaring rule {}", rule); + log.trace("Declaring rule {}", rule); } @Override @@ -350,7 +350,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) ); - log.info("FUNC: ctx {} => {}", ctx, expr); + log.trace("FUNC: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -384,7 +384,7 @@ public void exitNested(RuleLangParser.NestedContext ctx) { final Expression object = exprs.get(ctx.fieldSet); final Expression field = exprs.get(ctx.field); final FieldAccessExpression expr = new FieldAccessExpression(object, field); - log.info("FIELDACCESS: ctx {} => {}", ctx, expr); + log.trace("FIELDACCESS: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -392,7 +392,7 @@ public void exitNested(RuleLangParser.NestedContext ctx) { public void exitNot(RuleLangParser.NotContext ctx) { final LogicalExpression expression = upgradeBoolFunctionExpression(ctx.expression()); final NotExpression expr = new NotExpression(expression); - log.info("NOT: ctx {} => {}", ctx, expr); + log.trace("NOT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -404,7 +404,7 @@ public void exitAnd(RuleLangParser.AndContext ctx) { final LogicalExpression right = upgradeBoolFunctionExpression(ctx.right); final AndExpression expr = new AndExpression(left, right); - log.info("AND: ctx {} => {}", ctx, expr); + log.trace("AND: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -421,7 +421,7 @@ public void exitOr(RuleLangParser.OrContext ctx) { final LogicalExpression left = upgradeBoolFunctionExpression(ctx.left); final LogicalExpression right = upgradeBoolFunctionExpression(ctx.right); final OrExpression expr = new OrExpression(left, right); - log.info("OR: ctx {} => {}", ctx, expr); + log.trace("OR: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -431,7 +431,7 @@ public void exitEquality(RuleLangParser.EqualityContext ctx) { final Expression right = exprs.get(ctx.right); final boolean equals = ctx.equality.getText().equals("=="); final EqualityExpression expr = new EqualityExpression(left, right, equals); - log.info("EQUAL: ctx {} => {}", ctx, expr); + log.trace("EQUAL: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -441,7 +441,7 @@ public void exitComparison(RuleLangParser.ComparisonContext ctx) { final Expression right = exprs.get(ctx.right); final String operator = ctx.comparison.getText(); final ComparisonExpression expr = new ComparisonExpression(left, right, operator); - log.info("COMPARE: ctx {} => {}", ctx, expr); + log.trace("COMPARE: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -449,14 +449,14 @@ public void exitComparison(RuleLangParser.ComparisonContext ctx) { public void exitInteger(RuleLangParser.IntegerContext ctx) { // TODO handle different radix and length final LongExpression expr = new LongExpression(Long.parseLong(ctx.getText())); - log.info("INT: ctx {} => {}", ctx, expr); + log.trace("INT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitFloat(RuleLangParser.FloatContext ctx) { final DoubleExpression expr = new DoubleExpression(Double.parseDouble(ctx.getText())); - log.info("FLOAT: ctx {} => {}", ctx, expr); + log.trace("FLOAT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -470,14 +470,14 @@ public void exitChar(RuleLangParser.CharContext ctx) { public void exitString(RuleLangParser.StringContext ctx) { final String text = unquote(ctx.getText(), '\"'); final StringExpression expr = new StringExpression(text); - log.info("STRING: ctx {} => {}", ctx, expr); + log.trace("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitBoolean(RuleLangParser.BooleanContext ctx) { final BooleanExpression expr = new BooleanExpression(Boolean.valueOf(ctx.getText())); - log.info("BOOL: ctx {} => {}", ctx, expr); + log.trace("BOOL: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -523,7 +523,7 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { isIdIsFieldAccess.pop(); // reset for error checks final Expression fieldExpr = exprs.get(ctx.field); final MessageRefExpression expr = new MessageRefExpression(fieldExpr); - log.info("$MSG: ctx {} => {}", ctx, expr); + log.trace("$MSG: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -545,7 +545,7 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { expr = new VarRefExpression(identifierName); type = "VARREF"; } - log.info("{}: ctx {} => {}", type, ctx, expr); + log.trace("{}: ctx {} => {}", type, ctx, expr); exprs.put(ctx, expr); } @@ -563,7 +563,7 @@ public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { final IndexedAccessExpression expr = new IndexedAccessExpression(array, index); exprs.put(ctx, expr); - log.info("IDXACCESS: ctx {} => {}", ctx, expr); + log.trace("IDXACCESS: ctx {} => {}", ctx, expr); } } @@ -585,7 +585,7 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { log.error("Unable to retrieve expression for variable {}, this is a bug", name); return; } - log.info("Inferred type of variable {} to {}", name, expression.getType().getSimpleName()); + log.trace("Inferred type of variable {} to {}", name, expression.getType().getSimpleName()); varRefExpression.setType(expression.getType()); } } @@ -601,7 +601,7 @@ public RuleTypeChecker(ParseContext parseContext) { @Override public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { - log.info("Type tree {}", sb.toString()); + log.trace("Type tree {}", sb.toString()); } @Override @@ -841,7 +841,7 @@ public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ct public void exitInteger(RuleLangParser.IntegerContext ctx) { // TODO handle different radix and length final LongExpression expr = new LongExpression(Long.parseLong(ctx.getText())); - log.info("INT: ctx {} => {}", ctx, expr); + log.trace("INT: ctx {} => {}", ctx, expr); parseContext.exprs.put(ctx, expr); } @@ -849,7 +849,7 @@ public void exitInteger(RuleLangParser.IntegerContext ctx) { public void exitString(RuleLangParser.StringContext ctx) { final String text = unquote(ctx.getText(), '\"'); final StringExpression expr = new StringExpression(text); - log.info("STRING: ctx {} => {}", ctx, expr); + log.trace("STRING: ctx {} => {}", ctx, expr); parseContext.exprs.put(ctx, expr); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index f66fea2d14a0..3b52a88de99c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -91,7 +91,7 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t .build(); final PipelineDao save = pipelineService.save(pipelineDao); clusterBus.post(PipelinesChangedEvent.updatedPipelineId(save.id())); - log.info("Created new pipeline {}", save); + log.debug("Created new pipeline {}", save); return PipelineSource.fromDao(pipelineRuleParser, save); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 8438ae2609cf..42c329aa8842 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -92,7 +92,7 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No final RuleDao save = ruleService.save(newRuleSource); // TODO determine which pipelines could change because of this new rule (there could be pipelines referring to a previously unresolved rule) clusterBus.post(RulesChangedEvent.updatedRuleId(save.id())); - log.info("Created new rule {}", save); + log.debug("Created new rule {}", save); return RuleSource.fromDao(pipelineRuleParser, save); } From 5adacbf88e067f4c6269e5d31133e99bc284c4d2 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 15:56:56 +0100 Subject: [PATCH 155/521] update license headers --- .../pipelineprocessor/rest/BulkRuleRequest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java index 1dc31e2e7d0a..21fc4139eed4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import com.fasterxml.jackson.annotation.JsonAutoDetect; From 6dca36091a212a8b4094c9df5796d3aa31e6819b Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 4 Mar 2016 16:00:55 +0100 Subject: [PATCH 156/521] Rename artifactId to graylog-plugin-pipeline-processor --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b4e5ea1557fd..810f68a1b71c 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ org.graylog.plugins - pipeline-processor + graylog-plugin-pipeline-processor 1.0.0-SNAPSHOT jar From 37df531f17eaa4c2ea38fee98039fe58e2c1eb58 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 16:18:50 +0100 Subject: [PATCH 157/521] Add autofocus to rule form --- src/web/rules/RuleForm.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index c31171e51015..90f9bf063316 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -143,6 +143,7 @@ const RuleForm = React.createClass({ id={this._getId('description')} label="Description (optional)" onChange={this._onDescriptionChange} + autoFocus value={this.state.rule.description}/> From c5dd5dcfa43905d98a1f0111d31234ae4940a15d Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 16:28:38 +0100 Subject: [PATCH 158/521] Disable jsx-closing-bracket-location eslint rule --- .eslintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc b/.eslintrc index 12bf5d0b6b40..209f58ef2627 100644 --- a/.eslintrc +++ b/.eslintrc @@ -17,6 +17,7 @@ "react/prefer-es6-class": 0, "arrow-body-style": 0, "react/jsx-indent-props": 0, + "react/jsx-closing-bracket-location": 0, }, } From 269581f1d459f7f0baf6c7fe8b570ec49c13a984 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 17:15:07 +0100 Subject: [PATCH 159/521] add a silent mode to the parser, to avoid creating too many log lines during interactive parsing --- .../parser/PipelineRuleParser.java | 26 +++++++++++--- .../processors/PipelineInterpreter.java | 2 +- .../pipelineprocessor/rest/RuleResource.java | 7 ++-- .../pipelineprocessor/rest/RuleSource.java | 2 +- .../functions/FunctionsSnippetsTest.java | 14 ++++---- .../parser/PipelineRuleParserTest.java | 36 +++++++++---------- 6 files changed, 52 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index c386858a3ca6..fcf7d8460ad9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -101,8 +101,8 @@ public PipelineRuleParser(FunctionRegistry functionRegistry) { private static final Logger log = LoggerFactory.getLogger(PipelineRuleParser.class); public static final ParseTreeWalker WALKER = ParseTreeWalker.DEFAULT; - public Rule parseRule(String rule) throws ParseException { - final ParseContext parseContext = new ParseContext(); + public Rule parseRule(String rule, boolean silent) throws ParseException { + final ParseContext parseContext = new ParseContext(silent); final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(rule)); @@ -134,7 +134,7 @@ public Rule parseRule(String rule) throws ParseException { } public List parsePipelines(String pipelines) throws ParseException { - final ParseContext parseContext = new ParseContext(); + final ParseContext parseContext = new ParseContext(false); final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(pipelines)); @@ -157,7 +157,7 @@ public List parsePipelines(String pipelines) throws ParseException { } public Pipeline parsePipeline(String id, String source) { - final ParseContext parseContext = new ParseContext(); + final ParseContext parseContext = new ParseContext(false); final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(source)); @@ -582,7 +582,11 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { final String name = varRefExpression.varName(); final Expression expression = parseContext.getDefinedVar(name); if (expression == null) { - log.error("Unable to retrieve expression for variable {}, this is a bug", name); + if (parseContext.isSilent()) { + log.debug("Unable to retrieve expression for variable {}, this is a bug", name); + } else { + log.error("Unable to retrieve expression for variable {}, this is a bug", name); + } return; } log.trace("Inferred type of variable {} to {}", name, expression.getType().getSimpleName()); @@ -716,6 +720,10 @@ public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { private static class ParseContext { private final ParseTreeProperty exprs = new ParseTreeProperty<>(); private final ParseTreeProperty> args = new ParseTreeProperty<>(); + /** + * Should the parser be more silent about its error logging, useful for interactive parsing in the UI. + */ + private final boolean silent; private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); private Set errors = Sets.newHashSet(); // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information @@ -725,6 +733,10 @@ private static class ParseContext { private Map varDecls = Maps.newHashMap(); public List pipelines = Lists.newArrayList(); + public ParseContext(boolean silent) { + this.silent = silent; + } + public ParseTreeProperty expressions() { return exprs; } @@ -782,6 +794,10 @@ public Expression getDefinedVar(String name) { public ParseTreeProperty> argumentLists() { return argsLists; } + + public boolean isSilent() { + return silent; + } } private class PipelineAstBuilder extends RuleLangBaseListener { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 71df1f85269d..b42acbd1abaf 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -115,7 +115,7 @@ private synchronized void reload() { for (RuleDao ruleDao : ruleService.loadAll()) { Rule rule; try { - rule = pipelineRuleParser.parseRule(ruleDao.source()); + rule = pipelineRuleParser.parseRule(ruleDao.source(), false); } catch (ParseException e) { rule = Rule.alwaysFalse("Failed to parse rule: " + ruleDao.id()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 42c329aa8842..04f657a21ff2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -78,7 +78,7 @@ public RuleResource(RuleService ruleService, public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { final Rule rule; try { - rule = pipelineRuleParser.parseRule(ruleSource.source()); + rule = pipelineRuleParser.parseRule(ruleSource.source(), false); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -102,7 +102,8 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { final Rule rule; try { - rule = pipelineRuleParser.parseRule(ruleSource.source()); + // be silent about parse errors here, many requests will result in invalid syntax + rule = pipelineRuleParser.parseRule(ruleSource.source(), true); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -151,7 +152,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, final RuleDao ruleDao = ruleService.load(id); final Rule rule; try { - rule = pipelineRuleParser.parseRule(update.source()); + rule = pipelineRuleParser.parseRule(update.source(), false); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index 0bacc9726a68..ea6d3e9faf41 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -90,7 +90,7 @@ public static RuleSource create(@Id @ObjectId @JsonProperty("_id") @Nullable Str public static RuleSource fromDao(PipelineRuleParser parser, RuleDao dao) { Set errors = null; try { - parser.parseRule(dao.source()); + parser.parseRule(dao.source(), false); } catch (ParseException e) { errors = e.getErrors(); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index dc7cf89e5334..5923599ad66c 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -171,7 +171,7 @@ public void jsonpath() { " \"expensive\": 10\n" + "}"; - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); final Message message = evaluateRule(rule, new Message(json, "test", Tools.nowUTC())); assertThat(message.hasField("author_first")).isTrue(); @@ -181,7 +181,7 @@ public void jsonpath() { @Test public void substring() { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule); assertThat(actionsTriggered.get()).isTrue(); @@ -195,7 +195,7 @@ public void dates() { try { final Rule rule; try { - rule = parser.parseRule(ruleForTest()); + rule = parser.parseRule(ruleForTest(), false); } catch (ParseException e) { fail("Should not fail to parse", e); return; @@ -215,7 +215,7 @@ public void dates() { @Test public void digests() { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule); assertThat(actionsTriggered.get()).isTrue(); @@ -224,7 +224,7 @@ public void digests() { @Test public void regexMatch() { try { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); final Message message = evaluateRule(rule); assertNotNull(message); assertTrue(message.hasField("matched_regex")); @@ -236,7 +236,7 @@ public void regexMatch() { @Test public void strings() { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); final Message message = evaluateRule(rule); assertThat(actionsTriggered.get()).isTrue(); @@ -247,7 +247,7 @@ public void strings() { @Test public void ipMatching() { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); final Message in = new Message("test", "test", Tools.nowUTC()); in.addField("ip", "192.168.1.20"); final Message message = evaluateRule(rule, in); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java index 5527da94e51c..135b267f088a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -263,14 +263,14 @@ public void tearDown() { @Test public void basicRule() throws Exception { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); Assert.assertNotNull("rule should be successfully parsed", rule); } @Test public void undeclaredIdentifier() throws Exception { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); fail("should throw error: undeclared variable x"); } catch (ParseException e) { assertEquals(2, @@ -283,7 +283,7 @@ public void undeclaredIdentifier() throws Exception { @Test public void declaredFunction() throws Exception { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); } catch (ParseException e) { fail("Should not fail to resolve function 'false'"); } @@ -292,7 +292,7 @@ public void declaredFunction() throws Exception { @Test public void undeclaredFunction() throws Exception { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); fail("should throw error: undeclared function 'unknown'"); } catch (ParseException e) { assertTrue("Should find error UndeclaredFunction", @@ -303,7 +303,7 @@ public void undeclaredFunction() throws Exception { @Test public void singleArgFunction() throws Exception { try { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); final Message message = evaluateRule(rule); assertNotNull(message); @@ -316,7 +316,7 @@ public void singleArgFunction() throws Exception { @Test public void positionalArguments() throws Exception { try { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule); assertTrue(actionsTriggered.get()); @@ -328,7 +328,7 @@ public void positionalArguments() throws Exception { @Test public void inferVariableType() throws Exception { try { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule); } catch (ParseException e) { @@ -339,7 +339,7 @@ public void inferVariableType() throws Exception { @Test public void invalidArgType() throws Exception { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); } catch (ParseException e) { assertEquals(2, e.getErrors().size()); assertTrue("Should only find IncompatibleArgumentType errors", @@ -350,7 +350,7 @@ public void invalidArgType() throws Exception { @Test public void booleanValuedFunctionAsCondition() throws Exception { try { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule); assertTrue("actions should have triggered", actionsTriggered.get()); @@ -361,7 +361,7 @@ public void booleanValuedFunctionAsCondition() throws Exception { @Test public void messageRef() throws Exception { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); Message message = new Message("hello test", "source", DateTime.now()); message.addField("responseCode", 500); final Message processedMsg = evaluateRule(rule, message); @@ -372,7 +372,7 @@ public void messageRef() throws Exception { @Test public void messageRefQuotedField() throws Exception { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); Message message = new Message("hello test", "source", DateTime.now()); message.addField("@specialfieldname", "string"); evaluateRule(rule, message); @@ -382,7 +382,7 @@ public void messageRefQuotedField() throws Exception { @Test public void optionalArguments() throws Exception { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); Message message = new Message("hello test", "source", DateTime.now()); evaluateRule(rule, message); @@ -392,7 +392,7 @@ public void optionalArguments() throws Exception { @Test public void optionalParamsMustBeNamed() throws Exception { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); } catch (ParseException e) { assertEquals(1, e.getErrors().stream().count()); assertTrue(e.getErrors().stream().allMatch(error -> error instanceof OptionalParametersMustBeNamed)); @@ -402,7 +402,7 @@ public void optionalParamsMustBeNamed() throws Exception { @Test public void mapArrayLiteral() { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); Message message = new Message("hello test", "source", DateTime.now()); evaluateRule(rule, message); assertTrue(actionsTriggered.get()); @@ -411,7 +411,7 @@ public void mapArrayLiteral() { @Test public void typedFieldAccess() throws Exception { try { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule, new Message("hallo", "test", DateTime.now())); assertTrue("condition should be true", actionsTriggered.get()); } catch (ParseException e) { @@ -441,7 +441,7 @@ public void pipelineDeclaration() throws Exception { @Test public void indexedAccess() { - final Rule rule = parser.parseRule(ruleForTest()); + final Rule rule = parser.parseRule(ruleForTest(), false); evaluateRule(rule, new Message("hallo", "test", DateTime.now())); assertTrue("condition should be true", actionsTriggered.get()); @@ -450,7 +450,7 @@ public void indexedAccess() { @Test public void indexedAccessWrongType() { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); } catch (ParseException e) { assertEquals(1, e.getErrors().size()); assertEquals(NonIndexableType.class, Iterables.getOnlyElement(e.getErrors()).getClass()); @@ -460,7 +460,7 @@ public void indexedAccessWrongType() { @Test public void indexedAccessWrongIndexType() { try { - parser.parseRule(ruleForTest()); + parser.parseRule(ruleForTest(), false); } catch (ParseException e) { assertEquals(1, e.getErrors().size()); assertEquals(IncompatibleIndexType.class, Iterables.getOnlyElement(e.getErrors()).getClass()); From b1e65d4266ee0630b89b9b08b952577b80844520 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 17:31:21 +0100 Subject: [PATCH 160/521] add initial documentation links --- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 2 +- src/web/rules/RulesPage.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index eb81d81ffb05..2e98c1311bde 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -65,7 +65,7 @@ const PipelineConnectionsPage = React.createClass({ as input for the different pipelines you configure. - Read more about Graylog pipelines in the . + Read more about Graylog pipelines in the . diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index e1fbe30a385d..3c434023198a 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -34,7 +34,7 @@ const RulesPage = React.createClass({ - Read more about Graylog pipeline rules in the . From 353d7bf3e4e5c706f3fb02b895638f446694e8e3 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Fri, 4 Mar 2016 17:46:48 +0100 Subject: [PATCH 161/521] tweak text and doc links --- src/web/pipelines/PipelinesOverviewPage.jsx | 2 +- src/web/rules/RulesPage.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx index 08cf348431a9..37a9205f6ab9 100644 --- a/src/web/pipelines/PipelinesOverviewPage.jsx +++ b/src/web/pipelines/PipelinesOverviewPage.jsx @@ -20,7 +20,7 @@ const PipelinesOverviewPage = React.createClass({ - + {' '} diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 3c434023198a..12676a48f188 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -34,7 +34,7 @@ const RulesPage = React.createClass({ - Read more about Graylog pipeline rules in the . From f02c01c0b65ffc663919a3c580cea72b897e94e1 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 18:03:03 +0100 Subject: [PATCH 162/521] Delete getting started file --- GETTING-STARTED.md | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 GETTING-STARTED.md diff --git a/GETTING-STARTED.md b/GETTING-STARTED.md deleted file mode 100644 index dd82c9dbe190..000000000000 --- a/GETTING-STARTED.md +++ /dev/null @@ -1,26 +0,0 @@ -Getting started with your new Graylog plugin -============================================ - -Welcome to your new Graylog plugin! - -Please refer to http://docs.graylog.org/en/latest/pages/plugins.html for documentation on how to write -plugins for Graylog. - -Travis CI ---------- - -There is a `.travis.yml` template in this project which is prepared to automatically -deploy the plugin artifacts (JAR, DEB, RPM) to GitHub releases. - -You just have to add your encrypted GitHub access token to the `.travis.yml`. -The token can be generated in your [GitHub personal access token settings](https://github.com/settings/tokens). - -Before Travis CI works, you have to enable it. Install the Travis CI command line -application and execute `travis enable`. - -To encrypt your GitHub access token you can use `travis encrypt`. - -Alternatively you can use `travis setup -f releases` to automatically create a GitHub -access token and add it to the `.travis.yml` file. **Attention:** doing this -will replace some parts of the `.travis.yml` file and you have to restore previous -settings. From b926920141877bfe71b3d00e730a950391a76ec9 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 18:03:05 +0100 Subject: [PATCH 163/521] Add CoC, contributing, and copying files --- CODE_OF_CONDUCT.md | 74 +++++ CONTRIBUTING.md | 12 + COPYING | 674 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 760 insertions(+) create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 COPYING diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..b0e0c635fb03 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at . All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000000..f55e0caf6d76 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,12 @@ +Please follow [the instructions on graylog.org](https://www.graylog.org/contributing-to-graylog/). + +#### Code of Conduct + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Please read and understand the [Code of Conduct](https://github.com/Graylog2/graylog-plugin-collector/blob/master/CODE_OF_CONDUCT.md). diff --git a/COPYING b/COPYING new file mode 100644 index 000000000000..94a9ed024d38 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From 4c9861a0cd4850bc0b290bf1491f6db2388ad446 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 18:05:12 +0100 Subject: [PATCH 164/521] Update license headers --- .../benchmarks/pipeline/DeclaringClassBenchmark.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java index 78ff72942bf9..61f738c234e1 100644 --- a/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java +++ b/benchmarks/src/main/java/org/graylog/benchmarks/pipeline/DeclaringClassBenchmark.java @@ -1,18 +1,18 @@ /** - * This file is part of Graylog. + * This file is part of Graylog Pipeline Processor. * - * Graylog is free software: you can redistribute it and/or modify + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * - * Graylog is distributed in the hope that it will be useful, + * Graylog Pipeline Processor is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with Graylog. If not, see . + * along with Graylog Pipeline Processor. If not, see . */ package org.graylog.benchmarks.pipeline; From 6269612462b4c27d83308e696e6a3a287049130b Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 4 Mar 2016 18:07:23 +0100 Subject: [PATCH 165/521] Use right url to project --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f55e0caf6d76..469be2a981a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,4 +9,4 @@ size, disability, ethnicity, gender identity and expression, level of experience nationality, personal appearance, race, religion, or sexual identity and orientation. -Please read and understand the [Code of Conduct](https://github.com/Graylog2/graylog-plugin-collector/blob/master/CODE_OF_CONDUCT.md). +Please read and understand the [Code of Conduct](https://github.com/Graylog2/graylog-plugin-pipeline-processor/blob/master/CODE_OF_CONDUCT.md). From 28e637107795e1224c5feee7fb28276f62c292dd Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 4 Mar 2016 18:32:37 +0100 Subject: [PATCH 166/521] Bump Graylog server version to 2.0.0-alpha.5 and adjust metadata --- package.json | 2 +- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 371a64e2c039..939281775d07 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "keywords": [ "graylog" ], - "author": "Kay Roepke ", + "author": "Graylog, Inc. ", "license": "GPL-3.0", "dependencies": { "brace": "^0.7.0", diff --git a/pom.xml b/pom.xml index 810f68a1b71c..a01507a6a559 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-alpha.5-SNAPSHOT + 2.0.0-alpha.5 /usr/share/graylog-server/plugin 4.5.1 1.7.13 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index 824b5e342572..ae4d608476bf 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 0, 0, "SNAPSHOT"); + return new Version(1, 0, 0, "alpha.5"); } @Override From fab69060a7880e1a609afe277df0acfc7ef25ccd Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 4 Mar 2016 18:36:48 +0100 Subject: [PATCH 167/521] Fix scm entry in pom.xml --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a01507a6a559..32303a63b54a 100644 --- a/pom.xml +++ b/pom.xml @@ -24,9 +24,9 @@ - scm:git:git@github.com:graylog-plugin-pipeline-processor.git - scm:git:git@github.com:graylog-plugin-pipeline-processor.git - https://github.com/graylog-plugin-pipeline-processor + scm:git:git@github.com:Graylog2/graylog-plugin-pipeline-processor.git + scm:git:git@github.com:Graylog2/graylog-plugin-pipeline-processor.git + https://github.com/Graylog2/graylog-plugin-pipeline-processor HEAD From af793df35f73cafcb7a357ac67e42821fa201679 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Fri, 4 Mar 2016 17:38:00 +0000 Subject: [PATCH 168/521] [graylog-plugin-pipeline-processor-release] prepare release 1.0.0-alpha.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 32303a63b54a..b854f9a1b4e3 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-SNAPSHOT + 1.0.0-alpha.5 jar ${project.artifactId} From 7172e4099f22250c0a108fd8ab2663638f9067f3 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Fri, 4 Mar 2016 17:38:12 +0000 Subject: [PATCH 169/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b854f9a1b4e3..0ee0a7501abe 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-alpha.5 + 1.0.0-alpha.6-SNAPSHOT jar ${project.artifactId} From 546bbd93955c5633fc60611769384b3408d5b22c Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 4 Mar 2016 18:59:07 +0100 Subject: [PATCH 170/521] Bump Graylog server dependency to 2.0.0-alpha.6-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ee0a7501abe..fc2299f9ed6e 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-alpha.5 + 2.0.0-alpha.6-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From 978fca6d30fc26b19c9801024d33a9544daac4ac Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 7 Mar 2016 11:48:16 +0100 Subject: [PATCH 171/521] UI changes in pipelines connection page - Make connected pipelines clickable - Improve messages, including links to facilitate navigation. - Remove button to streams page --- src/web/pipeline-connections/Connection.jsx | 16 ++++++---- .../pipeline-connections/ConnectionForm.jsx | 30 ++++++++++++++----- .../PipelineConnections.jsx | 10 +++---- .../PipelineConnectionsPage.jsx | 22 +++++++------- 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx index 2129b6a6be74..d02975cc4f20 100644 --- a/src/web/pipeline-connections/Connection.jsx +++ b/src/web/pipeline-connections/Connection.jsx @@ -1,5 +1,6 @@ import React from 'react'; import { Col } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; import { DataTable, EntityListItem } from 'components/common'; import ConnectionForm from './ConnectionForm'; @@ -18,7 +19,9 @@ const Connection = React.createClass({ _pipelineRowFormatter(pipeline) { return ( - {pipeline.title} + + {pipeline.title} + {pipeline.description} ); @@ -34,15 +37,16 @@ const Connection = React.createClass({ headerCellFormatter={this._pipelineHeaderFormatter} rows={pipelines} dataRowFormatter={this._pipelineRowFormatter} + noDataText={'This stream has no connected pipelines. Click on "Edit connections" to add one.'} filterLabel="" - filterKeys={[]}/> + filterKeys={[]} /> ); }, render() { const actions = ( - + ); const content = ( @@ -52,11 +56,11 @@ const Connection = React.createClass({ ); return ( - + contentRow={content} /> ); }, }); diff --git a/src/web/pipeline-connections/ConnectionForm.jsx b/src/web/pipeline-connections/ConnectionForm.jsx index 3ed01376fcb6..177ed547044d 100644 --- a/src/web/pipeline-connections/ConnectionForm.jsx +++ b/src/web/pipeline-connections/ConnectionForm.jsx @@ -1,6 +1,7 @@ import React, { PropTypes } from 'react'; import Reflux from 'reflux'; import { Input, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; import { Select, SelectableList } from 'components/common'; import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; @@ -48,13 +49,13 @@ const ConnectionForm = React.createClass({ _onStreamChange(newStream) { const connection = ObjectUtils.clone(this.state.connection); connection.stream = this.props.streams.filter(s => s.id === newStream)[0]; - this.setState({connection: connection}); + this.setState({ connection }); }, _onConnectionsChange(newRules) { const connection = ObjectUtils.clone(this.state.connection); connection.pipelines = newRules; - this.setState({connection: connection}); + this.setState({ connection }); }, _closeModal() { @@ -82,7 +83,7 @@ const ConnectionForm = React.createClass({ } return streams.map(s => { - return {value: s.id, label: s.title}; + return { value: s.id, label: s.title }; }); }, @@ -93,7 +94,7 @@ const ConnectionForm = React.createClass({ return pipelines .map(pipeline => { - return {value: pipeline.id, label: pipeline.title}; + return { value: pipeline.id, label: pipeline.title }; }); }, @@ -113,14 +114,27 @@ const ConnectionForm = React.createClass({ let streamSelector; if (this.props.create) { + const streamHelp = ( + + Select the stream you want to connect pipelines to, or create one in the{' '} + Streams page. + + ); streamSelector = ( - ); } + const pipelineHelp = ( + + Select the pipelines to connect to this stream, or create one in the{' '} + Pipelines Overview page. + + ); + return (
    diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index 90e1f49e935f..fecfc6c7e9c3 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -20,7 +20,7 @@ const PipelineConnections = React.createClass({ }, _updateFilteredStreams(filteredStreams) { - this.setState({filteredStreams: filteredStreams}); + this.setState({ filteredStreams }); }, _updateConnection(newConnection, callback) { @@ -35,7 +35,7 @@ const PipelineConnections = React.createClass({ return ( s.id === c.stream_id)[0]} pipelines={this.props.pipelines.filter(p => c.pipeline_ids.indexOf(p.id) !== -1)} - onUpdate={this._updateConnection}/> + onUpdate={this._updateConnection} /> ); }); @@ -50,16 +50,16 @@ const PipelineConnections = React.createClass({ displayKey={'title'} filterSuggestions={[]} searchInKeys={['title', 'description']} - onDataFiltered={this._updateFilteredStreams}/> + onDataFiltered={this._updateFilteredStreams} />
    - +
    + items={formattedConnections} />
    ); }, diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 2e98c1311bde..3f7289a0c606 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -1,4 +1,4 @@ -import React, {PropTypes} from 'react'; +import React, { PropTypes } from 'react'; import Reflux from 'reflux'; import { Button, Row, Col } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; @@ -7,8 +7,6 @@ import DocumentationLink from 'components/support/DocumentationLink'; import { PageHeader, Spinner } from 'components/common'; import PipelineConnections from './PipelineConnections'; -import Routes from 'routing/Routes'; - import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; import PipelineConnectionsActions from 'pipeline-connections/PipelineConnectionsActions'; @@ -33,8 +31,12 @@ const PipelineConnectionsPage = React.createClass({ const store = this.context.storeProvider.getStore('Streams'); store.listStreams().then((streams) => { - streams.push({id: 'default', title: 'Incoming messages', description: 'Default stream of all incoming messages.'}); - this.setState({streams: streams}); + streams.push({ + id: 'default', + title: 'Incoming messages', + description: 'Default stream of all incoming messages.', + }); + this.setState({ streams }); }); }, @@ -54,25 +56,21 @@ const PipelineConnectionsPage = React.createClass({ } else { content = ( + connections={this.state.connections} onConnectionsChange={this._updateConnections} /> ); } return ( - + Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} as input for the different pipelines you configure. - Read more about Graylog pipelines in the . + Read more about Graylog pipelines in the . - - - -   From 65fb40cb1ebca945f6fe3898ff5a320542728e99 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 7 Mar 2016 17:36:14 +0100 Subject: [PATCH 172/521] Use DocsHelper to get documentation links --- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 4 +++- src/web/rules/RulesPage.jsx | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 3f7289a0c606..9b10f9f465f7 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -12,6 +12,8 @@ import PipelinesStore from 'pipelines/PipelinesStore'; import PipelineConnectionsActions from 'pipeline-connections/PipelineConnectionsActions'; import PipelineConnectionsStore from 'pipeline-connections/PipelineConnectionsStore'; +import DocsHelper from 'util/DocsHelper'; + const PipelineConnectionsPage = React.createClass({ contextTypes: { storeProvider: PropTypes.object, @@ -67,7 +69,7 @@ const PipelineConnectionsPage = React.createClass({ as input for the different pipelines you configure. - Read more about Graylog pipelines in the . + Read more about Graylog pipelines in the . diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 12676a48f188..1b01c365095b 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -34,8 +34,8 @@ const RulesPage = React.createClass({ - Read more about Graylog pipeline rules in the . + Read more about Graylog pipeline rules in the . From 3ada94befb84a2116d99c452bb843fd142f41c0e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 7 Mar 2016 17:39:20 +0100 Subject: [PATCH 173/521] Small UI improvements - Add some consistency with buttons - Remove inline styles and use existing CSS classes - Add some links in forms --- src/web/pipelines/PipelinesOverviewPage.jsx | 2 +- .../pipelines/ProcessingTimelineComponent.css | 8 ++++++++ .../pipelines/ProcessingTimelineComponent.jsx | 20 +++++++------------ src/web/pipelines/StageForm.jsx | 10 +++++++++- src/web/rules/RulesPage.jsx | 6 +++++- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx index 37a9205f6ab9..82ffb3b9f96c 100644 --- a/src/web/pipelines/PipelinesOverviewPage.jsx +++ b/src/web/pipelines/PipelinesOverviewPage.jsx @@ -20,7 +20,7 @@ const PipelinesOverviewPage = React.createClass({ - + {' '} diff --git a/src/web/pipelines/ProcessingTimelineComponent.css b/src/web/pipelines/ProcessingTimelineComponent.css index d3bf83d421e2..114d637d5970 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.css +++ b/src/web/pipelines/ProcessingTimelineComponent.css @@ -15,4 +15,12 @@ .pipeline-stage.used-stage { background-color: #FFFFFF; +} + +.pipeline-name { + max-width: 300px; + overflow-x: hidden; + text-overflow: ellipsis; + white-space: nowrap; + width: 300px; } \ No newline at end of file diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index c7eb7c22a569..948aca537967 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -33,19 +33,12 @@ const ProcessingTimelineComponent = React.createClass({ }, _headerCellFormatter(header) { - const style = {}; - switch (header) { - case 'Pipeline': - style.width = 300; - break; - case 'Actions': - style.width = 120; - break; - default: - // Nothing to see here + let className; + if (header === 'Actions') { + className = 'actions'; } - return {header}; + return {header}; }, _formatStages(pipeline, stages) { @@ -70,8 +63,9 @@ const ProcessingTimelineComponent = React.createClass({ _pipelineFormatter(pipeline) { return ( - - {pipeline.title} + + {pipeline.title}
    + {pipeline.description} {this._formatStages(pipeline, pipeline.stages)} diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx index 8de11074054b..0c6bdefd4e1b 100644 --- a/src/web/pipelines/StageForm.jsx +++ b/src/web/pipelines/StageForm.jsx @@ -1,6 +1,7 @@ import React, { PropTypes } from 'react'; import Reflux from 'reflux'; import { Input, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; import { SelectableList } from 'components/common'; import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; @@ -85,6 +86,13 @@ const StageForm = React.createClass({ triggerButtonContent = Edit; } + const rulesHelp = ( + + Select the rules evaluated on this stage, or create one in the{' '} + Pipeline Rules page. + + ); + return ( +     +   From 8e903abb1ca05b99f5a98329a8db60f2e2404686 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 7 Mar 2016 17:40:04 +0100 Subject: [PATCH 174/521] Code styling changes --- src/web/pipelines/Pipeline.jsx | 10 ++++---- src/web/pipelines/PipelineForm.jsx | 10 ++++---- .../pipelines/ProcessingTimelineComponent.jsx | 6 ++--- src/web/pipelines/Stage.jsx | 14 ++++++----- src/web/pipelines/StageForm.jsx | 14 +++++------ src/web/rules/RuleList.jsx | 23 +++++++++---------- src/web/rules/RulesPage.jsx | 6 ++--- 7 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index dae8d8f39cd5..dd87e0e66391 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -1,4 +1,4 @@ -import React, {PropTypes} from 'react'; +import React, { PropTypes } from 'react'; import { Row, Col } from 'react-bootstrap'; import { EntityList } from 'components/common'; @@ -37,7 +37,7 @@ const Pipeline = React.createClass({ _formatStage(stage, maxStage) { return ( + onUpdate={this._updateStage(stage)} onDelete={this._deleteStage(stage)} /> ); }, @@ -52,13 +52,13 @@ const Pipeline = React.createClass({
    - +

    Description

    -

    {this.props.pipeline.description}

    +

    {this.props.pipeline.description}

     - +
    ); }, diff --git a/src/web/pipelines/PipelineForm.jsx b/src/web/pipelines/PipelineForm.jsx index 6b2023cafe6d..c233f1dd5187 100644 --- a/src/web/pipelines/PipelineForm.jsx +++ b/src/web/pipelines/PipelineForm.jsx @@ -1,4 +1,4 @@ -import React, { PropTypes } from 'react'; +import React from 'react'; import { Input, Button } from 'react-bootstrap'; import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; @@ -7,7 +7,7 @@ import FormsUtils from 'util/FormsUtils'; const PipelineForm = React.createClass({ propTypes: { - pipeline: PropTypes.object, + pipeline: React.PropTypes.object, create: React.PropTypes.bool, save: React.PropTypes.func.isRequired, validatePipeline: React.PropTypes.func.isRequired, @@ -44,7 +44,7 @@ const PipelineForm = React.createClass({ _onChange(event) { const pipeline = ObjectUtils.clone(this.state.pipeline); pipeline[event.target.name] = FormsUtils.getValueFromInput(event.target); - this.setState({pipeline: pipeline}); + this.setState({ pipeline }); }, _closeModal() { @@ -90,7 +90,7 @@ const PipelineForm = React.createClass({ required onChange={this._onChange} help="Pipeline name." - value={this.state.pipeline.title}/> + value={this.state.pipeline.title} /> + value={this.state.pipeline.description} />     diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 948aca537967..2bd2da362ef2 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -98,13 +98,13 @@ const ProcessingTimelineComponent = React.createClass({ render() { if (!this.state.pipelines) { - return ; + return ; } if (this.state.pipelines.length === 0) { return (
    -
    +
    There are no pipelines configured in your system. Create one to start processing your messages. @@ -126,7 +126,7 @@ const ProcessingTimelineComponent = React.createClass({ rows={this.state.pipelines} dataRowFormatter={this._pipelineFormatter} filterLabel="Filter pipelines" - filterKeys={['title']}/> + filterKeys={['title']} />
    ); }, diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index e3e8b939e49a..65c7137036d5 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -1,4 +1,4 @@ -import React, {PropTypes} from 'react'; +import React, { PropTypes } from 'react'; import Reflux from 'reflux'; import { Col, Button } from 'react-bootstrap'; @@ -22,7 +22,7 @@ const Stage = React.createClass({ _ruleRowFormatter(rule) { return ( - {rule.title} + {rule.title} {rule.description} ); @@ -38,8 +38,9 @@ const Stage = React.createClass({ headerCellFormatter={this._ruleHeaderFormatter} rows={rules} dataRowFormatter={this._ruleRowFormatter} + noDataText="This stage has no rules yet. Click on edit to add some." filterLabel="" - filterKeys={[]}/> + filterKeys={[]} /> ); }, @@ -49,7 +50,8 @@ const Stage = React.createClass({ const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules`)}`; const actions = [ - , + , , ]; @@ -70,7 +72,7 @@ const Stage = React.createClass({ if (this.state.rules) { content = this._formatRules(this.props.stage.rules.map(name => this.state.rules.filter(r => r.title === name)[0])); } else { - content = ; + content = ; } return ( @@ -78,7 +80,7 @@ const Stage = React.createClass({ titleSuffix={suffix} actions={actions} description={description} - contentRow={{content}}/> + contentRow={{content}} /> ); }, }); diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx index 0c6bdefd4e1b..bf872f45c121 100644 --- a/src/web/pipelines/StageForm.jsx +++ b/src/web/pipelines/StageForm.jsx @@ -48,13 +48,13 @@ const StageForm = React.createClass({ _onChange(event) { const stage = ObjectUtils.clone(this.state.stage); stage[event.target.name] = FormsUtils.getValueFromInput(event.target); - this.setState({stage: stage}); + this.setState({ stage }); }, _onRulesChange(newRules) { const stage = ObjectUtils.clone(this.state.stage); stage.rules = newRules; - this.setState({stage: stage}); + this.setState({ stage }); }, _closeModal() { @@ -74,7 +74,7 @@ const StageForm = React.createClass({ _getFormattedOptions(rules) { return rules ? rules.map(rule => { - return {value: rule.title, label: rule.title}; + return { value: rule.title, label: rule.title }; }) : []; }, @@ -111,7 +111,7 @@ const StageForm = React.createClass({ autoFocus onChange={this._onChange} help="Stage priority. The lower the number, the earlier it will execute." - value={this.state.stage.stage}/> + value={this.state.stage.stage} /> + checked={this.state.stage.match_all} /> + checked={!this.state.stage.match_all} /> + onChange={this._onRulesChange} selectedOptions={this.state.stage.rules} /> diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index bd586f12fb5b..4da209bb634b 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -1,6 +1,5 @@ -import React, {PropTypes} from 'react'; -import DataTable from 'components/common/DataTable'; -import Timestamp from 'components/common/Timestamp'; +import React, { PropTypes } from 'react'; +import { DataTable, Timestamp } from 'components/common'; import { Button } from 'react-bootstrap'; @@ -9,7 +8,7 @@ import RulesActions from './RulesActions'; const RuleList = React.createClass({ propTypes: { - rules: PropTypes.array.isRequired + rules: PropTypes.array.isRequired, }, _save(rule, callback) { @@ -22,7 +21,7 @@ const RuleList = React.createClass({ }, _delete(rule) { - if (window.confirm('Do you really want to delete rule ' + rule.title + '?')) { + if (window.confirm(`Do you really want to delete rule "${rule.title}"?`)) { RulesActions.delete(rule.id); } }, @@ -35,8 +34,8 @@ const RuleList = React.createClass({ return {header}; }, _ruleInfoFormatter(rule) { - let actions = [ - , + const actions = [ + ,  , , ]; @@ -45,15 +44,15 @@ const RuleList = React.createClass({ {rule.title} {rule.description} - - - {actions} + + + {actions} ); }, render() { - var filterKeys = ["title", "description"]; - var headers = ["Title", "Description", "Created at", "Last modified", "Actions"]; + const filterKeys = ['title', 'description']; + const headers = ['Title', 'Description', 'Created at', 'Last modified', 'Actions']; return (
    diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index b5de8d0f4be7..c3bb945c57ad 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -14,12 +14,12 @@ import RulesStore from './RulesStore'; import RulesActions from './RulesActions'; const RulesPage = React.createClass({ - mixins: [ - Reflux.connect(RulesStore), - ], contextTypes: { storeProvider: React.PropTypes.object, }, + mixins: [ + Reflux.connect(RulesStore), + ], componentDidMount() { RulesActions.list(); }, From 4cbb4e0d8d938d97e414fe6cbbb78d9540b7244a Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 7 Mar 2016 18:33:13 +0100 Subject: [PATCH 175/521] More UI changes to improve consistency --- src/web/pipeline-connections/Connection.jsx | 6 ++++-- src/web/pipelines/Pipeline.css | 16 ++++++++++++++++ src/web/pipelines/Pipeline.jsx | 20 +++++++++++++++----- src/web/pipelines/PipelineDetailsPage.jsx | 2 +- src/web/rules/RuleList.jsx | 2 +- 5 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 src/web/pipelines/Pipeline.css diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx index d02975cc4f20..6443b2d1b4b8 100644 --- a/src/web/pipeline-connections/Connection.jsx +++ b/src/web/pipeline-connections/Connection.jsx @@ -2,7 +2,7 @@ import React from 'react'; import { Col } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; -import { DataTable, EntityListItem } from 'components/common'; +import { DataTable, EntityListItem, Timestamp } from 'components/common'; import ConnectionForm from './ConnectionForm'; const Connection = React.createClass({ @@ -23,12 +23,14 @@ const Connection = React.createClass({ {pipeline.title} {pipeline.description} + + ); }, _formatPipelines(pipelines) { - const headers = ['Title', 'Description']; + const headers = ['Title', 'Description', 'Created', 'Last modified']; return ( dt { + text-align: left; + width: 90px; +} + +dl.pipeline-dl > dt:after { + content: ':'; +} + +dl.pipeline-dl > dd { + margin-left: 100px; +} \ No newline at end of file diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index dd87e0e66391..783395ecc6ac 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -1,10 +1,12 @@ import React, { PropTypes } from 'react'; import { Row, Col } from 'react-bootstrap'; -import { EntityList } from 'components/common'; +import { EntityList, Timestamp } from 'components/common'; import Stage from './Stage'; import StageForm from './StageForm'; +import {} from './Pipeline.css'; + const Pipeline = React.createClass({ propTypes: { pipeline: PropTypes.object.isRequired, @@ -42,8 +44,9 @@ const Pipeline = React.createClass({ }, render() { - const maxStage = this.props.pipeline.stages.reduce((max, currentStage) => Math.max(max, currentStage.stage), -Infinity); - const formattedStages = this.props.pipeline.stages + const pipeline = this.props.pipeline; + const maxStage = pipeline.stages.reduce((max, currentStage) => Math.max(max, currentStage.stage), -Infinity); + const formattedStages = pipeline.stages .sort((s1, s2) => s1.stage - s2.stage) .map(stage => this._formatStage(stage, maxStage)); @@ -54,8 +57,15 @@ const Pipeline = React.createClass({
    -

    Description

    -

    {this.props.pipeline.description}

    +

    Information

    +
    +
    Description
    +
    {pipeline.description}
    +
    Created
    +
    +
    Last modified
    +
    +
    diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index 710cfdd1dd14..c1295f0d649e 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -46,7 +46,7 @@ const PipelineDetailsPage = React.createClass({ return (
    - Pipeline "{this.state.pipeline.title}"} titleSize={9} buttonSize={3}> + Pipeline {this.state.pipeline.title}} titleSize={9} buttonSize={3}> Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} rules are evaluated and applied. Messages can go through one or more stages. diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index 4da209bb634b..e7ee0e0c8131 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -52,7 +52,7 @@ const RuleList = React.createClass({ }, render() { const filterKeys = ['title', 'description']; - const headers = ['Title', 'Description', 'Created at', 'Last modified', 'Actions']; + const headers = ['Title', 'Description', 'Created', 'Last modified', 'Actions']; return (
    From 6da6882a769a1c545d3c320369ae9928eddf1154 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 8 Mar 2016 14:59:51 +0100 Subject: [PATCH 176/521] improved error handling for function calls - Function::evaluate is now marked as Nullable - ParameterDescriptor::required is marked as Nullable - FunctionExpression catches all exceptions and adds them to the evaluation context - the pipeline interpreter adds eval errors to the message in a special field (gl2_processing_error) - all expressions properly handle null values and return null values where appropriate - all built-in functions properly handle null values for required parameters and return null where appropriate - other null accesses in functions will trigger an exception caught be the function expressions - added test case for simple null value check --- .../pipelineprocessor/EvaluationContext.java | 36 +++++++++++++++++++ .../ast/expressions/Expression.java | 3 ++ .../expressions/FieldAccessExpression.java | 6 +++- .../ast/expressions/FunctionExpression.java | 13 +++++-- .../expressions/IndexedAccessExpression.java | 3 ++ .../ast/expressions/MessageRefExpression.java | 3 ++ .../ast/functions/ParameterDescriptor.java | 1 + .../conversion/StringConversion.java | 3 ++ .../functions/dates/FormatDate.java | 3 ++ .../functions/dates/ParseDate.java | 3 ++ .../functions/ips/CidrMatch.java | 4 ++- .../functions/ips/IpAddressConversion.java | 24 +++++++++++-- .../functions/json/SelectJsonPath.java | 4 ++- .../functions/strings/Abbreviate.java | 6 +++- .../functions/strings/Substring.java | 6 +++- .../parser/PipelineRuleParser.java | 3 +- .../processors/PipelineInterpreter.java | 23 ++++++++++++ .../pipelineprocessor/BaseParserTest.java | 11 ++++++ .../functions/FunctionsSnippetsTest.java | 12 +++++++ .../pipelineprocessor/functions/evalError.txt | 6 ++++ 20 files changed, 162 insertions(+), 11 deletions(-) create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java index 03fcecff9b02..8a068e834bd1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java @@ -18,11 +18,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; import org.graylog2.plugin.Messages; import org.joda.time.DateTime; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -43,6 +45,7 @@ public void define(String identifier, Class type, Object value) { private final Message message; private Map ruleVars; private List createdMessages = Lists.newArrayList(); + private List evalErrors; private EvaluationContext() { this(new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"))); // first Graylog release @@ -51,6 +54,7 @@ private EvaluationContext() { public EvaluationContext(Message message) { this.message = message; ruleVars = Maps.newHashMap(); + evalErrors = Lists.newArrayList(); } public void define(String identifier, Class type, Object value) { @@ -81,6 +85,18 @@ public static EvaluationContext emptyContext() { return EMPTY_CONTEXT; } + public void addEvaluationError(int line, int charPositionInLine, FunctionDescriptor descriptor, Exception e) { + evalErrors.add(new EvalError(line, charPositionInLine, descriptor, e)); + } + + public boolean hasEvaluationErrors() { + return evalErrors.size() > 0; + } + + public List evaluationErrors() { + return Collections.unmodifiableList(evalErrors); + } + public class TypedValue { private final Class type; private final Object value; @@ -98,4 +114,24 @@ public Object getValue() { return value; } } + + public static class EvalError { + private final int line; + private final int charPositionInLine; + private final FunctionDescriptor descriptor; + private final Exception e; + + public EvalError(int line, int charPositionInLine, FunctionDescriptor descriptor, Exception e) { + this.line = line; + this.charPositionInLine = charPositionInLine; + this.descriptor = descriptor; + this.e = e; + } + + @Override + public String toString() { + return "In call to function '" + descriptor.name() + "' at " + line + ":" + charPositionInLine + + " an exception was thrown: " + e.getMessage(); + } + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java index f49db4b233ca..2a6b3f8a3507 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java @@ -18,10 +18,13 @@ import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import javax.annotation.Nullable; + public interface Expression { boolean isConstant(); + @Nullable Object evaluate(EvaluationContext context); Class getType(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java index b766a4d35b16..8340b197d0f4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -42,7 +42,11 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context) { final Object bean = this.object.evaluate(context); - final String fieldName = field.evaluate(context).toString(); + final Object fieldValue = field.evaluate(context); + if (bean == null || fieldValue == null) { + return null; + } + final String fieldName = fieldValue.toString(); try { Object property = PropertyUtils.getProperty(bean, fieldName); if (property == null) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java index 6956e39773c8..9fc1f5b9bb0c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java @@ -23,11 +23,15 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; public class FunctionExpression implements Expression { + private final int line; + private final int charPositionInLine; private final FunctionArgs args; private final Function function; private final FunctionDescriptor descriptor; - public FunctionExpression(FunctionArgs args) { + public FunctionExpression(int line, int charPositionInLine, FunctionArgs args) { + this.line = line; + this.charPositionInLine = charPositionInLine; this.args = args; this.function = args.getFunction(); this.descriptor = this.function.descriptor(); @@ -51,7 +55,12 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context) { - return descriptor.returnType().cast(function.evaluate(args, context)); + try { + return descriptor.returnType().cast(function.evaluate(args, context)); + } catch (Exception e) { + context.addEvaluationError(line, charPositionInLine, descriptor, e); + return null; + } } @Override diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java index a0440366b0c3..e47a214c69fd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java @@ -42,6 +42,9 @@ public boolean isConstant() { public Object evaluate(EvaluationContext context) { final Object idxObj = this.index.evaluate(context); final Object indexable = indexableObject.evaluate(context); + if (idxObj == null || indexable == null) { + return null; + } if (idxObj instanceof Long) { int idx = Ints.saturatedCast((long) idxObj); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java index 77bb44b01f1a..dc84cde96564 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java @@ -33,6 +33,9 @@ public boolean isConstant() { @Override public Object evaluate(EvaluationContext context) { final Object fieldName = fieldExpr.evaluate(context); + if (fieldName == null) { + return null; + } return context.currentMessage().getField(fieldName.toString()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index 0abaab4a340d..d46a4ed505c6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -87,6 +87,7 @@ public static Builder type(String name, Class typeClas return ParameterDescriptor.param().type(typeClass).transformedType(transformedClass).name(name); } + @Nullable public R required(FunctionArgs args, EvaluationContext context) { final Object precomputedValue = args.getPreComputedValue(name()); if (precomputedValue != null) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 4d98d4eea60f..4b76f718aa83 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -59,6 +59,9 @@ protected boolean removeEldestEntry(Map.Entry, Class> eldest) { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { final Object evaluated = valueParam.required(args, context); + if (evaluated == null) { + return null; + } // fast path for the most common targets if (evaluated instanceof String || evaluated instanceof Number diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java index b23d17b816b2..9a11e28faf9b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java @@ -51,6 +51,9 @@ public FormatDate() { public String evaluate(FunctionArgs args, EvaluationContext context) { final DateTime dateTime = value.required(args, context); final DateTimeFormatter formatter = format.required(args, context); + if (dateTime == null || formatter == null) { + return null; + } final DateTimeZone timeZone = timeZoneParam.optional(args, context).orElse(DateTimeZone.UTC); return formatter.withZone(timeZone).print(dateTime); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java index 2fc4ca9076db..04d72b9ef21b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java @@ -55,6 +55,9 @@ protected ImmutableList params() { public DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { final String dateString = valueParam.required(args, context); final String pattern = patternParam.required(args, context); + if (dateString == null || pattern == null) { + return null; + } final DateTimeFormatter formatter = DateTimeFormat.forPattern(pattern).withZone(timezone); return formatter.parseDateTime(dateString); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java index 8690440c06df..b5d16ae89058 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java @@ -51,7 +51,9 @@ public CidrMatch() { public Boolean evaluate(FunctionArgs args, EvaluationContext context) { final CIDR cidr = cidrParam.required(args, context); final IpAddress ipAddress = ipParam.required(args, context); - + if (cidr == null || ipAddress == null) { + return null; + } return cidr.contains(ipAddress.inetAddress()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index 89749810380b..c3a1bd94f06c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -24,6 +24,8 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import java.net.InetAddress; +import java.util.IllegalFormatException; +import java.util.Optional; import static com.google.common.collect.ImmutableList.of; @@ -31,17 +33,32 @@ public class IpAddressConversion extends AbstractFunction { public static final String NAME = "toip"; private final ParameterDescriptor ipParam; + private final ParameterDescriptor defaultParam; public IpAddressConversion() { ipParam = ParameterDescriptor.object("ip").build(); + defaultParam = ParameterDescriptor.string("default").optional().build(); } @Override public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { final String ipString = String.valueOf(ipParam.required(args, context)); - final InetAddress inetAddress = InetAddresses.forString(ipString); - return new IpAddress(inetAddress); + try { + final InetAddress inetAddress = InetAddresses.forString(ipString); + return new IpAddress(inetAddress); + } catch (IllegalFormatException e) { + final Optional defaultValue = defaultParam.optional(args, context); + if (!defaultValue.isPresent()) { + return null; + } + try { + return new IpAddress(InetAddresses.forString(defaultValue.get())); + } catch (IllegalFormatException e1) { + log.warn("Parameter `default` for toip() is not a valid IP address: {}", defaultValue.get()); + throw e1; + } + } } @Override @@ -50,7 +67,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(IpAddress.class) .params(of( - ipParam + ipParam, + defaultParam )) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java index 3e64916b5c64..28dcc1b72d54 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -66,7 +66,9 @@ public SelectJsonPath(ObjectMapper objectMapper) { public Map evaluate(FunctionArgs args, EvaluationContext context) { final JsonNode json = jsonParam.required(args, context); final Map paths = pathsParam.required(args, context); - + if (json == null || paths == null) { + return null; + } return paths .entrySet().stream() .collect(toMap( diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java index 5cf394d029f3..2aabd57a0baa 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -42,7 +42,11 @@ public Abbreviate() { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { final String value = valueParam.required(args, context); - final Long maxWidth = Math.max(widthParam.required(args, context), 4L); + final Long required = widthParam.required(args, context); + if (required == null) { + return null; + } + final Long maxWidth = Math.max(required, 4L); return StringUtils.abbreviate(value, saturatedCast(maxWidth)); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java index a1c8b40703ce..aa29242351f0 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java @@ -42,7 +42,11 @@ public Substring() { @Override public String evaluate(FunctionArgs args, EvaluationContext context) { final String value = valueParam.required(args, context); - final int start = Ints.saturatedCast(startParam.required(args, context)); + final Long startValue = startParam.required(args, context); + if (value == null || startValue == null) { + return null; + } + final int start = Ints.saturatedCast(startValue); final int end = Ints.saturatedCast(endParam.optional(args, context).orElse((long) value.length())); return StringUtils.substring(value, start, end); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index fcf7d8460ad9..465c93dbc8ba 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -347,7 +347,8 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { } final FunctionExpression expr = new FunctionExpression( - new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) + ctx.getStart().getLine(), ctx.getStart().getCharPositionInLine(), + new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) ); log.trace("FUNC: ctx {} => {}", ctx, expr); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index b42acbd1abaf..e70f44d869af 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -249,6 +249,13 @@ public Messages process(Messages messages) { final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); for (Rule rule : stage.getRules()) { if (rule.when().evaluateBool(context)) { + if (context.hasEvaluationErrors()) { + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(message, lastError.toString()); + log.debug("Encountered evaluation error during condition, skipping rule actions: {}", + lastError); + continue; + } log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); rulesToRun.add(rule); } else { @@ -259,6 +266,14 @@ public Messages process(Messages messages) { log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); for (Statement statement : rule.then()) { statement.evaluate(context); + if (context.hasEvaluationErrors()) { + // if the last statement resulted in an error, do not continue to execute this rules + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(message, lastError.toString()); + log.debug("Encountered evaluation error, skipping rest of the rule: {}", + lastError); + break; + } } } // stage needed to match all rule conditions to enable the next stage, @@ -314,6 +329,14 @@ public Messages process(Messages messages) { return new MessageCollection(fullyProcessed); } + private void appendProcessingError(Message message, String errorString) { + if (message.hasField("gl2_processing_error")) { + message.addField("gl2_processing_error", message.getFieldAs(String.class, "gl2_processor_error") + "," + errorString); + } else { + message.addField("gl2_processing_error", errorString); + } + } + @Subscribe public void handleRuleChanges(RulesChangedEvent event) { event.deletedRuleIds().forEach(id -> { diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java index ff28ded0837c..2325da3775df 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java @@ -79,6 +79,17 @@ public void setup() { actionsTriggered.set(false); } + protected EvaluationContext contextForRuleEval(Rule rule, Message message) { + final EvaluationContext context = new EvaluationContext(message); + if (rule.when().evaluateBool(context)) { + + for (Statement statement : rule.then()) { + statement.evaluate(context); + } + } + return context; + } + protected Message evaluateRule(Rule rule, Message message) { final EvaluationContext context = new EvaluationContext(message); if (rule.when().evaluateBool(context)) { diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 5923599ad66c..f2355223776a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -17,7 +17,9 @@ package org.graylog.plugins.pipelineprocessor.functions; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; import org.graylog.plugins.pipelineprocessor.BaseParserTest; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.functions.conversion.BooleanConversion; @@ -258,4 +260,14 @@ public void ipMatching() { assertThat(message.getField("ipv6_anon")).isEqualTo("2001:db8::"); } + @Test + public void evalError() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final EvaluationContext context = contextForRuleEval(rule, new Message("test", "test", Tools.nowUTC())); + + assertThat(context).isNotNull(); + assertThat(context.hasEvaluationErrors()).isTrue(); + assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'toip' at 5:28 an exception was thrown: 'null' is not an IP string literal."); + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt new file mode 100644 index 000000000000..4c276e1d821c --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt @@ -0,0 +1,6 @@ +rule "trigger null" +when + true +then + set_field("this_is_null", toip($message.does_not_exist)); +end \ No newline at end of file From 2e1b448688a6aa46cac53897a5c5e5fb0f0e650f Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 8 Mar 2016 15:08:53 +0100 Subject: [PATCH 177/521] add rule name to gl2_processing_error message --- .../processors/PipelineInterpreter.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index e70f44d869af..64c1a1d12734 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -251,7 +251,7 @@ public Messages process(Messages messages) { if (rule.when().evaluateBool(context)) { if (context.hasEvaluationErrors()) { final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); - appendProcessingError(message, lastError.toString()); + appendProcessingError(rule, message, lastError.toString()); log.debug("Encountered evaluation error during condition, skipping rule actions: {}", lastError); continue; @@ -269,7 +269,7 @@ public Messages process(Messages messages) { if (context.hasEvaluationErrors()) { // if the last statement resulted in an error, do not continue to execute this rules final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); - appendProcessingError(message, lastError.toString()); + appendProcessingError(rule, message, lastError.toString()); log.debug("Encountered evaluation error, skipping rest of the rule: {}", lastError); break; @@ -329,11 +329,12 @@ public Messages process(Messages messages) { return new MessageCollection(fullyProcessed); } - private void appendProcessingError(Message message, String errorString) { + private void appendProcessingError(Rule rule, Message message, String errorString) { + final String msg = "For rule '" + rule.name() + "': " + errorString; if (message.hasField("gl2_processing_error")) { - message.addField("gl2_processing_error", message.getFieldAs(String.class, "gl2_processor_error") + "," + errorString); + message.addField("gl2_processing_error", message.getFieldAs(String.class, "gl2_processor_error") + "," + msg); } else { - message.addField("gl2_processing_error", errorString); + message.addField("gl2_processing_error", msg); } } From 791b3bf1ba6cb77e1b1be741bcc7963d43f7f668 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 8 Mar 2016 15:23:27 +0100 Subject: [PATCH 178/521] extract gl2_processing_error into a constant to avoid typos --- .../pipelineprocessor/processors/PipelineInterpreter.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 64c1a1d12734..97c7db254f28 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -73,6 +73,8 @@ public class PipelineInterpreter implements MessageProcessor { private static final Logger log = LoggerFactory.getLogger(PipelineInterpreter.class); + public static final String GL2_PROCESSING_ERROR = "gl2_processing_error"; + private final RuleService ruleService; private final PipelineService pipelineService; private final PipelineStreamConnectionsService pipelineStreamConnectionsService; @@ -331,10 +333,10 @@ public Messages process(Messages messages) { private void appendProcessingError(Rule rule, Message message, String errorString) { final String msg = "For rule '" + rule.name() + "': " + errorString; - if (message.hasField("gl2_processing_error")) { - message.addField("gl2_processing_error", message.getFieldAs(String.class, "gl2_processor_error") + "," + msg); + if (message.hasField(GL2_PROCESSING_ERROR)) { + message.addField(GL2_PROCESSING_ERROR, message.getFieldAs(String.class, GL2_PROCESSING_ERROR) + "," + msg); } else { - message.addField("gl2_processing_error", msg); + message.addField(GL2_PROCESSING_ERROR, msg); } } From abc8220355e9d823b1c0cafc0c102271deabe3ce Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 8 Mar 2016 18:16:14 +0100 Subject: [PATCH 179/521] add optional message parameter to message-functions - allows to properly use synthetic messages created with create_message - added tests --- .../functions/messages/HasField.java | 11 ++++-- .../functions/messages/RemoveField.java | 11 ++++-- .../functions/messages/RouteToStream.java | 10 ++++-- .../functions/messages/SetField.java | 10 ++++-- .../functions/messages/SetFields.java | 11 +++--- .../functions/FunctionsSnippetsTest.java | 34 +++++++++++++++++-- .../functions/newlyCreatedMessage.txt | 12 +++++++ 7 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java index 6f394428dc2b..49691d176ad4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -22,21 +22,28 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class HasField extends AbstractFunction { public static final String NAME = "has_field"; public static final String FIELD = "field"; private final ParameterDescriptor fieldParam; + private final ParameterDescriptor messageParam; public HasField() { fieldParam = ParameterDescriptor.string(FIELD).build(); + messageParam = type("message", Message.class).optional().build(); } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { final String field = fieldParam.required(args, context); - return context.currentMessage().hasField(field); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + + return message.hasField(field); } @Override @@ -44,7 +51,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(ImmutableList.of(fieldParam)) + .params(ImmutableList.of(fieldParam, messageParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java index cab516d78f97..bb2325269968 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -22,21 +22,28 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class RemoveField extends AbstractFunction { public static final String NAME = "remove_field"; public static final String FIELD = "field"; private final ParameterDescriptor fieldParam; + private final ParameterDescriptor messageParam; public RemoveField() { fieldParam = ParameterDescriptor.string(FIELD).build(); + messageParam = type("message", Message.class).optional().build(); } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { final String field = fieldParam.required(args, context); - context.currentMessage().removeField(field); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + + message.removeField(field); return null; } @@ -45,7 +52,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(ImmutableList.of(fieldParam)) + .params(ImmutableList.of(fieldParam, messageParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java index 3c86855833f4..38c4018c8adc 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -25,11 +25,13 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog2.database.NotFoundException; +import org.graylog2.plugin.Message; import org.graylog2.plugin.streams.Stream; import org.graylog2.streams.StreamService; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class RouteToStream extends AbstractFunction { @@ -37,6 +39,7 @@ public class RouteToStream extends AbstractFunction { private static final String ID_ARG = "id"; private static final String NAME_ARG = "name"; private final StreamService streamService; + private final ParameterDescriptor messageParam; private final ParameterDescriptor nameParam; private final ParameterDescriptor idParam; @@ -45,6 +48,7 @@ public RouteToStream(StreamService streamService) { this.streamService = streamService; streamService.loadAllEnabled(); + messageParam = type("message", Message.class).optional().build(); nameParam = string(NAME_ARG).optional().build(); idParam = string(ID_ARG).optional().build(); } @@ -76,7 +80,8 @@ public Void evaluate(FunctionArgs args, EvaluationContext context) { } // TODO needs message stack in context to pick message if (!stream.isPaused()) { - context.currentMessage().addStream(stream); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + message.addStream(stream); } return null; } @@ -88,7 +93,8 @@ public FunctionDescriptor descriptor() { .returnType(Void.class) .params(of( nameParam, - idParam)) + idParam, + messageParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index e7a30e9da83a..17063f9c617e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -22,10 +22,12 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class SetField extends AbstractFunction { @@ -35,10 +37,12 @@ public class SetField extends AbstractFunction { private final ParameterDescriptor fieldParam; private final ParameterDescriptor valueParam; + private final ParameterDescriptor messageParam; public SetField() { fieldParam = string(FIELD).build(); valueParam = object(VALUE).build(); + messageParam = type("message", Message.class).optional().build(); } @Override @@ -47,7 +51,8 @@ public Void evaluate(FunctionArgs args, EvaluationContext context) { final Object value = valueParam.required(args, context); if (!Strings.isNullOrEmpty(field)) { - context.currentMessage().addField(field, value); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + message.addField(field, value); } return null; } @@ -58,7 +63,8 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(of(fieldParam, - valueParam)) + valueParam, + messageParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java index fa43e5774028..ecee09e9926c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -21,26 +21,31 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; import java.util.Map; import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class SetFields extends AbstractFunction { public static final String NAME = "set_fields"; public static final String FIELDS = "fields"; private final ParameterDescriptor fieldsParam; + private final ParameterDescriptor messageParam; public SetFields() { fieldsParam = ParameterDescriptor.type(FIELDS, Map.class).build(); + messageParam = type("message", Message.class).optional().build(); } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { //noinspection unchecked final Map fields = fieldsParam.required(args, context); - context.currentMessage().addFields(fields); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + message.addFields(fields); return null; } @@ -49,9 +54,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(of( - fieldsParam - )) + .params(of(fieldsParam, messageParam)) .build(); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index f2355223776a..c2213143f4fd 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.graylog.plugins.pipelineprocessor.BaseParserTest; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.Rule; @@ -42,6 +43,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; @@ -58,7 +60,9 @@ import org.graylog2.plugin.InstantMillisProvider; import org.graylog2.plugin.Message; import org.graylog2.plugin.Tools; +import org.graylog2.plugin.streams.Stream; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.graylog2.streams.StreamService; import org.joda.time.DateTime; import org.joda.time.DateTimeUtils; import org.junit.Assert; @@ -71,6 +75,8 @@ import static org.assertj.core.api.Assertions.fail; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class FunctionsSnippetsTest extends BaseParserTest { @@ -93,8 +99,16 @@ public static void registerFunctions() { functions.put(DropMessage.NAME, new DropMessage()); functions.put(CreateMessage.NAME, new CreateMessage()); - // TODO needs mock - //functions.put(RouteToStream.NAME, new RouteToStream())); + + // route to stream mocks + final StreamService streamService = mock(StreamService.class); + final Stream stream = mock(Stream.class); + when(stream.isPaused()).thenReturn(false); + when(stream.getTitle()).thenReturn("some name"); + when(stream.getId()).thenReturn("id"); + when(streamService.loadAll()).thenReturn(Lists.newArrayList(stream)); + + functions.put(RouteToStream.NAME, new RouteToStream(streamService)); // input related functions // TODO needs mock @@ -270,4 +284,20 @@ public void evalError() { assertThat(context.hasEvaluationErrors()).isTrue(); assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'toip' at 5:28 an exception was thrown: 'null' is not an IP string literal."); } + + @Test + public void newlyCreatedMessage() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final EvaluationContext context = contextForRuleEval(rule, new Message("test", "test", Tools.nowUTC())); + + final Message origMessage = context.currentMessage(); + final Message newMessage = Iterables.getOnlyElement(context.createdMessages()); + + assertThat(origMessage).isNotSameAs(newMessage); + assertThat(newMessage.hasField("removed_again")).isFalse(); + assertThat(newMessage.getFieldAs(Boolean.class, "has_source")).isTrue(); + assertThat(newMessage.getFieldAs(String.class, "only_in")).isEqualTo("new message"); + assertThat(newMessage.getFieldAs(String.class, "multi")).isEqualTo("new message"); + + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt new file mode 100644 index 000000000000..d4c63c6fb095 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt @@ -0,0 +1,12 @@ +rule "operate on newly created message" +when true +then + let x = create_message("new", "synthetic", now()); + + set_field("removed_again", "foo", x); + set_field("only_in", "new message", x); + set_fields({ multi: "new message" }, x); + set_field("has_source", has_field("source", x), x); + route_to_stream(name: "some stream", message: x); + remove_field("removed_again", x); +end \ No newline at end of file From 45012cb8feb4e97f1d455002fd7215b974d19b89 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 9 Mar 2016 14:45:44 +0100 Subject: [PATCH 180/521] improve error handling in functions to track exception location Exceptions originating in function calls always triggered an eval error, even if we wanted to suppress them with the new is_null/is_not_null functions. For each expression, keep track of the start token to annotate exceptions originating during the evaluation. As a first step function expressions propagate the innermost exception which other functions can catch and suppress. Also implements the is_null/is_not_null functions which suppress exceptions and null values and turn them into booleans. --- .../pipelineprocessor/EvaluationContext.java | 24 ++- .../plugins/pipelineprocessor/ast/Rule.java | 3 +- .../FunctionEvaluationException.java | 38 +++++ .../LocationAwareEvalException.java | 32 ++++ .../ast/expressions/AbstractExpression.java | 61 ++++++++ .../ast/expressions/AndExpression.java | 9 +- .../expressions/ArrayLiteralExpression.java | 14 +- .../ast/expressions/BinaryExpression.java | 10 +- .../ast/expressions/BooleanExpression.java | 7 +- .../BooleanValuedFunctionWrapper.java | 12 +- .../ast/expressions/ComparisonExpression.java | 11 +- .../ast/expressions/ConstantExpression.java | 7 +- .../ast/expressions/DoubleExpression.java | 7 +- .../ast/expressions/EqualityExpression.java | 11 +- .../expressions/FieldAccessExpression.java | 16 +- .../ast/expressions/FieldRefExpression.java | 8 +- .../ast/expressions/FunctionExpression.java | 21 +-- .../expressions/IndexedAccessExpression.java | 16 +- .../ast/expressions/LongExpression.java | 7 +- .../ast/expressions/MapLiteralExpression.java | 12 +- .../ast/expressions/MessageRefExpression.java | 12 +- .../ast/expressions/NotExpression.java | 7 +- .../ast/expressions/OrExpression.java | 9 +- .../ast/expressions/StringExpression.java | 7 +- .../ast/expressions/UnaryExpression.java | 11 +- .../ast/expressions/VarRefExpression.java | 8 +- .../ast/functions/AbstractFunction.java | 6 +- .../ast/functions/Function.java | 6 +- .../ast/functions/FunctionArgs.java | 12 +- .../ast/functions/ParameterDescriptor.java | 6 +- .../ast/statements/FunctionStatement.java | 16 +- .../ast/statements/VarAssignStatement.java | 6 +- .../functions/IsNotNull.java | 54 +++++++ .../pipelineprocessor/functions/IsNull.java | 54 +++++++ .../functions/ProcessorFunctionsModule.java | 4 + .../parser/PipelineRuleParser.java | 143 +++++++++--------- .../parser/errors/IncompatibleTypes.java | 4 +- .../IndexedAccessExpressionTest.java | 35 +++-- .../functions/FunctionsSnippetsTest.java | 16 ++ .../functions/evalErrorSuppressed.txt | 6 + 40 files changed, 528 insertions(+), 220 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java index 8a068e834bd1..b211b0c155ca 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java @@ -24,6 +24,7 @@ import org.graylog2.plugin.Messages; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; import java.util.Map; @@ -85,7 +86,7 @@ public static EvaluationContext emptyContext() { return EMPTY_CONTEXT; } - public void addEvaluationError(int line, int charPositionInLine, FunctionDescriptor descriptor, Exception e) { + public void addEvaluationError(int line, int charPositionInLine, @Nullable FunctionDescriptor descriptor, Throwable e) { evalErrors.add(new EvalError(line, charPositionInLine, descriptor, e)); } @@ -118,20 +119,31 @@ public Object getValue() { public static class EvalError { private final int line; private final int charPositionInLine; + @Nullable private final FunctionDescriptor descriptor; - private final Exception e; + private final Throwable throwable; - public EvalError(int line, int charPositionInLine, FunctionDescriptor descriptor, Exception e) { + public EvalError(int line, int charPositionInLine, @Nullable FunctionDescriptor descriptor, Throwable throwable) { this.line = line; this.charPositionInLine = charPositionInLine; this.descriptor = descriptor; - this.e = e; + this.throwable = throwable; } @Override public String toString() { - return "In call to function '" + descriptor.name() + "' at " + line + ":" + charPositionInLine + - " an exception was thrown: " + e.getMessage(); + final StringBuilder sb = new StringBuilder(); + if (descriptor != null) { + sb.append("In call to function '").append(descriptor.name()).append("' at "); + } else { + sb.append("At "); + } + return sb.append(line) + .append(":") + .append(charPositionInLine) + .append(" an exception was thrown: ") + .append(throwable.getMessage()) + .toString(); } } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java index 1cef97208534..4ac5271c9360 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java @@ -17,6 +17,7 @@ package org.graylog.plugins.pipelineprocessor.ast; import com.google.auto.value.AutoValue; +import org.antlr.v4.runtime.CommonToken; import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; @@ -38,7 +39,7 @@ public static Builder builder() { } public static Rule alwaysFalse(String name) { - return builder().name(name).when(new BooleanExpression(false)).then(Collections.emptyList()).build(); + return builder().name(name).when(new BooleanExpression(new CommonToken(-1), false)).then(Collections.emptyList()).build(); } @AutoValue.Builder public abstract static class Builder { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java new file mode 100644 index 000000000000..f6f3a6cb247e --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java @@ -0,0 +1,38 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.exceptions; + +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; + +public class FunctionEvaluationException extends LocationAwareEvalException { + private final FunctionExpression functionExpression; + private final Exception exception; + + public FunctionEvaluationException(FunctionExpression functionExpression, Exception exception) { + super(functionExpression.getStartToken(), exception); + this.functionExpression = functionExpression; + this.exception = exception; + } + + public FunctionExpression getFunctionExpression() { + return functionExpression; + } + + public Exception getException() { + return exception; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java new file mode 100644 index 000000000000..9c7d62522cca --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java @@ -0,0 +1,32 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.exceptions; + +import org.antlr.v4.runtime.Token; + +public class LocationAwareEvalException extends RuntimeException { + private final Token startToken; + + public LocationAwareEvalException(Token startToken, Throwable cause) { + super(cause); + this.startToken = startToken; + } + + public Token getStartToken() { + return startToken; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java new file mode 100644 index 000000000000..67ebd2973150 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java @@ -0,0 +1,61 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.FunctionEvaluationException; + +import javax.annotation.Nullable; + +import static org.graylog2.shared.utilities.ExceptionUtils.getRootCause; + +public abstract class AbstractExpression implements Expression { + + private final Token startToken; + + public AbstractExpression(Token startToken) { + this.startToken = startToken; + } + + public Token getStartToken() { + return startToken; + } + + @Override + @Nullable + public Object evaluate(EvaluationContext context) { + try { + return evaluateUnsafe(context); + } catch (FunctionEvaluationException fee) { + context.addEvaluationError(fee.getStartToken().getLine(), + fee.getStartToken().getCharPositionInLine(), + fee.getFunctionExpression().getFunction().descriptor(), + getRootCause(fee)); + } catch (Exception e) { + context.addEvaluationError(startToken.getLine(), startToken.getCharPositionInLine(), null, getRootCause(e)); + } + return null; + } + + @Nullable + /** + * This method is allow to throw exceptions. The outside world is supposed to call evaluate instead. + */ + public abstract Object evaluateUnsafe(EvaluationContext context); + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java index 01037e68b99d..06c10aa3edea 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java @@ -16,16 +16,17 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class AndExpression extends BinaryExpression implements LogicalExpression { - public AndExpression(LogicalExpression left, - LogicalExpression right) { - super(left, right); + public AndExpression(Token start, AbstractExpression left, + AbstractExpression right) { + super(start, left, right); } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return evaluateBool(context); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java index bb71eb8bfc59..1cd721cfb0e4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java @@ -17,15 +17,17 @@ package org.graylog.plugins.pipelineprocessor.ast.expressions; import com.google.common.base.Joiner; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import java.util.List; import java.util.stream.Collectors; -public class ArrayLiteralExpression implements Expression { - private final List elements; +public class ArrayLiteralExpression extends AbstractExpression { + private final List elements; - public ArrayLiteralExpression(List elements) { + public ArrayLiteralExpression(Token start, List elements) { + super(start); this.elements = elements; } @@ -35,9 +37,9 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { - return elements.stream() - .map(expression -> expression.evaluate(context)) + public Object evaluateUnsafe(EvaluationContext context) { + return elements.stream() + .map(expression -> expression.evaluateUnsafe(context)) .collect(Collectors.toList()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java index 63c0702b6f65..6e23f42cacae 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java @@ -16,12 +16,14 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; + public abstract class BinaryExpression extends UnaryExpression { - protected final Expression left; + protected final AbstractExpression left; - public BinaryExpression(Expression left, Expression right) { - super(right); + public BinaryExpression(Token start, AbstractExpression left, AbstractExpression right) { + super(start, right); this.left = left; } @@ -30,7 +32,7 @@ public boolean isConstant() { return left.isConstant() && right.isConstant(); } - public Expression left() { + public AbstractExpression left() { return left; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java index cb7bf03b2ef2..c6855516f3ff 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java @@ -16,18 +16,19 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class BooleanExpression extends ConstantExpression implements LogicalExpression { private final boolean value; - public BooleanExpression(boolean value) { - super(Boolean.class); + public BooleanExpression(Token start, boolean value) { + super(start, Boolean.class); this.value = value; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java index 426cb00cb58a..bde6626b13f8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -16,12 +16,14 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -public class BooleanValuedFunctionWrapper implements LogicalExpression { - private final Expression expr; +public class BooleanValuedFunctionWrapper extends AbstractExpression implements LogicalExpression { + private final AbstractExpression expr; - public BooleanValuedFunctionWrapper(Expression expr) { + public BooleanValuedFunctionWrapper(Token start, AbstractExpression expr) { + super(start); this.expr = expr; if (!expr.getType().equals(Boolean.class)) { throw new IllegalArgumentException("expr must be of boolean type"); @@ -30,7 +32,7 @@ public BooleanValuedFunctionWrapper(Expression expr) { @Override public boolean evaluateBool(EvaluationContext context) { - final Object value = expr.evaluate(context); + final Object value = expr.evaluateUnsafe(context); return value != null && (Boolean) value; } @@ -40,7 +42,7 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return evaluateBool(context); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java index 776762ef1509..df99340fbb2b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java @@ -16,18 +16,19 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class ComparisonExpression extends BinaryExpression implements LogicalExpression { private final String operator; - public ComparisonExpression(Expression left, Expression right, String operator) { - super(left, right); + public ComparisonExpression(Token start, AbstractExpression left, AbstractExpression right, String operator) { + super(start, left, right); this.operator = operator; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return evaluateBool(context); } @@ -39,8 +40,8 @@ public Class getType() { @Override public boolean evaluateBool(EvaluationContext context) { - final Object leftValue = this.left.evaluate(context); - final Object rightValue = this.right.evaluate(context); + final Object leftValue = this.left.evaluateUnsafe(context); + final Object rightValue = this.right.evaluateUnsafe(context); if (leftValue instanceof Double || rightValue instanceof Double) { return compareDouble(operator, (double) leftValue, (double) rightValue); } else { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java index dd215610f97c..d3cdecc1cec3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java @@ -16,11 +16,14 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; -public abstract class ConstantExpression implements Expression { +import org.antlr.v4.runtime.Token; + +public abstract class ConstantExpression extends AbstractExpression { private final Class type; - protected ConstantExpression(Class type) { + protected ConstantExpression(Token start, Class type) { + super(start); this.type = type; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java index ea476c7808d2..ef4a2574d629 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java @@ -16,18 +16,19 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class DoubleExpression extends ConstantExpression implements NumericExpression { private final double value; - public DoubleExpression(double value) { - super(Double.class); + public DoubleExpression(Token start, double value) { + super(start, Double.class); this.value = value; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java index 43272bb44879..45ab54d13b1c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java @@ -16,6 +16,7 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.joda.time.DateTime; import org.slf4j.Logger; @@ -26,13 +27,13 @@ public class EqualityExpression extends BinaryExpression implements LogicalExpre private final boolean checkEquality; - public EqualityExpression(Expression left, Expression right, boolean checkEquality) { - super(left, right); + public EqualityExpression(Token start, AbstractExpression left, AbstractExpression right, boolean checkEquality) { + super(start, left, right); this.checkEquality = checkEquality; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return evaluateBool(context); } @@ -43,8 +44,8 @@ public Class getType() { @Override public boolean evaluateBool(EvaluationContext context) { - final Object left = this.left.evaluate(context); - final Object right = this.right.evaluate(context); + final Object left = this.left.evaluateUnsafe(context); + final Object right = this.right.evaluateUnsafe(context); if (left == null) { log.warn("left expression evaluated to null, returning false: {}", this.left); return false; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java index 8340b197d0f4..87d0015e515e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -16,6 +16,7 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.apache.commons.beanutils.PropertyUtils; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.slf4j.Logger; @@ -23,13 +24,14 @@ import java.lang.reflect.InvocationTargetException; -public class FieldAccessExpression implements Expression { +public class FieldAccessExpression extends AbstractExpression { private static final Logger log = LoggerFactory.getLogger(FieldAccessExpression.class); - private final Expression object; - private final Expression field; + private final AbstractExpression object; + private final AbstractExpression field; - public FieldAccessExpression(Expression object, Expression field) { + public FieldAccessExpression(Token start, AbstractExpression object, AbstractExpression field) { + super(start); this.object = object; this.field = field; } @@ -40,9 +42,9 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { - final Object bean = this.object.evaluate(context); - final Object fieldValue = field.evaluate(context); + public Object evaluateUnsafe(EvaluationContext context) { + final Object bean = this.object.evaluateUnsafe(context); + final Object fieldValue = field.evaluateUnsafe(context); if (bean == null || fieldValue == null) { return null; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java index a5026c57f66a..bb7d8190fe6d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java @@ -16,12 +16,14 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -public class FieldRefExpression implements Expression { +public class FieldRefExpression extends AbstractExpression { private final String variableName; - public FieldRefExpression(String variableName) { + public FieldRefExpression(Token start, String variableName) { + super(start); this.variableName = variableName; } @@ -31,7 +33,7 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return variableName; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java index 9fc1f5b9bb0c..97bd78485c2d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java @@ -17,21 +17,21 @@ package org.graylog.plugins.pipelineprocessor.ast.expressions; import com.google.common.base.Joiner; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.FunctionEvaluationException; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.LocationAwareEvalException; import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; -public class FunctionExpression implements Expression { - private final int line; - private final int charPositionInLine; +public class FunctionExpression extends AbstractExpression { private final FunctionArgs args; private final Function function; private final FunctionDescriptor descriptor; - public FunctionExpression(int line, int charPositionInLine, FunctionArgs args) { - this.line = line; - this.charPositionInLine = charPositionInLine; + public FunctionExpression(Token start, FunctionArgs args) { + super(start); this.args = args; this.function = args.getFunction(); this.descriptor = this.function.descriptor(); @@ -54,12 +54,15 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { try { return descriptor.returnType().cast(function.evaluate(args, context)); + } catch (LocationAwareEvalException laee) { + // the exception already has a location from the input source, simply propagate it. + throw laee; } catch (Exception e) { - context.addEvaluationError(line, charPositionInLine, descriptor, e); - return null; + // we need to wrap the original exception to retain the position in the tree where the exception originated + throw new FunctionEvaluationException(this, e); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java index e47a214c69fd..fe3c639d8387 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java @@ -18,17 +18,19 @@ import com.google.common.collect.Iterables; import com.google.common.primitives.Ints; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import java.lang.reflect.Array; import java.util.List; import java.util.Map; -public class IndexedAccessExpression implements Expression { - private final Expression indexableObject; - private final Expression index; +public class IndexedAccessExpression extends AbstractExpression { + private final AbstractExpression indexableObject; + private final AbstractExpression index; - public IndexedAccessExpression(Expression indexableObject, Expression index) { + public IndexedAccessExpression(Token start, AbstractExpression indexableObject, AbstractExpression index) { + super(start); this.indexableObject = indexableObject; this.index = index; } @@ -39,9 +41,9 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { - final Object idxObj = this.index.evaluate(context); - final Object indexable = indexableObject.evaluate(context); + public Object evaluateUnsafe(EvaluationContext context) { + final Object idxObj = this.index.evaluateUnsafe(context); + final Object indexable = indexableObject.evaluateUnsafe(context); if (idxObj == null || indexable == null) { return null; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java index 0aa3f50ded7d..757a39ca11ac 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java @@ -16,18 +16,19 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class LongExpression extends ConstantExpression implements NumericExpression { private final long value; - public LongExpression(long value) { - super(Long.class); + public LongExpression(Token start, long value) { + super(start, Long.class); this.value = value; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java index d192b4e1a1f2..69addda912a5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java @@ -17,6 +17,7 @@ package org.graylog.plugins.pipelineprocessor.ast.expressions; import com.google.common.base.Joiner; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.jooq.lambda.Seq; import org.jooq.lambda.tuple.Tuple2; @@ -24,10 +25,11 @@ import java.util.HashMap; import java.util.Map; -public class MapLiteralExpression implements Expression { - private final HashMap map; +public class MapLiteralExpression extends AbstractExpression { + private final HashMap map; - public MapLiteralExpression(HashMap map) { + public MapLiteralExpression(Token start, HashMap map) { + super(start); this.map = map; } @@ -37,10 +39,10 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { // evaluate all values for each key and return the resulting map return Seq.seq(map) - .map(entry -> entry.map2(value -> value.evaluate(context))) + .map(entry -> entry.map2(value -> value.evaluateUnsafe(context))) .toMap(Tuple2::v1, Tuple2::v2); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java index dc84cde96564..632e9f4f4617 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java @@ -16,12 +16,14 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -public class MessageRefExpression implements Expression { - private final Expression fieldExpr; +public class MessageRefExpression extends AbstractExpression { + private final AbstractExpression fieldExpr; - public MessageRefExpression(Expression fieldExpr) { + public MessageRefExpression(Token start, AbstractExpression fieldExpr) { + super(start); this.fieldExpr = fieldExpr; } @@ -31,8 +33,8 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { - final Object fieldName = fieldExpr.evaluate(context); + public Object evaluateUnsafe(EvaluationContext context) { + final Object fieldName = fieldExpr.evaluateUnsafe(context); if (fieldName == null) { return null; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java index 3fb8b0d743dd..154df4c5c9dd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java @@ -16,15 +16,16 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class NotExpression extends UnaryExpression implements LogicalExpression { - public NotExpression(LogicalExpression right) { - super(right); + public NotExpression(Token start, AbstractExpression right) { + super(start, right); } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return !evaluateBool(context); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java index 1cbc95a2b128..19a9cc2f9f15 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java @@ -16,16 +16,17 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class OrExpression extends BinaryExpression implements LogicalExpression { - public OrExpression(LogicalExpression left, - LogicalExpression right) { - super(left, right); + public OrExpression(Token start, AbstractExpression left, + AbstractExpression right) { + super(start, left, right); } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return evaluateBool(context); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java index 1214c0aea20b..61e4b7dbcc90 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java @@ -16,19 +16,20 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class StringExpression extends ConstantExpression { private final String value; - public StringExpression(String value) { - super(String.class); + public StringExpression(Token start, String value) { + super(start, String.class); this.value = value; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return value; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java index db1d35205994..c905586df84b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java @@ -16,11 +16,14 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; -public abstract class UnaryExpression implements Expression { +import org.antlr.v4.runtime.Token; - protected final Expression right; +public abstract class UnaryExpression extends AbstractExpression { - public UnaryExpression(Expression right) { + protected final AbstractExpression right; + + public UnaryExpression(Token start, AbstractExpression right) { + super(start); this.right = right; } @@ -34,7 +37,7 @@ public Class getType() { return right.getType(); } - public Expression right() { + public AbstractExpression right() { return right; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java index 27c32408df59..2af54eef482d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java @@ -16,16 +16,18 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class VarRefExpression implements Expression { +public class VarRefExpression extends AbstractExpression { private static final Logger log = LoggerFactory.getLogger(VarRefExpression.class); private final String identifier; private Class type = Object.class; - public VarRefExpression(String identifier) { + public VarRefExpression(Token start, String identifier) { + super(start); this.identifier = identifier; } @@ -35,7 +37,7 @@ public boolean isConstant() { } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { final EvaluationContext.TypedValue typedValue = context.get(identifier); if (typedValue != null) { return typedValue.getValue(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java index c668f2e07628..303999bac764 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java @@ -17,7 +17,7 @@ package org.graylog.plugins.pipelineprocessor.ast.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; /** * Helper Function implementation which evaluates and memoizes all constant FunctionArgs. @@ -27,7 +27,7 @@ public abstract class AbstractFunction implements Function { @Override - public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { - return arg.evaluate(EvaluationContext.emptyContext()); + public Object preComputeConstantArgument(FunctionArgs args, String name, AbstractExpression arg) { + return arg.evaluateUnsafe(EvaluationContext.emptyContext()); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index cdd46b30a52a..99f55e70b38e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ public FunctionDescriptor descriptor() { }; default void preprocessArgs(FunctionArgs args) { - for (Map.Entry e : args.getConstantArgs().entrySet()) { + for (Map.Entry e : args.getConstantArgs().entrySet()) { final String name = e.getKey(); try { final Object value = preComputeConstantArgument(args, name, e.getValue()); @@ -72,7 +72,7 @@ default void preprocessArgs(FunctionArgs args) { * @param arg the expression tree for the argument * @return the precomputed value for the argument or null if the value should be dynamically calculated for each invocation */ - Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg); + Object preComputeConstantArgument(FunctionArgs args, String name, AbstractExpression arg); T evaluate(FunctionArgs args, EvaluationContext context); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index b157d7cca9f3..30dadda18366 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -17,7 +17,7 @@ package org.graylog.plugins.pipelineprocessor.ast.functions; import com.google.common.collect.Maps; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -31,25 +31,25 @@ public class FunctionArgs { @Nonnull - private final Map args; + private final Map args; private final Map constantValues = Maps.newHashMap(); private final Function function; private final FunctionDescriptor descriptor; - public FunctionArgs(Function func, Map args) { + public FunctionArgs(Function func, Map args) { function = func; descriptor = function.descriptor(); this.args = firstNonNull(args, Collections.emptyMap()); } @Nonnull - public Map getArgs() { + public Map getArgs() { return args; } @Nonnull - public Map getConstantArgs() { + public Map getConstantArgs() { return args.entrySet().stream() .filter(e -> e.getValue().isConstant()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -60,7 +60,7 @@ public boolean isPresent(String key) { } @Nullable - public Expression expression(String key) { + public AbstractExpression expression(String key) { return args.get(key); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index d46a4ed505c6..b232019d713f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -18,7 +18,7 @@ import com.google.auto.value.AutoValue; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import javax.annotation.Nullable; import java.util.Optional; @@ -93,11 +93,11 @@ public R required(FunctionArgs args, EvaluationContext context) { if (precomputedValue != null) { return transformedType().cast(precomputedValue); } - final Expression valueExpr = args.expression(name()); + final AbstractExpression valueExpr = args.expression(name()); if (valueExpr == null) { return null; } - final Object value = valueExpr.evaluate(context); + final Object value = valueExpr.evaluateUnsafe(context); return transformedType().cast(transform().apply(type().cast(value))); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java index 0dc18517b46b..57a220419ba3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java @@ -17,27 +17,19 @@ package org.graylog.plugins.pipelineprocessor.ast.statements; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; public class FunctionStatement implements Statement { - private static final Logger log = LoggerFactory.getLogger(FunctionStatement.class); - private final Expression functionExpression; + private final AbstractExpression functionExpression; - public FunctionStatement(Expression functionExpression) { + public FunctionStatement(AbstractExpression functionExpression) { this.functionExpression = functionExpression; } @Override public Object evaluate(EvaluationContext context) { - try { - return functionExpression.evaluate(context); - } catch (Exception e) { - log.debug("Exception during statement evaluation, skipping statement", e); - return null; - } + return functionExpression.evaluate(context); } @Override diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java index f88e4d57c98b..3ce5339c4239 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java @@ -17,13 +17,13 @@ package org.graylog.plugins.pipelineprocessor.ast.statements; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; public class VarAssignStatement implements Statement { private final String name; - private final Expression expr; + private final AbstractExpression expr; - public VarAssignStatement(String name, Expression expr) { + public VarAssignStatement(String name, AbstractExpression expr) { this.name = name; this.expr = expr; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java new file mode 100644 index 000000000000..7c5fd33114ce --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class IsNotNull extends AbstractFunction { + + public static final String NAME = "is_not_null"; + private final ParameterDescriptor valueParam; + + public IsNotNull() { + valueParam = ParameterDescriptor.type("value", Object.class).build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + try { + final Object value = valueParam.required(args, context); + return value != null; + } catch (Exception e) { + return false; + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(valueParam)) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java new file mode 100644 index 000000000000..adecd99b6e63 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class IsNull extends AbstractFunction { + + public static final String NAME = "is_null"; + private final ParameterDescriptor valueParam; + + public IsNull() { + valueParam = ParameterDescriptor.type("value", Object.class).build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + try { + final Object value = valueParam.required(args, context); + return value == null; + } catch (Exception e) { + return true; + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(valueParam)) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 3d18004782b3..4bf621508e30 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -108,6 +108,10 @@ protected void configure() { // ip handling addMessageProcessorFunction(CidrMatch.NAME, CidrMatch.class); addMessageProcessorFunction(IpAddressConversion.NAME, IpAddressConversion.class); + + // null support + addMessageProcessorFunction(IsNull.NAME, IsNull.class); + addMessageProcessorFunction(IsNotNull.NAME, IsNotNull.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 465c93dbc8ba..7a40647aaee5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -35,6 +35,7 @@ import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; @@ -43,7 +44,6 @@ import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; @@ -210,9 +210,9 @@ public void syntaxError(Recognizer recognizer, private class RuleAstBuilder extends RuleLangBaseListener { private final ParseContext parseContext; - private final ParseTreeProperty> args; - private final ParseTreeProperty> argsList; - private final ParseTreeProperty exprs; + private final ParseTreeProperty> args; + private final ParseTreeProperty> argsList; + private final ParseTreeProperty exprs; private final Set definedVars = Sets.newHashSet(); @@ -231,15 +231,15 @@ public RuleAstBuilder(ParseContext parseContext) { public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { final Rule.Builder ruleBuilder = Rule.builder(); ruleBuilder.name(unquote(ctx.name == null ? "" : ctx.name.getText(), '"')); - final Expression expr = exprs.get(ctx.condition); + final AbstractExpression expr = exprs.get(ctx.condition); LogicalExpression condition; if (expr instanceof LogicalExpression) { condition = (LogicalExpression) expr; } else if (expr != null && expr.getType().equals(Boolean.class)) { - condition = new BooleanValuedFunctionWrapper(expr); + condition = new BooleanValuedFunctionWrapper(ctx.getStart(), expr); } else { - condition = new BooleanExpression(false); + condition = new BooleanExpression(ctx.getStart(), false); log.debug("Unable to create condition, replacing with 'false'"); } ruleBuilder.when(condition); @@ -251,7 +251,7 @@ public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { @Override public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { - final Expression expr = exprs.get(ctx.functionCall()); + final AbstractExpression expr = exprs.get(ctx.functionCall()); final FunctionStatement functionStatement = new FunctionStatement(expr); parseContext.statements.add(functionStatement); } @@ -259,7 +259,7 @@ public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { @Override public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { final String name = unquote(ctx.varName.getText(), '`'); - final Expression expr = exprs.get(ctx.expression()); + final AbstractExpression expr = exprs.get(ctx.expression()); parseContext.defineVar(name, expr); definedVars.add(name); parseContext.statements.add(new VarAssignStatement(name, expr)); @@ -268,8 +268,8 @@ public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { @Override public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); - Map argsMap = this.args.get(ctx.arguments()); - final List positionalArgs = this.argsList.get(ctx.arguments()); + Map argsMap = this.args.get(ctx.arguments()); + final List positionalArgs = this.argsList.get(ctx.arguments()); final Function function = functionRegistry.resolve(name); if (function == null) { @@ -284,7 +284,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { parseContext.addError(new WrongNumberOfArgs(ctx, function, argsMap.size())); } else { // there are optional parameters, check that all required ones are present - final Map givenArguments = argsMap; + final Map givenArguments = argsMap; final List missingParams = params.stream() .filter(p -> !p.optional()) @@ -335,7 +335,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { // the remaining parameters were optional, so we can safely skip them break; } - final Expression argExpr = positionalArgs.get(i); + final AbstractExpression argExpr = positionalArgs.get(i); argsMap.put(p.name(), argExpr); i++; } @@ -347,8 +347,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { } final FunctionExpression expr = new FunctionExpression( - ctx.getStart().getLine(), ctx.getStart().getCharPositionInLine(), - new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) + ctx.getStart(), new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) ); log.trace("FUNC: ctx {} => {}", ctx, expr); @@ -357,10 +356,10 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { @Override public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { - final Map argMap = Maps.newHashMap(); + final Map argMap = Maps.newHashMap(); for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { final String argName = unquote(propAssignmentContext.Identifier().getText(), '`'); - final Expression argValue = exprs.get(propAssignmentContext.expression()); + final AbstractExpression argValue = exprs.get(propAssignmentContext.expression()); argMap.put(argName, argValue); } args.put(ctx, argMap); @@ -368,7 +367,7 @@ public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { @Override public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { - List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); + List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); expressions.addAll(ctx.expression().stream().map(exprs::get).collect(toList())); argsList.put(ctx, expressions); } @@ -382,17 +381,17 @@ public void enterNested(RuleLangParser.NestedContext ctx) { @Override public void exitNested(RuleLangParser.NestedContext ctx) { isIdIsFieldAccess.pop(); // reset for error checks - final Expression object = exprs.get(ctx.fieldSet); - final Expression field = exprs.get(ctx.field); - final FieldAccessExpression expr = new FieldAccessExpression(object, field); + final AbstractExpression object = exprs.get(ctx.fieldSet); + final AbstractExpression field = exprs.get(ctx.field); + final FieldAccessExpression expr = new FieldAccessExpression(ctx.getStart(), object, field); log.trace("FIELDACCESS: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitNot(RuleLangParser.NotContext ctx) { - final LogicalExpression expression = upgradeBoolFunctionExpression(ctx.expression()); - final NotExpression expr = new NotExpression(expression); + final AbstractExpression expression = upgradeBoolFunctionExpression(ctx.expression()); + final NotExpression expr = new NotExpression(ctx.getStart(), expression); log.trace("NOT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -401,47 +400,47 @@ public void exitNot(RuleLangParser.NotContext ctx) { public void exitAnd(RuleLangParser.AndContext ctx) { // if the expressions are function calls but boolean valued, upgrade them, // we allow testing boolean valued functions without explicit comparison operator - final LogicalExpression left = upgradeBoolFunctionExpression(ctx.left); - final LogicalExpression right = upgradeBoolFunctionExpression(ctx.right); + final AbstractExpression left = upgradeBoolFunctionExpression(ctx.left); + final AbstractExpression right = upgradeBoolFunctionExpression(ctx.right); - final AndExpression expr = new AndExpression(left, right); + final AndExpression expr = new AndExpression(ctx.getStart(), left, right); log.trace("AND: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } - private LogicalExpression upgradeBoolFunctionExpression(RuleLangParser.ExpressionContext leftExprContext) { - Expression leftExpr = exprs.get(leftExprContext); + private AbstractExpression upgradeBoolFunctionExpression(RuleLangParser.ExpressionContext leftExprContext) { + AbstractExpression leftExpr = exprs.get(leftExprContext); if (leftExpr instanceof FunctionExpression && leftExpr.getType().equals(Boolean.class)) { - leftExpr = new BooleanValuedFunctionWrapper(leftExpr); + leftExpr = new BooleanValuedFunctionWrapper(leftExprContext.getStart(), leftExpr); } - return (LogicalExpression) leftExpr; + return leftExpr; } @Override public void exitOr(RuleLangParser.OrContext ctx) { - final LogicalExpression left = upgradeBoolFunctionExpression(ctx.left); - final LogicalExpression right = upgradeBoolFunctionExpression(ctx.right); - final OrExpression expr = new OrExpression(left, right); + final AbstractExpression left = upgradeBoolFunctionExpression(ctx.left); + final AbstractExpression right = upgradeBoolFunctionExpression(ctx.right); + final OrExpression expr = new OrExpression(ctx.getStart(), left, right); log.trace("OR: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitEquality(RuleLangParser.EqualityContext ctx) { - final Expression left = exprs.get(ctx.left); - final Expression right = exprs.get(ctx.right); + final AbstractExpression left = exprs.get(ctx.left); + final AbstractExpression right = exprs.get(ctx.right); final boolean equals = ctx.equality.getText().equals("=="); - final EqualityExpression expr = new EqualityExpression(left, right, equals); + final EqualityExpression expr = new EqualityExpression(ctx.getStart(), left, right, equals); log.trace("EQUAL: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitComparison(RuleLangParser.ComparisonContext ctx) { - final Expression left = exprs.get(ctx.left); - final Expression right = exprs.get(ctx.right); + final AbstractExpression left = exprs.get(ctx.left); + final AbstractExpression right = exprs.get(ctx.right); final String operator = ctx.comparison.getText(); - final ComparisonExpression expr = new ComparisonExpression(left, right, operator); + final ComparisonExpression expr = new ComparisonExpression(ctx.getStart(), left, right, operator); log.trace("COMPARE: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -449,14 +448,14 @@ public void exitComparison(RuleLangParser.ComparisonContext ctx) { @Override public void exitInteger(RuleLangParser.IntegerContext ctx) { // TODO handle different radix and length - final LongExpression expr = new LongExpression(Long.parseLong(ctx.getText())); + final LongExpression expr = new LongExpression(ctx.getStart(), Long.parseLong(ctx.getText())); log.trace("INT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitFloat(RuleLangParser.FloatContext ctx) { - final DoubleExpression expr = new DoubleExpression(Double.parseDouble(ctx.getText())); + final DoubleExpression expr = new DoubleExpression(ctx.getStart(), Double.parseDouble(ctx.getText())); log.trace("FLOAT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -470,14 +469,14 @@ public void exitChar(RuleLangParser.CharContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { final String text = unquote(ctx.getText(), '\"'); - final StringExpression expr = new StringExpression(text); + final StringExpression expr = new StringExpression(ctx.getStart(), text); log.trace("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @Override public void exitBoolean(RuleLangParser.BooleanContext ctx) { - final BooleanExpression expr = new BooleanExpression(Boolean.valueOf(ctx.getText())); + final BooleanExpression expr = new BooleanExpression(ctx.getStart(), Boolean.valueOf(ctx.getText())); log.trace("BOOL: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -491,19 +490,19 @@ public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { @Override public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { - final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); - exprs.put(ctx, new ArrayLiteralExpression(elements)); + final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); + exprs.put(ctx, new ArrayLiteralExpression(ctx.getStart(), elements)); } @Override public void exitMapLiteralExpr(RuleLangParser.MapLiteralExprContext ctx) { - final HashMap map = Maps.newHashMap(); + final HashMap map = Maps.newHashMap(); for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { final String key = unquote(propAssignmentContext.Identifier().getText(), '`'); - final Expression value = exprs.get(propAssignmentContext.expression()); + final AbstractExpression value = exprs.get(propAssignmentContext.expression()); map.put(key, value); } - exprs.put(ctx, new MapLiteralExpression(map)); + exprs.put(ctx, new MapLiteralExpression(ctx.getStart(), map)); } @Override @@ -522,8 +521,8 @@ public void enterMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { isIdIsFieldAccess.pop(); // reset for error checks - final Expression fieldExpr = exprs.get(ctx.field); - final MessageRefExpression expr = new MessageRefExpression(fieldExpr); + final AbstractExpression fieldExpr = exprs.get(ctx.field); + final MessageRefExpression expr = new MessageRefExpression(ctx.getStart(), fieldExpr); log.trace("$MSG: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } @@ -536,14 +535,14 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { if (!isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { parseContext.addError(new UndeclaredVariable(ctx)); } - final Expression expr; + final AbstractExpression expr; String type; // if the identifier is also a declared variable name prefer the variable if (isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { - expr = new FieldRefExpression(identifierName); + expr = new FieldRefExpression(ctx.getStart(), identifierName); type = "FIELDREF"; } else { - expr = new VarRefExpression(identifierName); + expr = new VarRefExpression(ctx.getStart(), identifierName); type = "VARREF"; } log.trace("{}: ctx {} => {}", type, ctx, expr); @@ -559,10 +558,10 @@ public void exitFunc(RuleLangParser.FuncContext ctx) { @Override public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { - final Expression array = exprs.get(ctx.array); - final Expression index = exprs.get(ctx.index); + final AbstractExpression array = exprs.get(ctx.array); + final AbstractExpression index = exprs.get(ctx.index); - final IndexedAccessExpression expr = new IndexedAccessExpression(array, index); + final IndexedAccessExpression expr = new IndexedAccessExpression(ctx.getStart(), array, index); exprs.put(ctx, expr); log.trace("IDXACCESS: ctx {} => {}", ctx, expr); } @@ -577,11 +576,11 @@ public RuleTypeAnnotator(ParseContext parseContext) { @Override public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { - final Expression expr = parseContext.expressions().get(ctx); + final AbstractExpression expr = parseContext.expressions().get(ctx); if (expr instanceof VarRefExpression) { final VarRefExpression varRefExpression = (VarRefExpression) expr; final String name = varRefExpression.varName(); - final Expression expression = parseContext.getDefinedVar(name); + final AbstractExpression expression = parseContext.getDefinedVar(name); if (expression == null) { if (parseContext.isSilent()) { log.debug("Unable to retrieve expression for variable {}, this is a bug", name); @@ -646,7 +645,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final FunctionDescriptor descriptor = expr.getFunction().descriptor(); final FunctionArgs args = expr.getArgs(); for (ParameterDescriptor p : descriptor.params()) { - final Expression argExpr = args.expression(p.name()); + final AbstractExpression argExpr = args.expression(p.name()); if (argExpr != null && !p.type().isAssignableFrom(argExpr.getType())) { parseContext.addError(new IncompatibleArgumentType(ctx, expr, p, argExpr)); } @@ -663,7 +662,7 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void enterEveryRule(ParserRuleContext ctx) { - final Expression expression = parseContext.expressions().get(ctx); + final AbstractExpression expression = parseContext.expressions().get(ctx); if (expression != null && !parseContext.isInnerNode(ctx)) { sb.append(" ( "); sb.append(expression.getClass().getSimpleName()); @@ -675,7 +674,7 @@ public void enterEveryRule(ParserRuleContext ctx) { @Override public void exitEveryRule(ParserRuleContext ctx) { - final Expression expression = parseContext.expressions().get(ctx); + final AbstractExpression expression = parseContext.expressions().get(ctx); if (expression != null && !parseContext.isInnerNode(ctx)) { sb.append(" ) "); } @@ -719,30 +718,30 @@ public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { * Being used by tree walkers or visitors to perform AST construction, type checking and so on. */ private static class ParseContext { - private final ParseTreeProperty exprs = new ParseTreeProperty<>(); - private final ParseTreeProperty> args = new ParseTreeProperty<>(); + private final ParseTreeProperty exprs = new ParseTreeProperty<>(); + private final ParseTreeProperty> args = new ParseTreeProperty<>(); /** * Should the parser be more silent about its error logging, useful for interactive parsing in the UI. */ private final boolean silent; - private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); + private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); private Set errors = Sets.newHashSet(); // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information private Set innerNodes = new IdentityHashSet<>(); public List statements = Lists.newArrayList(); public List rules = Lists.newArrayList(); - private Map varDecls = Maps.newHashMap(); + private Map varDecls = Maps.newHashMap(); public List pipelines = Lists.newArrayList(); public ParseContext(boolean silent) { this.silent = silent; } - public ParseTreeProperty expressions() { + public ParseTreeProperty expressions() { return exprs; } - public ParseTreeProperty> arguments() { + public ParseTreeProperty> arguments() { return args; } @@ -784,15 +783,15 @@ public boolean isInnerNode(RuleContext node) { * @param expr expression * @return true if successful, false if previously declared */ - public boolean defineVar(String name, Expression expr) { + public boolean defineVar(String name, AbstractExpression expr) { return varDecls.put(name, expr) == null; } - public Expression getDefinedVar(String name) { + public AbstractExpression getDefinedVar(String name) { return varDecls.get(name); } - public ParseTreeProperty> argumentLists() { + public ParseTreeProperty> argumentLists() { return argsLists; } @@ -857,7 +856,7 @@ public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ct @Override public void exitInteger(RuleLangParser.IntegerContext ctx) { // TODO handle different radix and length - final LongExpression expr = new LongExpression(Long.parseLong(ctx.getText())); + final LongExpression expr = new LongExpression(ctx.getStart(), Long.parseLong(ctx.getText())); log.trace("INT: ctx {} => {}", ctx, expr); parseContext.exprs.put(ctx, expr); } @@ -865,7 +864,7 @@ public void exitInteger(RuleLangParser.IntegerContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { final String text = unquote(ctx.getText(), '\"'); - final StringExpression expr = new StringExpression(text); + final StringExpression expr = new StringExpression(ctx.getStart(), text); log.trace("STRING: ctx {} => {}", ctx, expr); parseContext.exprs.put(ctx, expr); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java index 757c7399d853..8de99dfc8ed8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java @@ -17,8 +17,8 @@ package org.graylog.plugins.pipelineprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; -import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; public class IncompatibleTypes extends ParseError { @@ -37,7 +37,7 @@ public String toString() { return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); } - private String exprString(Expression e) { + private String exprString(AbstractExpression e) { return "(" + e.toString() + ") : " + e.getType().getSimpleName(); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java index 274a8527745b..8366c9de5308 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java @@ -19,6 +19,8 @@ import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog2.plugin.Message; import org.joda.time.DateTime; @@ -30,6 +32,7 @@ public class IndexedAccessExpressionTest { + public static final Token START = new CommonToken(-1); private EvaluationContext context; @Before @@ -40,9 +43,9 @@ public void setup() { @Test public void accessArray() { int ary[] = new int[] {23}; - final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(ary), num(0)); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(ary), num(0)); - final Object evaluate = idxExpr.evaluate(context); + final Object evaluate = idxExpr.evaluateUnsafe(context); assertThat(evaluate).isOfAnyClassIn(Integer.class); assertThat(evaluate).isEqualTo(23); } @@ -50,9 +53,9 @@ public void accessArray() { @Test public void accessList() { final ImmutableList list = ImmutableList.of(23); - final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(list), num(0)); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(list), num(0)); - final Object evaluate = idxExpr.evaluate(context); + final Object evaluate = idxExpr.evaluateUnsafe(context); assertThat(evaluate).isOfAnyClassIn(Integer.class); assertThat(evaluate).isEqualTo(23); } @@ -71,9 +74,9 @@ protected Integer computeNext() { return 23; } }; - final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(iterable), num(0)); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(iterable), num(0)); - final Object evaluate = idxExpr.evaluate(context); + final Object evaluate = idxExpr.evaluateUnsafe(context); assertThat(evaluate).isOfAnyClassIn(Integer.class); assertThat(evaluate).isEqualTo(23); } @@ -81,27 +84,27 @@ protected Integer computeNext() { @Test public void accessMap() { final ImmutableMap map = ImmutableMap.of("string", 23); - final IndexedAccessExpression idxExpr = new IndexedAccessExpression(obj(map), string("string")); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(map), string("string")); - final Object evaluate = idxExpr.evaluate(context); + final Object evaluate = idxExpr.evaluateUnsafe(context); assertThat(evaluate).isEqualTo(23); } @Test public void invalidObject() { - final IndexedAccessExpression expression = new IndexedAccessExpression(obj(23), num(0)); + final IndexedAccessExpression expression = new IndexedAccessExpression(START, obj(23), num(0)); // this should throw an exception assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> expression.evaluate(context)); + .isThrownBy(() -> expression.evaluateUnsafe(context)); } - private static Expression num(long idx) { - return new LongExpression(idx); + private static AbstractExpression num(long idx) { + return new LongExpression(START, idx); } - private static Expression string(String string) { - return new StringExpression(string); + private static AbstractExpression string(String string) { + return new StringExpression(START, string); } private static ConstantObjectExpression obj(Object object) { @@ -112,12 +115,12 @@ private static class ConstantObjectExpression extends ConstantExpression { private final Object object; protected ConstantObjectExpression(Object object) { - super(object.getClass()); + super(START, object.getClass()); this.object = object; } @Override - public Object evaluate(EvaluationContext context) { + public Object evaluateUnsafe(EvaluationContext context) { return object; } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index c2213143f4fd..d8d7afb50cd0 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -144,6 +144,9 @@ public static void registerFunctions() { functions.put(IpAddressConversion.NAME, new IpAddressConversion()); functions.put(CidrMatch.NAME, new CidrMatch()); + functions.put(IsNull.NAME, new IsNull()); + functions.put(IsNotNull.NAME, new IsNotNull()); + functionRegistry = new FunctionRegistry(functions); } @@ -285,6 +288,19 @@ public void evalError() { assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'toip' at 5:28 an exception was thrown: 'null' is not an IP string literal."); } + @Test + public void evalErrorSuppressed() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final Message message = new Message("test", "test", Tools.nowUTC()); + message.addField("this_field_was_set", true); + final EvaluationContext context = contextForRuleEval(rule, message); + + assertThat(context).isNotNull(); + assertThat(context.hasEvaluationErrors()).isFalse(); + assertThat(actionsTriggered.get()).isTrue(); + } + @Test public void newlyCreatedMessage() { final Rule rule = parser.parseRule(ruleForTest(), false); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt new file mode 100644 index 000000000000..1d14675ce0f0 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt @@ -0,0 +1,6 @@ +rule "suppressing exceptions/nulls" +when + is_null(toip($message.does_not_exist)) && is_not_null($message.this_field_was_set) +then + trigger_test(); +end From 718d2f002392f6e4d62f712c3def79dba5e43967 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 9 Mar 2016 15:16:51 +0100 Subject: [PATCH 181/521] pull most of AbstractExpression up into Expression this allows using the interface instead and is closer to the intention renamed AbstractExpression into BaseExpression to make intent clearer (it only stores the start token now) --- .../ast/expressions/AbstractExpression.java | 61 ------------ .../ast/expressions/AndExpression.java | 4 +- .../expressions/ArrayLiteralExpression.java | 6 +- .../ast/expressions/BaseExpression.java | 34 +++++++ .../ast/expressions/BinaryExpression.java | 6 +- .../BooleanValuedFunctionWrapper.java | 6 +- .../ast/expressions/ComparisonExpression.java | 2 +- .../ast/expressions/ConstantExpression.java | 2 +- .../ast/expressions/EqualityExpression.java | 2 +- .../ast/expressions/Expression.java | 26 ++++- .../expressions/FieldAccessExpression.java | 8 +- .../ast/expressions/FieldRefExpression.java | 2 +- .../ast/expressions/FunctionExpression.java | 2 +- .../expressions/IndexedAccessExpression.java | 8 +- .../ast/expressions/MapLiteralExpression.java | 6 +- .../ast/expressions/MessageRefExpression.java | 6 +- .../ast/expressions/NotExpression.java | 2 +- .../ast/expressions/OrExpression.java | 4 +- .../ast/expressions/UnaryExpression.java | 8 +- .../ast/expressions/VarRefExpression.java | 2 +- .../ast/functions/AbstractFunction.java | 4 +- .../ast/functions/Function.java | 6 +- .../ast/functions/FunctionArgs.java | 12 +-- .../ast/functions/ParameterDescriptor.java | 4 +- .../ast/statements/FunctionStatement.java | 6 +- .../ast/statements/VarAssignStatement.java | 6 +- .../parser/PipelineRuleParser.java | 96 +++++++++---------- .../parser/errors/IncompatibleTypes.java | 4 +- .../IndexedAccessExpressionTest.java | 4 +- 29 files changed, 168 insertions(+), 171 deletions(-) delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java deleted file mode 100644 index 67ebd2973150..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AbstractExpression.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor.ast.expressions; - -import org.antlr.v4.runtime.Token; -import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.exceptions.FunctionEvaluationException; - -import javax.annotation.Nullable; - -import static org.graylog2.shared.utilities.ExceptionUtils.getRootCause; - -public abstract class AbstractExpression implements Expression { - - private final Token startToken; - - public AbstractExpression(Token startToken) { - this.startToken = startToken; - } - - public Token getStartToken() { - return startToken; - } - - @Override - @Nullable - public Object evaluate(EvaluationContext context) { - try { - return evaluateUnsafe(context); - } catch (FunctionEvaluationException fee) { - context.addEvaluationError(fee.getStartToken().getLine(), - fee.getStartToken().getCharPositionInLine(), - fee.getFunctionExpression().getFunction().descriptor(), - getRootCause(fee)); - } catch (Exception e) { - context.addEvaluationError(startToken.getLine(), startToken.getCharPositionInLine(), null, getRootCause(e)); - } - return null; - } - - @Nullable - /** - * This method is allow to throw exceptions. The outside world is supposed to call evaluate instead. - */ - public abstract Object evaluateUnsafe(EvaluationContext context); - -} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java index 06c10aa3edea..d5bac0f602d5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java @@ -20,8 +20,8 @@ import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class AndExpression extends BinaryExpression implements LogicalExpression { - public AndExpression(Token start, AbstractExpression left, - AbstractExpression right) { + public AndExpression(Token start, Expression left, + Expression right) { super(start, left, right); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java index 1cd721cfb0e4..215e7c621f8b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java @@ -23,10 +23,10 @@ import java.util.List; import java.util.stream.Collectors; -public class ArrayLiteralExpression extends AbstractExpression { - private final List elements; +public class ArrayLiteralExpression extends BaseExpression { + private final List elements; - public ArrayLiteralExpression(Token start, List elements) { + public ArrayLiteralExpression(Token start, List elements) { super(start); this.elements = elements; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java new file mode 100644 index 000000000000..5b77538d0507 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; + +public abstract class BaseExpression implements Expression { + + private final Token startToken; + + public BaseExpression(Token startToken) { + this.startToken = startToken; + } + + @Override + public Token getStartToken() { + return startToken; + } + +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java index 6e23f42cacae..616e8a241a04 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java @@ -20,9 +20,9 @@ public abstract class BinaryExpression extends UnaryExpression { - protected final AbstractExpression left; + protected final Expression left; - public BinaryExpression(Token start, AbstractExpression left, AbstractExpression right) { + public BinaryExpression(Token start, Expression left, Expression right) { super(start, right); this.left = left; } @@ -32,7 +32,7 @@ public boolean isConstant() { return left.isConstant() && right.isConstant(); } - public AbstractExpression left() { + public Expression left() { return left; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java index bde6626b13f8..e703f77f3dfe 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -19,10 +19,10 @@ import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -public class BooleanValuedFunctionWrapper extends AbstractExpression implements LogicalExpression { - private final AbstractExpression expr; +public class BooleanValuedFunctionWrapper extends BaseExpression implements LogicalExpression { + private final Expression expr; - public BooleanValuedFunctionWrapper(Token start, AbstractExpression expr) { + public BooleanValuedFunctionWrapper(Token start, Expression expr) { super(start); this.expr = expr; if (!expr.getType().equals(Boolean.class)) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java index df99340fbb2b..f0177f464f48 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java @@ -22,7 +22,7 @@ public class ComparisonExpression extends BinaryExpression implements LogicalExpression { private final String operator; - public ComparisonExpression(Token start, AbstractExpression left, AbstractExpression right, String operator) { + public ComparisonExpression(Token start, Expression left, Expression right, String operator) { super(start, left, right); this.operator = operator; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java index d3cdecc1cec3..4ffe9832a7bd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java @@ -18,7 +18,7 @@ import org.antlr.v4.runtime.Token; -public abstract class ConstantExpression extends AbstractExpression { +public abstract class ConstantExpression extends BaseExpression { private final Class type; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java index 45ab54d13b1c..b154b5fee2c6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java @@ -27,7 +27,7 @@ public class EqualityExpression extends BinaryExpression implements LogicalExpre private final boolean checkEquality; - public EqualityExpression(Token start, AbstractExpression left, AbstractExpression right, boolean checkEquality) { + public EqualityExpression(Token start, Expression left, Expression right, boolean checkEquality) { super(start, left, right); this.checkEquality = checkEquality; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java index 2a6b3f8a3507..7e759c1a2124 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java @@ -16,16 +16,40 @@ */ package org.graylog.plugins.pipelineprocessor.ast.expressions; +import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.FunctionEvaluationException; import javax.annotation.Nullable; +import static org.graylog2.shared.utilities.ExceptionUtils.getRootCause; + public interface Expression { boolean isConstant(); + Token getStartToken(); + @Nullable - Object evaluate(EvaluationContext context); + default Object evaluate(EvaluationContext context) { + try { + return evaluateUnsafe(context); + } catch (FunctionEvaluationException fee) { + context.addEvaluationError(fee.getStartToken().getLine(), + fee.getStartToken().getCharPositionInLine(), + fee.getFunctionExpression().getFunction().descriptor(), + getRootCause(fee)); + } catch (Exception e) { + context.addEvaluationError(getStartToken().getLine(), getStartToken().getCharPositionInLine(), null, getRootCause(e)); + } + return null; + } Class getType(); + + /** + * This method is allow to throw exceptions. The outside world is supposed to call evaluate instead. + */ + @Nullable + Object evaluateUnsafe(EvaluationContext context); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java index 87d0015e515e..46aee1ed4d7a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -24,13 +24,13 @@ import java.lang.reflect.InvocationTargetException; -public class FieldAccessExpression extends AbstractExpression { +public class FieldAccessExpression extends BaseExpression { private static final Logger log = LoggerFactory.getLogger(FieldAccessExpression.class); - private final AbstractExpression object; - private final AbstractExpression field; + private final Expression object; + private final Expression field; - public FieldAccessExpression(Token start, AbstractExpression object, AbstractExpression field) { + public FieldAccessExpression(Token start, Expression object, Expression field) { super(start); this.object = object; this.field = field; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java index bb7d8190fe6d..a3ac623cff56 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java @@ -19,7 +19,7 @@ import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -public class FieldRefExpression extends AbstractExpression { +public class FieldRefExpression extends BaseExpression { private final String variableName; public FieldRefExpression(Token start, String variableName) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java index 97bd78485c2d..6c5cc84ef769 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java @@ -25,7 +25,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; -public class FunctionExpression extends AbstractExpression { +public class FunctionExpression extends BaseExpression { private final FunctionArgs args; private final Function function; private final FunctionDescriptor descriptor; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java index fe3c639d8387..d051286246bf 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java @@ -25,11 +25,11 @@ import java.util.List; import java.util.Map; -public class IndexedAccessExpression extends AbstractExpression { - private final AbstractExpression indexableObject; - private final AbstractExpression index; +public class IndexedAccessExpression extends BaseExpression { + private final Expression indexableObject; + private final Expression index; - public IndexedAccessExpression(Token start, AbstractExpression indexableObject, AbstractExpression index) { + public IndexedAccessExpression(Token start, Expression indexableObject, Expression index) { super(start); this.indexableObject = indexableObject; this.index = index; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java index 69addda912a5..27f76efb7ec2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java @@ -25,10 +25,10 @@ import java.util.HashMap; import java.util.Map; -public class MapLiteralExpression extends AbstractExpression { - private final HashMap map; +public class MapLiteralExpression extends BaseExpression { + private final HashMap map; - public MapLiteralExpression(Token start, HashMap map) { + public MapLiteralExpression(Token start, HashMap map) { super(start); this.map = map; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java index 632e9f4f4617..9b8d2d60a1f1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java @@ -19,10 +19,10 @@ import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -public class MessageRefExpression extends AbstractExpression { - private final AbstractExpression fieldExpr; +public class MessageRefExpression extends BaseExpression { + private final Expression fieldExpr; - public MessageRefExpression(Token start, AbstractExpression fieldExpr) { + public MessageRefExpression(Token start, Expression fieldExpr) { super(start); this.fieldExpr = fieldExpr; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java index 154df4c5c9dd..18627c499b08 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java @@ -20,7 +20,7 @@ import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class NotExpression extends UnaryExpression implements LogicalExpression { - public NotExpression(Token start, AbstractExpression right) { + public NotExpression(Token start, Expression right) { super(start, right); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java index 19a9cc2f9f15..467161494a88 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java @@ -20,8 +20,8 @@ import org.graylog.plugins.pipelineprocessor.EvaluationContext; public class OrExpression extends BinaryExpression implements LogicalExpression { - public OrExpression(Token start, AbstractExpression left, - AbstractExpression right) { + public OrExpression(Token start, Expression left, + Expression right) { super(start, left, right); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java index c905586df84b..acf77f718398 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java @@ -18,11 +18,11 @@ import org.antlr.v4.runtime.Token; -public abstract class UnaryExpression extends AbstractExpression { +public abstract class UnaryExpression extends BaseExpression { - protected final AbstractExpression right; + protected final Expression right; - public UnaryExpression(Token start, AbstractExpression right) { + public UnaryExpression(Token start, Expression right) { super(start); this.right = right; } @@ -37,7 +37,7 @@ public Class getType() { return right.getType(); } - public AbstractExpression right() { + public Expression right() { return right; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java index 2af54eef482d..cbd0bae75729 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java @@ -21,7 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class VarRefExpression extends AbstractExpression { +public class VarRefExpression extends BaseExpression { private static final Logger log = LoggerFactory.getLogger(VarRefExpression.class); private final String identifier; private Class type = Object.class; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java index 303999bac764..2af0254f388a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java @@ -17,7 +17,7 @@ package org.graylog.plugins.pipelineprocessor.ast.functions; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; /** * Helper Function implementation which evaluates and memoizes all constant FunctionArgs. @@ -27,7 +27,7 @@ public abstract class AbstractFunction implements Function { @Override - public Object preComputeConstantArgument(FunctionArgs args, String name, AbstractExpression arg) { + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { return arg.evaluateUnsafe(EvaluationContext.emptyContext()); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index 99f55e70b38e..cdd46b30a52a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableList; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ public FunctionDescriptor descriptor() { }; default void preprocessArgs(FunctionArgs args) { - for (Map.Entry e : args.getConstantArgs().entrySet()) { + for (Map.Entry e : args.getConstantArgs().entrySet()) { final String name = e.getKey(); try { final Object value = preComputeConstantArgument(args, name, e.getValue()); @@ -72,7 +72,7 @@ default void preprocessArgs(FunctionArgs args) { * @param arg the expression tree for the argument * @return the precomputed value for the argument or null if the value should be dynamically calculated for each invocation */ - Object preComputeConstantArgument(FunctionArgs args, String name, AbstractExpression arg); + Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg); T evaluate(FunctionArgs args, EvaluationContext context); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index 30dadda18366..b157d7cca9f3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -17,7 +17,7 @@ package org.graylog.plugins.pipelineprocessor.ast.functions; import com.google.common.collect.Maps; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -31,25 +31,25 @@ public class FunctionArgs { @Nonnull - private final Map args; + private final Map args; private final Map constantValues = Maps.newHashMap(); private final Function function; private final FunctionDescriptor descriptor; - public FunctionArgs(Function func, Map args) { + public FunctionArgs(Function func, Map args) { function = func; descriptor = function.descriptor(); this.args = firstNonNull(args, Collections.emptyMap()); } @Nonnull - public Map getArgs() { + public Map getArgs() { return args; } @Nonnull - public Map getConstantArgs() { + public Map getConstantArgs() { return args.entrySet().stream() .filter(e -> e.getValue().isConstant()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -60,7 +60,7 @@ public boolean isPresent(String key) { } @Nullable - public AbstractExpression expression(String key) { + public Expression expression(String key) { return args.get(key); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index b232019d713f..f2b411aa7cbd 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -18,7 +18,7 @@ import com.google.auto.value.AutoValue; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import javax.annotation.Nullable; import java.util.Optional; @@ -93,7 +93,7 @@ public R required(FunctionArgs args, EvaluationContext context) { if (precomputedValue != null) { return transformedType().cast(precomputedValue); } - final AbstractExpression valueExpr = args.expression(name()); + final Expression valueExpr = args.expression(name()); if (valueExpr == null) { return null; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java index 57a220419ba3..e466dd68ff46 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java @@ -17,13 +17,13 @@ package org.graylog.plugins.pipelineprocessor.ast.statements; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; public class FunctionStatement implements Statement { - private final AbstractExpression functionExpression; + private final Expression functionExpression; - public FunctionStatement(AbstractExpression functionExpression) { + public FunctionStatement(Expression functionExpression) { this.functionExpression = functionExpression; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java index 3ce5339c4239..f88e4d57c98b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java @@ -17,13 +17,13 @@ package org.graylog.plugins.pipelineprocessor.ast.statements; import org.graylog.plugins.pipelineprocessor.EvaluationContext; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; public class VarAssignStatement implements Statement { private final String name; - private final AbstractExpression expr; + private final Expression expr; - public VarAssignStatement(String name, AbstractExpression expr) { + public VarAssignStatement(String name, Expression expr) { this.name = name; this.expr = expr; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 7a40647aaee5..ebacbfaa4d94 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -35,7 +35,6 @@ import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.ast.Stage; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; @@ -44,6 +43,7 @@ import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; @@ -210,9 +210,9 @@ public void syntaxError(Recognizer recognizer, private class RuleAstBuilder extends RuleLangBaseListener { private final ParseContext parseContext; - private final ParseTreeProperty> args; - private final ParseTreeProperty> argsList; - private final ParseTreeProperty exprs; + private final ParseTreeProperty> args; + private final ParseTreeProperty> argsList; + private final ParseTreeProperty exprs; private final Set definedVars = Sets.newHashSet(); @@ -231,7 +231,7 @@ public RuleAstBuilder(ParseContext parseContext) { public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { final Rule.Builder ruleBuilder = Rule.builder(); ruleBuilder.name(unquote(ctx.name == null ? "" : ctx.name.getText(), '"')); - final AbstractExpression expr = exprs.get(ctx.condition); + final Expression expr = exprs.get(ctx.condition); LogicalExpression condition; if (expr instanceof LogicalExpression) { @@ -251,7 +251,7 @@ public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { @Override public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { - final AbstractExpression expr = exprs.get(ctx.functionCall()); + final Expression expr = exprs.get(ctx.functionCall()); final FunctionStatement functionStatement = new FunctionStatement(expr); parseContext.statements.add(functionStatement); } @@ -259,7 +259,7 @@ public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { @Override public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { final String name = unquote(ctx.varName.getText(), '`'); - final AbstractExpression expr = exprs.get(ctx.expression()); + final Expression expr = exprs.get(ctx.expression()); parseContext.defineVar(name, expr); definedVars.add(name); parseContext.statements.add(new VarAssignStatement(name, expr)); @@ -268,8 +268,8 @@ public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { @Override public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final String name = ctx.funcName.getText(); - Map argsMap = this.args.get(ctx.arguments()); - final List positionalArgs = this.argsList.get(ctx.arguments()); + Map argsMap = this.args.get(ctx.arguments()); + final List positionalArgs = this.argsList.get(ctx.arguments()); final Function function = functionRegistry.resolve(name); if (function == null) { @@ -284,7 +284,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { parseContext.addError(new WrongNumberOfArgs(ctx, function, argsMap.size())); } else { // there are optional parameters, check that all required ones are present - final Map givenArguments = argsMap; + final Map givenArguments = argsMap; final List missingParams = params.stream() .filter(p -> !p.optional()) @@ -335,7 +335,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { // the remaining parameters were optional, so we can safely skip them break; } - final AbstractExpression argExpr = positionalArgs.get(i); + final Expression argExpr = positionalArgs.get(i); argsMap.put(p.name(), argExpr); i++; } @@ -356,10 +356,10 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { @Override public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { - final Map argMap = Maps.newHashMap(); + final Map argMap = Maps.newHashMap(); for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { final String argName = unquote(propAssignmentContext.Identifier().getText(), '`'); - final AbstractExpression argValue = exprs.get(propAssignmentContext.expression()); + final Expression argValue = exprs.get(propAssignmentContext.expression()); argMap.put(argName, argValue); } args.put(ctx, argMap); @@ -367,7 +367,7 @@ public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { @Override public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { - List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); + List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); expressions.addAll(ctx.expression().stream().map(exprs::get).collect(toList())); argsList.put(ctx, expressions); } @@ -381,8 +381,8 @@ public void enterNested(RuleLangParser.NestedContext ctx) { @Override public void exitNested(RuleLangParser.NestedContext ctx) { isIdIsFieldAccess.pop(); // reset for error checks - final AbstractExpression object = exprs.get(ctx.fieldSet); - final AbstractExpression field = exprs.get(ctx.field); + final Expression object = exprs.get(ctx.fieldSet); + final Expression field = exprs.get(ctx.field); final FieldAccessExpression expr = new FieldAccessExpression(ctx.getStart(), object, field); log.trace("FIELDACCESS: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -390,7 +390,7 @@ public void exitNested(RuleLangParser.NestedContext ctx) { @Override public void exitNot(RuleLangParser.NotContext ctx) { - final AbstractExpression expression = upgradeBoolFunctionExpression(ctx.expression()); + final Expression expression = upgradeBoolFunctionExpression(ctx.expression()); final NotExpression expr = new NotExpression(ctx.getStart(), expression); log.trace("NOT: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -400,16 +400,16 @@ public void exitNot(RuleLangParser.NotContext ctx) { public void exitAnd(RuleLangParser.AndContext ctx) { // if the expressions are function calls but boolean valued, upgrade them, // we allow testing boolean valued functions without explicit comparison operator - final AbstractExpression left = upgradeBoolFunctionExpression(ctx.left); - final AbstractExpression right = upgradeBoolFunctionExpression(ctx.right); + final Expression left = upgradeBoolFunctionExpression(ctx.left); + final Expression right = upgradeBoolFunctionExpression(ctx.right); final AndExpression expr = new AndExpression(ctx.getStart(), left, right); log.trace("AND: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); } - private AbstractExpression upgradeBoolFunctionExpression(RuleLangParser.ExpressionContext leftExprContext) { - AbstractExpression leftExpr = exprs.get(leftExprContext); + private Expression upgradeBoolFunctionExpression(RuleLangParser.ExpressionContext leftExprContext) { + Expression leftExpr = exprs.get(leftExprContext); if (leftExpr instanceof FunctionExpression && leftExpr.getType().equals(Boolean.class)) { leftExpr = new BooleanValuedFunctionWrapper(leftExprContext.getStart(), leftExpr); } @@ -418,8 +418,8 @@ private AbstractExpression upgradeBoolFunctionExpression(RuleLangParser.Expressi @Override public void exitOr(RuleLangParser.OrContext ctx) { - final AbstractExpression left = upgradeBoolFunctionExpression(ctx.left); - final AbstractExpression right = upgradeBoolFunctionExpression(ctx.right); + final Expression left = upgradeBoolFunctionExpression(ctx.left); + final Expression right = upgradeBoolFunctionExpression(ctx.right); final OrExpression expr = new OrExpression(ctx.getStart(), left, right); log.trace("OR: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -427,8 +427,8 @@ public void exitOr(RuleLangParser.OrContext ctx) { @Override public void exitEquality(RuleLangParser.EqualityContext ctx) { - final AbstractExpression left = exprs.get(ctx.left); - final AbstractExpression right = exprs.get(ctx.right); + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); final boolean equals = ctx.equality.getText().equals("=="); final EqualityExpression expr = new EqualityExpression(ctx.getStart(), left, right, equals); log.trace("EQUAL: ctx {} => {}", ctx, expr); @@ -437,8 +437,8 @@ public void exitEquality(RuleLangParser.EqualityContext ctx) { @Override public void exitComparison(RuleLangParser.ComparisonContext ctx) { - final AbstractExpression left = exprs.get(ctx.left); - final AbstractExpression right = exprs.get(ctx.right); + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); final String operator = ctx.comparison.getText(); final ComparisonExpression expr = new ComparisonExpression(ctx.getStart(), left, right, operator); log.trace("COMPARE: ctx {} => {}", ctx, expr); @@ -490,16 +490,16 @@ public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { @Override public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { - final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); + final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); exprs.put(ctx, new ArrayLiteralExpression(ctx.getStart(), elements)); } @Override public void exitMapLiteralExpr(RuleLangParser.MapLiteralExprContext ctx) { - final HashMap map = Maps.newHashMap(); + final HashMap map = Maps.newHashMap(); for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { final String key = unquote(propAssignmentContext.Identifier().getText(), '`'); - final AbstractExpression value = exprs.get(propAssignmentContext.expression()); + final Expression value = exprs.get(propAssignmentContext.expression()); map.put(key, value); } exprs.put(ctx, new MapLiteralExpression(ctx.getStart(), map)); @@ -521,7 +521,7 @@ public void enterMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { isIdIsFieldAccess.pop(); // reset for error checks - final AbstractExpression fieldExpr = exprs.get(ctx.field); + final Expression fieldExpr = exprs.get(ctx.field); final MessageRefExpression expr = new MessageRefExpression(ctx.getStart(), fieldExpr); log.trace("$MSG: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -535,7 +535,7 @@ public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { if (!isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { parseContext.addError(new UndeclaredVariable(ctx)); } - final AbstractExpression expr; + final Expression expr; String type; // if the identifier is also a declared variable name prefer the variable if (isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { @@ -558,8 +558,8 @@ public void exitFunc(RuleLangParser.FuncContext ctx) { @Override public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { - final AbstractExpression array = exprs.get(ctx.array); - final AbstractExpression index = exprs.get(ctx.index); + final Expression array = exprs.get(ctx.array); + final Expression index = exprs.get(ctx.index); final IndexedAccessExpression expr = new IndexedAccessExpression(ctx.getStart(), array, index); exprs.put(ctx, expr); @@ -576,11 +576,11 @@ public RuleTypeAnnotator(ParseContext parseContext) { @Override public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { - final AbstractExpression expr = parseContext.expressions().get(ctx); + final Expression expr = parseContext.expressions().get(ctx); if (expr instanceof VarRefExpression) { final VarRefExpression varRefExpression = (VarRefExpression) expr; final String name = varRefExpression.varName(); - final AbstractExpression expression = parseContext.getDefinedVar(name); + final Expression expression = parseContext.getDefinedVar(name); if (expression == null) { if (parseContext.isSilent()) { log.debug("Unable to retrieve expression for variable {}, this is a bug", name); @@ -645,7 +645,7 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { final FunctionDescriptor descriptor = expr.getFunction().descriptor(); final FunctionArgs args = expr.getArgs(); for (ParameterDescriptor p : descriptor.params()) { - final AbstractExpression argExpr = args.expression(p.name()); + final Expression argExpr = args.expression(p.name()); if (argExpr != null && !p.type().isAssignableFrom(argExpr.getType())) { parseContext.addError(new IncompatibleArgumentType(ctx, expr, p, argExpr)); } @@ -662,7 +662,7 @@ public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { @Override public void enterEveryRule(ParserRuleContext ctx) { - final AbstractExpression expression = parseContext.expressions().get(ctx); + final Expression expression = parseContext.expressions().get(ctx); if (expression != null && !parseContext.isInnerNode(ctx)) { sb.append(" ( "); sb.append(expression.getClass().getSimpleName()); @@ -674,7 +674,7 @@ public void enterEveryRule(ParserRuleContext ctx) { @Override public void exitEveryRule(ParserRuleContext ctx) { - final AbstractExpression expression = parseContext.expressions().get(ctx); + final Expression expression = parseContext.expressions().get(ctx); if (expression != null && !parseContext.isInnerNode(ctx)) { sb.append(" ) "); } @@ -718,30 +718,30 @@ public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { * Being used by tree walkers or visitors to perform AST construction, type checking and so on. */ private static class ParseContext { - private final ParseTreeProperty exprs = new ParseTreeProperty<>(); - private final ParseTreeProperty> args = new ParseTreeProperty<>(); + private final ParseTreeProperty exprs = new ParseTreeProperty<>(); + private final ParseTreeProperty> args = new ParseTreeProperty<>(); /** * Should the parser be more silent about its error logging, useful for interactive parsing in the UI. */ private final boolean silent; - private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); + private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); private Set errors = Sets.newHashSet(); // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information private Set innerNodes = new IdentityHashSet<>(); public List statements = Lists.newArrayList(); public List rules = Lists.newArrayList(); - private Map varDecls = Maps.newHashMap(); + private Map varDecls = Maps.newHashMap(); public List pipelines = Lists.newArrayList(); public ParseContext(boolean silent) { this.silent = silent; } - public ParseTreeProperty expressions() { + public ParseTreeProperty expressions() { return exprs; } - public ParseTreeProperty> arguments() { + public ParseTreeProperty> arguments() { return args; } @@ -783,15 +783,15 @@ public boolean isInnerNode(RuleContext node) { * @param expr expression * @return true if successful, false if previously declared */ - public boolean defineVar(String name, AbstractExpression expr) { + public boolean defineVar(String name, Expression expr) { return varDecls.put(name, expr) == null; } - public AbstractExpression getDefinedVar(String name) { + public Expression getDefinedVar(String name) { return varDecls.get(name); } - public ParseTreeProperty> argumentLists() { + public ParseTreeProperty> argumentLists() { return argsLists; } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java index 8de99dfc8ed8..757c7399d853 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java @@ -17,8 +17,8 @@ package org.graylog.plugins.pipelineprocessor.parser.errors; import com.fasterxml.jackson.annotation.JsonProperty; -import org.graylog.plugins.pipelineprocessor.ast.expressions.AbstractExpression; import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; public class IncompatibleTypes extends ParseError { @@ -37,7 +37,7 @@ public String toString() { return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); } - private String exprString(AbstractExpression e) { + private String exprString(Expression e) { return "(" + e.toString() + ") : " + e.getType().getSimpleName(); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java index 8366c9de5308..3c08705190f2 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java @@ -99,11 +99,11 @@ public void invalidObject() { .isThrownBy(() -> expression.evaluateUnsafe(context)); } - private static AbstractExpression num(long idx) { + private static Expression num(long idx) { return new LongExpression(START, idx); } - private static AbstractExpression string(String string) { + private static Expression string(String string) { return new StringExpression(START, string); } From 76a8b5c8eca27d1bee8921a6fe9cab8c60963d17 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 11 Mar 2016 22:31:25 +0100 Subject: [PATCH 182/521] Format RuleForm --- src/web/rules/RuleForm.jsx | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index 90f9bf063316..4def03046122 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -1,6 +1,5 @@ import React, { PropTypes } from 'react'; -import { Row, Col } from 'react-bootstrap'; import { Input } from 'react-bootstrap'; import AceEditor from 'react-ace'; @@ -12,7 +11,6 @@ import 'brace/theme/chrome'; import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; const RuleForm = React.createClass({ - parseTimer: undefined, propTypes: { rule: PropTypes.object, create: React.PropTypes.bool, @@ -28,7 +26,7 @@ const RuleForm = React.createClass({ description: '', source: '', }, - } + }; }, getInitialState() { @@ -39,11 +37,11 @@ const RuleForm = React.createClass({ id: rule.id, title: rule.title, description: rule.description, - source: rule.source + source: rule.source, }, editor: undefined, parseErrors: [], - } + }; }, componentWillUnmount() { @@ -53,6 +51,8 @@ const RuleForm = React.createClass({ } }, + parseTimer: undefined, + openModal() { this.refs.modal.open(); }, @@ -60,13 +60,13 @@ const RuleForm = React.createClass({ _updateEditor() { const session = this.state.editor.session; const annotations = this.state.parseErrors.map(e => { - return {row: e.line - 1, column: e.position_in_line - 1, text: e.reason, type: "error"} + return { row: e.line - 1, column: e.position_in_line - 1, text: e.reason, type: 'error' }; }); session.setAnnotations(annotations); }, _setParseErrors(errors) { - this.setState({parseErrors: errors}, this._updateEditor); + this.setState({ parseErrors: errors }, this._updateEditor); }, _onSourceChange(value) { @@ -76,7 +76,7 @@ const RuleForm = React.createClass({ } const rule = this.state.rule; rule.source = value; - this.setState({rule: rule}); + this.setState({ rule }); if (this.props.validateRule) { // have the caller validate the rule after typing stopped for a while. usually this will mean send to server to parse @@ -87,17 +87,17 @@ const RuleForm = React.createClass({ _onDescriptionChange(event) { const rule = this.state.rule; rule.description = event.target.value; - this.setState({rule: rule}); + this.setState({ rule }); }, _onTitleChange(event) { const rule = this.state.rule; rule.title = event.target.value; - this.setState({rule: rule}); + this.setState({ rule }); }, _onLoad(editor) { - this.setState({editor: editor}); + this.setState({ editor }); }, _getId(prefixIdName) { @@ -111,7 +111,7 @@ const RuleForm = React.createClass({ _saved() { this._closeModal(); if (this.props.create) { - this.setState({rule: {}}); + this.setState({ rule: {} }); } }, @@ -144,18 +144,18 @@ const RuleForm = React.createClass({ label="Description (optional)" onChange={this._onDescriptionChange} autoFocus - value={this.state.rule.description}/> + value={this.state.rule.description} /> -
    +
    Date: Sat, 12 Mar 2016 12:18:06 +0100 Subject: [PATCH 183/521] Use a single page to create and edit rules That gives us the opportunity to show some help, examples, and also view rules directly from the pipeline details page. --- src/web/index.jsx | 12 ++- src/web/pipelines/Stage.jsx | 7 +- src/web/rules/Rule.jsx | 69 +++++++++++++ src/web/rules/RuleDetailsPage.jsx | 58 +++++++++++ src/web/rules/RuleForm.jsx | 82 +++++++--------- src/web/rules/RuleHelper.jsx | 157 ++++++++++++++++++++++++++++++ src/web/rules/RuleList.jsx | 47 +++++---- src/web/rules/RulesActions.jsx | 13 +-- src/web/rules/RulesStore.js | 71 ++++++++++---- 9 files changed, 414 insertions(+), 102 deletions(-) create mode 100644 src/web/rules/Rule.jsx create mode 100644 src/web/rules/RuleDetailsPage.jsx create mode 100644 src/web/rules/RuleHelper.jsx diff --git a/src/web/index.jsx b/src/web/index.jsx index a53b456f9bc2..2c8de8b6fcc3 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -4,16 +4,18 @@ import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage'; import PipelineDetailsPage from 'pipelines/PipelineDetailsPage'; import PipelineConnectionsPage from 'pipeline-connections/PipelineConnectionsPage'; import RulesPage from 'rules/RulesPage'; +import RuleDetailsPage from 'rules/RuleDetailsPage'; PluginStore.register(new PluginManifest(packageJson, { routes: [ - {path: '/system/pipelines', component: PipelineConnectionsPage}, - {path: '/system/pipelines/overview', component: PipelinesOverviewPage}, - {path: '/system/pipelines/rules', component: RulesPage}, - {path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage}, + { path: '/system/pipelines', component: PipelineConnectionsPage }, + { path: '/system/pipelines/overview', component: PipelinesOverviewPage }, + { path: '/system/pipelines/rules', component: RulesPage }, + { path: '/system/pipelines/rules/:ruleId', component: RuleDetailsPage }, + { path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage }, ], systemnavigation: [ - {path: '/system/pipelines', description: 'Pipelines'}, + { path: '/system/pipelines', description: 'Pipelines' }, ], })); diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 65c7137036d5..449789324273 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -1,6 +1,7 @@ import React, { PropTypes } from 'react'; import Reflux from 'reflux'; import { Col, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; import { DataTable, EntityListItem, Spinner } from 'components/common'; import RulesStore from 'rules/RulesStore'; @@ -22,7 +23,11 @@ const Stage = React.createClass({ _ruleRowFormatter(rule) { return ( - {rule.title} + + + {rule.title} + + {rule.description} ); diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx new file mode 100644 index 000000000000..13f83a1dca78 --- /dev/null +++ b/src/web/rules/Rule.jsx @@ -0,0 +1,69 @@ +import React from 'react'; +import { Row, Col, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { PageHeader } from 'components/common'; +import DocumentationLink from 'components/support/DocumentationLink'; + +import DocsHelper from 'util/DocsHelper'; + +import RuleForm from './RuleForm'; +import RuleHelper from './RuleHelper'; + +const Rule = React.createClass({ + propTypes: { + rule: React.PropTypes.object, + create: React.PropTypes.bool, + onSave: React.PropTypes.func.isRequired, + validateRule: React.PropTypes.func.isRequired, + history: React.PropTypes.object.isRequired, + }, + + render() { + let title; + if (this.props.create) { + title = 'Create pipeline rule'; + } else { + title = Pipeline rule {this.props.rule.title}; + } + + return ( +
    + + + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list{' '} + of actions.{' '} + Graylog evaluates the condition against a message and executes the actions if the condition is satisfied. + + + + Read more about Graylog pipeline rules in the . + + + + + + +   + + + + + + + + + + + + + + +
    + ); + }, +}); + +export default Rule; diff --git a/src/web/rules/RuleDetailsPage.jsx b/src/web/rules/RuleDetailsPage.jsx new file mode 100644 index 000000000000..9fe47f1b7b32 --- /dev/null +++ b/src/web/rules/RuleDetailsPage.jsx @@ -0,0 +1,58 @@ +import React from 'react'; +import Reflux from 'reflux'; + +import { Spinner } from 'components/common'; + +import Rule from './Rule'; +import RulesStore from './RulesStore'; +import RulesActions from './RulesActions'; + +function filterRules(state) { + return state.rules ? state.rules.filter(r => r.id === this.props.params.ruleId)[0] : undefined; +} + +const RuleDetailsPage = React.createClass({ + propTypes: { + params: React.PropTypes.object.isRequired, + history: React.PropTypes.object.isRequired, + }, + + mixins: [Reflux.connectFilter(RulesStore, 'rule', filterRules)], + + componentDidMount() { + if (this.props.params.ruleId !== 'new') { + RulesActions.get(this.props.params.ruleId); + } + }, + + _save(rule, callback) { + let promise; + if (rule.id) { + promise = RulesActions.update.triggerPromise(rule); + } else { + promise = RulesActions.save.triggerPromise(rule); + } + promise.then(() => callback()); + }, + + _validateRule(rule, setErrorsCb) { + RulesActions.parse(rule, setErrorsCb); + }, + + _isLoading() { + return this.props.params.ruleId !== 'new' && !this.state.rule; + }, + + render() { + if (this._isLoading()) { + return ; + } + + return ( + + ); + }, +}); + +export default RuleDetailsPage; \ No newline at end of file diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index 4def03046122..d5610e50b626 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -1,6 +1,5 @@ import React, { PropTypes } from 'react'; - -import { Input } from 'react-bootstrap'; +import { Row, Col, Button, Input } from 'react-bootstrap'; import AceEditor from 'react-ace'; import brace from 'brace'; @@ -8,14 +7,13 @@ import brace from 'brace'; import 'brace/mode/text'; import 'brace/theme/chrome'; -import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; - const RuleForm = React.createClass({ propTypes: { rule: PropTypes.object, create: React.PropTypes.bool, - save: React.PropTypes.func.isRequired, + onSave: React.PropTypes.func.isRequired, validateRule: React.PropTypes.func.isRequired, + history: React.PropTypes.object.isRequired, }, getDefaultProps() { @@ -53,10 +51,6 @@ const RuleForm = React.createClass({ parseTimer: undefined, - openModal() { - this.refs.modal.open(); - }, - _updateEditor() { const session = this.state.editor.session; const annotations = this.state.parseErrors.map(e => { @@ -104,50 +98,39 @@ const RuleForm = React.createClass({ return this.state.name !== undefined ? prefixIdName + this.state.name : prefixIdName; }, - _closeModal() { - this.refs.modal.close(); + _goBack() { + this.props.history.goBack(); }, _saved() { - this._closeModal(); - if (this.props.create) { - this.setState({ rule: {} }); - } + this.props.history.pushState(null, '/system/pipelines/rules'); }, _save() { if (this.state.parseErrors.length === 0) { - this.props.save(this.state.rule, this._saved); + this.props.onSave(this.state.rule, this._saved); } }, + _submit(event) { + event.preventDefault(); + this._save(); + }, + render() { - let triggerButtonContent; - if (this.props.create) { - triggerButtonContent = 'Add new rule'; - } else { - triggerButtonContent = Edit; - } return ( - - - -
    - - - -
    +
    +
    + + + +
    -
    - - + +
    + + + +
    + + +
    + +     + ); }, }); -export default RuleForm; \ No newline at end of file +export default RuleForm; diff --git a/src/web/rules/RuleHelper.jsx b/src/web/rules/RuleHelper.jsx new file mode 100644 index 000000000000..e6b9ddfec311 --- /dev/null +++ b/src/web/rules/RuleHelper.jsx @@ -0,0 +1,157 @@ +import React from 'react'; +import { Row, Col, Panel, Tabs, Tab } from 'react-bootstrap'; + +import DocumentationLink from 'components/support/DocumentationLink'; + +import DocsHelper from 'util/DocsHelper'; + +const RuleHelper = React.createClass({ + ruleTemplate: `rule "function howto" +when + has_field("transaction_date") +then + // the following date format assumes there's no time zone in the string + let new_date = parse_date(tostring($message.transaction_date), "yyyy-MM-dd HH:mm:ss"); + set_field("transaction_year", new_date.year); +end`, + + render() { + return ( + + + +

    + Read the {' '} + to gain a better understanding of how Graylog pipeline rules work. +

    + +     + + + + + 
    +                  {this.ruleTemplate}
+
    +     + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FunctionDescription
    tobool(any)Converts the single parameter to a boolean value using its string value.
    todouble(any, [default: double])Converts the first parameter to a double floating point value.
    tolong(any, [default: long])Converts the first parameter to a long integer value.
    tostring(any, [default: string])Converts the first parameter to its string representation.
    capitalize(value: string)Capitalizes a String changing the first letter to title case.
    uppercase(value: string, [locale: string])Converts a String to upper case.
    lowercase(value: string, [locale: string])Converts a String to lower case.
    contains(value: string, search: string, [ignore_case: boolean])Checks if a string contains another string.
    substring(value: string, start: long, [end: long])Returns a substring of value with the given start and end offsets. +
    regex(pattern: string, value: string, [group_names: array[string])Match a regular expression against a string, with matcher groups.
    parse_date(value: string, pattern: string, [timezone: string])Parses a date and time from the given string, according to a strict pattern.
    flex_parse_date(value: string, [default: DateTime], [timezone: string])Attempts to parse a date and time using the Natty date parser.
    format_date(value: DateTime, format: string, [timezone: string])Formats a date and time according to a given formatter pattern.
    parse_json(value: string)Parse a string into a JSON tree.
    toip(ip: string)Converts the given string to an IP object.
    cidr_match(cidr: string, ip: IpAddress)Checks whether the given IP matches a CIDR pattern.
    from_input(id: string | name: string)Checks whether the current message was received by the given input.
    route_to_stream(id: string | name: string, [message: Message])Assigns the current message to the specified stream.
    drop_message(message: Message)This currently processed message will be removed from the processing pipeline after the rule + finishes. +
    has_field(field: string, [message: Message])Checks whether the currently processed message contains the named field.
    remove_field(field: string, [message: Message])Removes the named field from the currently processed message.
    set_field(field: string, value: any, [message: Message])Sets the name field to the given value in the currently processed message.
    set_fields(fields: Map<string, any>, [message: Message])Sets multiple fields to the given values in the currently processed message.
    +
    +

    See all functions in the .

    +     +     + +     +     + ); + }, +}); + +export default RuleHelper; diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index e7ee0e0c8131..8e49c857bd3d 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -2,8 +2,8 @@ import React, { PropTypes } from 'react'; import { DataTable, Timestamp } from 'components/common'; import { Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; -import RuleForm from './RuleForm'; import RulesActions from './RulesActions'; const RuleList = React.createClass({ @@ -11,38 +11,36 @@ const RuleList = React.createClass({ rules: PropTypes.array.isRequired, }, - _save(rule, callback) { - if (rule.id) { - RulesActions.update(rule); - } else { - RulesActions.save(rule); - } - callback(); - }, - _delete(rule) { - if (window.confirm(`Do you really want to delete rule "${rule.title}"?`)) { - RulesActions.delete(rule.id); - } - }, - - _validateRule(rule, setErrorsCb) { - RulesActions.parse(rule, setErrorsCb); + return () => { + if (window.confirm(`Do you really want to delete rule "${rule.title}"?`)) { + RulesActions.delete(rule.id); + } + }; }, _headerCellFormatter(header) { return {header}; }, + _ruleInfoFormatter(rule) { const actions = [ - , + ,  , - , + + + , ]; return ( - {rule.title} + + + {rule.title} + + {rule.description} @@ -67,15 +65,14 @@ const RuleList = React.createClass({ filterLabel="Filter Rules" filterKeys={filterKeys}>
    - + + +
    ); - } + }, }); export default RuleList; diff --git a/src/web/rules/RulesActions.jsx b/src/web/rules/RulesActions.jsx index 29935a45c1fa..70f5bc48f230 100644 --- a/src/web/rules/RulesActions.jsx +++ b/src/web/rules/RulesActions.jsx @@ -1,12 +1,13 @@ import Reflux from 'reflux'; const RulesActions = Reflux.createActions({ - 'delete': {asyncResult: true}, - 'list': {asyncResult: true}, - 'save': {asyncResult: true}, - 'update': {asyncResult: true}, - 'parse': {asyncResult: true}, - 'multiple': {asyncResult: true}, + delete: { asyncResult: true }, + list: { asyncResult: true }, + get: { asyncResult: true }, + save: { asyncResult: true }, + update: { asyncResult: true }, + parse: { asyncResult: true }, + multiple: { asyncResult: true }, }); export default RulesActions; \ No newline at end of file diff --git a/src/web/rules/RulesStore.js b/src/web/rules/RulesStore.js index 90d87ff8bf99..ac407ea707d4 100644 --- a/src/web/rules/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -13,7 +13,21 @@ const RulesStore = Reflux.createStore({ rules: undefined, getInitialState() { - return {rules: this.rules}; + return { rules: this.rules }; + }, + + _updateRulesState(rule) { + if (!this.rules) { + this.rules = [rule]; + } else { + const doesRuleExist = this.rules.some(r => r.id === rule.id); + if (doesRuleExist) { + this.rules = this.rules.map(r => r.id === rule.id ? rule : r); + } else { + this.rules.push(rule); + } + } + this.trigger({ rules: this.rules }); }, list() { @@ -25,47 +39,64 @@ const RulesStore = Reflux.createStore({ const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule'); return fetch('GET', url).then((response) => { this.rules = response; - this.trigger({rules: response}); + this.trigger({ rules: response }); }, failCallback); }, + get(ruleId) { + const failCallback = (error) => { + UserNotification.error(`Fetching rule "${ruleId}" failed with status: ${error.message}`, + `Could not retrieve processing rule "${ruleId}"`); + }; + + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/rule/${ruleId}`); + const promise = fetch('GET', url); + promise.then(this._updateRulesState, failCallback); + + return promise; + }, + save(ruleSource) { const failCallback = (error) => { - UserNotification.error('Saving rule failed with status: ' + error.message, - 'Could not save processing rule'); + UserNotification.error(`Saving rule "${ruleSource.title}" failed with status: ${error.message}`, + `Could not save processing rule "${ruleSource.title}"`); }; - const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule'); + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/rule`); const rule = { title: ruleSource.title, description: ruleSource.description, source: ruleSource.source, }; - return fetch('POST', url, rule).then((response) => { - if (this.rules === undefined) { - this.rules = [response]; - } else { - this.rules.push(response); - } - this.trigger({rules: this.rules}); + const promise = fetch('POST', url, rule); + promise.then((response) => { + this._updateRulesState(response); + UserNotification.success(`Rule "${response.title}" created successfully`); }, failCallback); + + RulesActions.save.promise(promise); + return promise; }, update(ruleSource) { const failCallback = (error) => { - UserNotification.error('Updating rule failed with status: ' + error.message, - 'Could not update processing rule'); + UserNotification.error(`Updating rule "${ruleSource.title}" failed with status: ${error.message}`, + `Could not update processing rule "${ruleSource.title}"`); }; - const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/' + ruleSource.id); + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/rule/${ruleSource.id}`); const rule = { id: ruleSource.id, title: ruleSource.title, description: ruleSource.description, source: ruleSource.source, }; - return fetch('PUT', url, rule).then((response) => { - this.rules = this.rules.map((e) => e.id === response.id ? response : e); - this.trigger({rules: this.rules}); + const promise = fetch('PUT', url, rule); + promise.then((response) => { + this._updateRulesState(response); + UserNotification.success(`Rule "${response.title}" updated successfully`); }, failCallback); + + RulesActions.update.promise(promise); + return promise; }, delete(ruleId) { const failCallback = (error) => { @@ -75,7 +106,7 @@ const RulesStore = Reflux.createStore({ const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/' + ruleId); return fetch('DELETE', url).then(() => { this.rules = this.rules.filter((el) => el.id !== ruleId); - this.trigger({rules: this.rules}); + this.trigger({ rules: this.rules }); }, failCallback); }, parse(ruleSource, callback) { @@ -101,7 +132,7 @@ const RulesStore = Reflux.createStore({ }, multiple(ruleNames, callback) { const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/multiple'); - const promise = fetch('POST', url, {rules: ruleNames}); + const promise = fetch('POST', url, { rules: ruleNames }); promise.then(callback); return promise; From 94c34665a1d7621592303608a5610c6487ba3422 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 14 Mar 2016 10:53:24 +0100 Subject: [PATCH 184/521] Remove link from rule title It was doing the same as the edit button, which is somehow confusing. --- src/web/rules/RuleList.jsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index 8e49c857bd3d..194d8c07f57b 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -36,11 +36,7 @@ const RuleList = React.createClass({ return ( - - - {rule.title} - - + {rule.title} {rule.description} From d835deb1f62f6c27663979c9cb5d7f65d08328e3 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 14 Mar 2016 10:54:04 +0100 Subject: [PATCH 185/521] Add hint about the rule title --- src/web/rules/RuleForm.jsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index d5610e50b626..88ea4122a35f 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -121,6 +121,10 @@ const RuleForm = React.createClass({ return (
    + + - +
    Date: Mon, 14 Mar 2016 11:22:15 +0100 Subject: [PATCH 186/521] Revert "Remove link from rule title" This reverts commit 94c34665a1d7621592303608a5610c6487ba3422. Most elements in the pipeline UI are clickable in the name, so this is actually adding some consistency. --- src/web/rules/RuleList.jsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index 194d8c07f57b..8e49c857bd3d 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -36,7 +36,11 @@ const RuleList = React.createClass({ return ( - {rule.title} + + + {rule.title} + + {rule.description} From a770fe13bbdd173ec4ccd91b5f4951524e8a5d4f Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 14 Mar 2016 11:25:21 +0100 Subject: [PATCH 187/521] Show notification after deleting rule Also fix some copy-and-waste --- src/web/rules/RuleList.jsx | 2 +- src/web/rules/RulesStore.js | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index 8e49c857bd3d..cac5e12fb181 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -14,7 +14,7 @@ const RuleList = React.createClass({ _delete(rule) { return () => { if (window.confirm(`Do you really want to delete rule "${rule.title}"?`)) { - RulesActions.delete(rule.id); + RulesActions.delete(rule); } }; }, diff --git a/src/web/rules/RulesStore.js b/src/web/rules/RulesStore.js index ac407ea707d4..fb2217a6c0cb 100644 --- a/src/web/rules/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -98,15 +98,16 @@ const RulesStore = Reflux.createStore({ RulesActions.update.promise(promise); return promise; }, - delete(ruleId) { + delete(rule) { const failCallback = (error) => { - UserNotification.error('Updating rule failed with status: ' + error.message, - 'Could not update processing rule'); + UserNotification.error(`Deleting rule "${rule.title}" failed with status: ${error.message}`, + `Could not delete processing rule "${rule.title}"`); }; - const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule/' + ruleId); + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/rule/${rule.id}`); return fetch('DELETE', url).then(() => { - this.rules = this.rules.filter((el) => el.id !== ruleId); + this.rules = this.rules.filter((el) => el.id !== rule.id); this.trigger({ rules: this.rules }); + UserNotification.success(`Rule "${rule.title}" was deleted successfully`); }, failCallback); }, parse(ruleSource, callback) { From 5feb52a8182acc49e1524378aa238f0fe66c9d10 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 14 Mar 2016 15:01:13 +0100 Subject: [PATCH 188/521] Simplify pipeline management Before the change, creating and editing pipelines had two steps: 1. Modify pipeline title and description (using a modal in the pipeline list page) 2. Modify stages in the pipeline details page The changes move the title and description form to the pipeline details page, so all steps to create and edit a pipeline are in the same place. --- src/web/pipelines/NewPipeline.jsx | 38 ++++++++ src/web/pipelines/Pipeline.jsx | 21 ++-- src/web/pipelines/PipelineDetails.jsx | 46 +++++++++ src/web/pipelines/PipelineDetailsPage.jsx | 59 +++++++++-- src/web/pipelines/PipelineForm.jsx | 97 ++++++++++++------- src/web/pipelines/PipelinesStore.js | 10 +- .../pipelines/ProcessingTimelineComponent.jsx | 31 +++--- 7 files changed, 229 insertions(+), 73 deletions(-) create mode 100644 src/web/pipelines/NewPipeline.jsx create mode 100644 src/web/pipelines/PipelineDetails.jsx diff --git a/src/web/pipelines/NewPipeline.jsx b/src/web/pipelines/NewPipeline.jsx new file mode 100644 index 000000000000..7bb4aacae0d6 --- /dev/null +++ b/src/web/pipelines/NewPipeline.jsx @@ -0,0 +1,38 @@ +import React from 'react'; +import { Row, Col } from 'react-bootstrap'; + +import PipelineDetails from './PipelineDetails'; + +const NewPipeline = React.createClass({ + propTypes: { + onChange: React.PropTypes.func.isRequired, + history: React.PropTypes.object.isRequired, + }, + + _onChange(newPipeline) { + this.props.onChange(newPipeline, this._goToPipeline); + }, + + _goToPipeline(pipeline) { + this.props.history.pushState(null, `/system/pipelines/${pipeline.id}`); + }, + + _goBack() { + this.props.history.goBack(); + }, + + render() { + return ( + + +

    + Give a name and description to the new pipeline. You can add stages to it when you save the changes. +

    + + +     + ); + }, +}); + +export default NewPipeline; diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index 783395ecc6ac..68abe683fe25 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -1,9 +1,10 @@ import React, { PropTypes } from 'react'; import { Row, Col } from 'react-bootstrap'; -import { EntityList, Timestamp } from 'components/common'; +import { EntityList } from 'components/common'; import Stage from './Stage'; import StageForm from './StageForm'; +import PipelineDetails from './PipelineDetails'; import {} from './Pipeline.css'; @@ -11,6 +12,7 @@ const Pipeline = React.createClass({ propTypes: { pipeline: PropTypes.object.isRequired, onStagesChange: PropTypes.func.isRequired, + onPipelineChange: PropTypes.func.isRequired, }, _saveStage(stage, callback) { @@ -52,20 +54,17 @@ const Pipeline = React.createClass({ return (
    - + +
    -

    Information

    -
    -
    Description
    -
    {pipeline.description}
    -
    Created
    -
    -
    Last modified
    -
    -
    +

    Pipeline Stages

    +

    + Stages are groups of conditions and actions which need to run in order, and provide the necessary{' '} + control flow to decide whether or not to run the rest of a pipeline. +

    diff --git a/src/web/pipelines/PipelineDetails.jsx b/src/web/pipelines/PipelineDetails.jsx new file mode 100644 index 000000000000..f47b2ed0950e --- /dev/null +++ b/src/web/pipelines/PipelineDetails.jsx @@ -0,0 +1,46 @@ +import React from 'react'; +import { Row, Col } from 'react-bootstrap'; + +import { Timestamp } from 'components/common'; +import PipelineForm from './PipelineForm'; + +const PipelineDetails = React.createClass({ + propTypes: { + pipeline: React.PropTypes.object, + create: React.PropTypes.bool, + onChange: React.PropTypes.func.isRequired, + }, + + render() { + if (this.props.create) { + return ; + } + + const pipeline = this.props.pipeline; + return ( +
    + + +
    + +
    +

    Details

    +
    +
    Title
    +
    {pipeline.title}
    +
    Description
    +
    {pipeline.description}
    +
    Created
    +
    +
    Last modified
    +
    +
    + +     +
    +
    + ); + }, +}); + +export default PipelineDetails; diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index c1295f0d649e..afce2ed56211 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -1,10 +1,11 @@ -import React, {PropTypes} from 'react'; +import React from 'react'; import Reflux from 'reflux'; import { Row, Col, Button } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; import { PageHeader, Spinner } from 'components/common'; import Pipeline from './Pipeline'; +import NewPipeline from './NewPipeline'; import SourceGenerator from 'logic/SourceGenerator'; import ObjectUtils from 'util/ObjectUtils'; @@ -18,16 +19,25 @@ function filterPipeline(state) { const PipelineDetailsPage = React.createClass({ propTypes: { - params: PropTypes.object.isRequired, + params: React.PropTypes.object.isRequired, + history: React.PropTypes.object.isRequired, }, mixins: [Reflux.connectFilter(PipelinesStore, 'pipeline', filterPipeline)], componentDidMount() { - PipelinesActions.get(this.props.params.pipelineId); + if (!this._isNewPipeline(this.props.params.pipelineId)) { + PipelinesActions.get(this.props.params.pipelineId); + } RulesStore.list(); }, + componentWillReceiveProps(nextProps) { + if (!this._isNewPipeline(nextProps.params.pipelineId)) { + PipelinesActions.get(nextProps.params.pipelineId); + } + }, + _onStagesChange(newStages, callback) { const newPipeline = ObjectUtils.clone(this.state.pipeline); newPipeline.stages = newStages; @@ -39,14 +49,49 @@ const PipelineDetailsPage = React.createClass({ } }, + _savePipeline(pipeline, callback) { + const requestPipeline = ObjectUtils.clone(pipeline); + requestPipeline.source = SourceGenerator.generatePipeline(pipeline); + let promise; + if (requestPipeline.id) { + promise = PipelinesActions.update(requestPipeline); + } else { + promise = PipelinesActions.save(requestPipeline); + } + + promise.then(p => callback(p)); + }, + + _isNewPipeline(pipelineId) { + return pipelineId === 'new'; + }, + + _isLoading() { + return !this._isNewPipeline(this.props.params.pipelineId) && !this.state.pipeline; + }, + render() { - if (!this.state.pipeline) { - return ; + if (this._isLoading()) { + return ; + } + + let title; + if (this._isNewPipeline(this.props.params.pipelineId)) { + title = 'New pipeline'; + } else { + title = Pipeline {this.state.pipeline.title}; + } + + let content; + if (this._isNewPipeline(this.props.params.pipelineId)) { + content = ; + } else { + content = ; } return (
    - Pipeline {this.state.pipeline.title}} titleSize={9} buttonSize={3}> + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} rules are evaluated and applied. Messages can go through one or more stages. @@ -68,7 +113,7 @@ const PipelineDetailsPage = React.createClass({ - + {content}
    diff --git a/src/web/pipelines/PipelineForm.jsx b/src/web/pipelines/PipelineForm.jsx index c233f1dd5187..534da0bab728 100644 --- a/src/web/pipelines/PipelineForm.jsx +++ b/src/web/pipelines/PipelineForm.jsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Input, Button } from 'react-bootstrap'; +import { Row, Col, Input, Button } from 'react-bootstrap'; import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; import ObjectUtils from 'util/ObjectUtils'; @@ -9,12 +9,15 @@ const PipelineForm = React.createClass({ propTypes: { pipeline: React.PropTypes.object, create: React.PropTypes.bool, + modal: React.PropTypes.bool, save: React.PropTypes.func.isRequired, validatePipeline: React.PropTypes.func.isRequired, + onCancel: React.PropTypes.func, }, getDefaultProps() { return { + modal: true, pipeline: { id: undefined, title: '', @@ -52,13 +55,20 @@ const PipelineForm = React.createClass({ }, _saved() { - this._closeModal(); + if (this.props.modal) { + this._closeModal(); + } + if (this.props.create) { this.setState(this.getInitialState()); } }, - _save() { + _save(event) { + if (event) { + event.preventDefault(); + } + this.props.save(this.state.pipeline, this._saved); }, @@ -67,41 +77,58 @@ const PipelineForm = React.createClass({ if (this.props.create) { triggerButtonContent = 'Add new pipeline'; } else { - triggerButtonContent = Edit; + triggerButtonContent = 'Edit pipeline details'; + } + + const content = ( +
    + + + +
    + ); + + if (this.props.modal) { + return ( + + + + {content} + + + ); } return ( - - - -
    - - - -
    -     -     + + {content} + + + + + + + ); }, }); diff --git a/src/web/pipelines/PipelinesStore.js b/src/web/pipelines/PipelinesStore.js index 79ded6de7f18..051c6b0d65e0 100644 --- a/src/web/pipelines/PipelinesStore.js +++ b/src/web/pipelines/PipelinesStore.js @@ -65,12 +65,15 @@ const PipelinesStore = Reflux.createStore({ description: pipelineSource.description, source: pipelineSource.source, }; - return fetch('POST', url, pipeline).then( + const promise = fetch('POST', url, pipeline); + promise.then( response => { this._updatePipelinesState(response); UserNotification.success(`Pipeline "${pipeline.title}" created successfully`); }, failCallback); + + PipelinesActions.save.promise(promise); }, update(pipelineSource) { @@ -85,12 +88,15 @@ const PipelinesStore = Reflux.createStore({ description: pipelineSource.description, source: pipelineSource.source, }; - return fetch('PUT', url, pipeline).then( + const promise = fetch('PUT', url, pipeline); + promise.then( response => { this._updatePipelinesState(response); UserNotification.success(`Pipeline "${pipeline.title}" updated successfully`); }, failCallback); + + PipelinesActions.update.promise(promise); }, delete(pipelineId) { const failCallback = (error) => { diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 2bd2da362ef2..0154a5956044 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -5,14 +5,10 @@ import { LinkContainer } from 'react-router-bootstrap'; import naturalSort from 'javascript-natural-sort'; import { DataTable, Spinner } from 'components/common'; -import PipelineForm from './PipelineForm'; import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; -import ObjectUtils from 'util/ObjectUtils'; -import SourceGenerator from 'logic/SourceGenerator'; - import {} from './ProcessingTimelineComponent.css'; const ProcessingTimelineComponent = React.createClass({ @@ -71,23 +67,14 @@ const ProcessingTimelineComponent = React.createClass({   - + + + ); }, - _savePipeline(pipeline, callback) { - const requestPipeline = ObjectUtils.clone(pipeline); - requestPipeline.source = SourceGenerator.generatePipeline(pipeline); - if (requestPipeline.id) { - PipelinesActions.update(requestPipeline); - } else { - PipelinesActions.save(requestPipeline); - } - callback(); - }, - _deletePipeline(pipeline) { return () => { if (confirm(`Do you really want to delete pipeline "${pipeline.title}"? This action cannot be undone.`)) { @@ -101,10 +88,18 @@ const ProcessingTimelineComponent = React.createClass({ return ; } + const addNewPipelineButton = ( +
    + + + +
    + ); + if (this.state.pipelines.length === 0) { return (
    -
    + {addNewPipelineButton} There are no pipelines configured in your system. Create one to start processing your messages. @@ -117,7 +112,7 @@ const ProcessingTimelineComponent = React.createClass({ const headers = ['Pipeline', 'ProcessingTimeline', 'Actions']; return (
    -
    + {addNewPipelineButton} Date: Tue, 15 Mar 2016 17:30:19 +0100 Subject: [PATCH 189/521] Make function names more consistent Function names should be lowercase and words be separated by an underscore, e. g. "to_string" instead of "tostring". --- .../functions/conversion/BooleanConversion.java | 2 +- .../functions/conversion/DoubleConversion.java | 2 +- .../functions/conversion/LongConversion.java | 2 +- .../functions/conversion/StringConversion.java | 2 +- .../functions/ips/IpAddressConversion.java | 4 ++-- .../functions/FunctionsSnippetsTest.java | 2 +- .../processors/PipelineInterpreterTest.java | 2 +- .../plugins/pipelineprocessor/functions/dates.txt | 2 +- .../pipelineprocessor/functions/evalError.txt | 4 ++-- .../functions/evalErrorSuppressed.txt | 2 +- .../pipelineprocessor/functions/ipMatching.txt | 8 ++++---- .../plugins/pipelineprocessor/functions/jsonpath.txt | 2 +- .../plugins/pipelineprocessor/parser/messageRef.txt | 4 ++-- .../parser/messageRefQuotedField.txt | 2 +- .../pipelineprocessor/parser/typedFieldAccess.txt | 4 ++-- src/web/rules/RuleHelper.jsx | 12 ++++++------ 16 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java index a8081196f396..500c3b055cb4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java @@ -26,7 +26,7 @@ import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; public class BooleanConversion extends AbstractFunction { - public static final String NAME = "tobool"; + public static final String NAME = "to_bool"; private final ParameterDescriptor valueParam; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java index 90b33681ee6e..78c4eb9cc64c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java @@ -30,7 +30,7 @@ public class DoubleConversion extends AbstractFunction { - public static final String NAME = "todouble"; + public static final String NAME = "to_double"; private static final String VALUE = "value"; private static final String DEFAULT = "default"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java index 999b3b35523f..b65687467898 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java @@ -30,7 +30,7 @@ public class LongConversion extends AbstractFunction { - public static final String NAME = "tolong"; + public static final String NAME = "to_long"; private static final String VALUE = "value"; private static final String DEFAULT = "default"; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 4b76f718aa83..7590c9b9f354 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -33,7 +33,7 @@ public class StringConversion extends AbstractFunction { - public static final String NAME = "tostring"; + public static final String NAME = "to_string"; // this is per-thread to save an expensive concurrent hashmap access private final ThreadLocal, Class>> declaringClassCache; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index c3a1bd94f06c..099b30ada849 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -31,7 +31,7 @@ public class IpAddressConversion extends AbstractFunction { - public static final String NAME = "toip"; + public static final String NAME = "to_ip"; private final ParameterDescriptor ipParam; private final ParameterDescriptor defaultParam; @@ -55,7 +55,7 @@ public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { try { return new IpAddress(InetAddresses.forString(defaultValue.get())); } catch (IllegalFormatException e1) { - log.warn("Parameter `default` for toip() is not a valid IP address: {}", defaultValue.get()); + log.warn("Parameter `default` for to_ip() is not a valid IP address: {}", defaultValue.get()); throw e1; } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index d8d7afb50cd0..97721f76d48e 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -285,7 +285,7 @@ public void evalError() { assertThat(context).isNotNull(); assertThat(context.hasEvaluationErrors()).isTrue(); - assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'toip' at 5:28 an exception was thrown: 'null' is not an IP string literal."); + assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'to_ip' at 5:28 an exception was thrown: 'null' is not an IP string literal."); } @Test diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java index b165d82c75d7..04db52aa9544 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -56,7 +56,7 @@ public void testCreateMessage() { "title", "description", "rule \"creates message\"\n" + - "when tostring($message.message) == \"original message\"\n" + + "when to_string($message.message) == \"original message\"\n" + "then\n" + " create_message(\"derived message\");\n" + "end", diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt index 42a3bed59653..051fc005b101 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt @@ -11,5 +11,5 @@ then trigger_test(); let date = parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ"); set_field("year", date.year); - set_field("timezone", tostring(date.zone)); + set_field("timezone", to_string(date.zone)); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt index 4c276e1d821c..33d692fdc9b2 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt @@ -2,5 +2,5 @@ rule "trigger null" when true then - set_field("this_is_null", toip($message.does_not_exist)); -end \ No newline at end of file + set_field("this_is_null", to_ip($message.does_not_exist)); +end diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt index 1d14675ce0f0..36e1873f912e 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt @@ -1,6 +1,6 @@ rule "suppressing exceptions/nulls" when - is_null(toip($message.does_not_exist)) && is_not_null($message.this_field_was_set) + is_null(to_ip($message.does_not_exist)) && is_not_null($message.this_field_was_set) then trigger_test(); end diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt index 216f22365b08..d38f8efd85e3 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt @@ -1,9 +1,9 @@ rule "ip handling" when - cidr_match("192.0.0.0/8", toip("192.168.1.50")) && - ! cidr_match("191.0.0.0/8", toip("192.168.1.50")) + cidr_match("192.0.0.0/8", to_ip("192.168.1.50")) && + ! cidr_match("191.0.0.0/8", to_ip("192.168.1.50")) then - set_field("ip_anon", tostring(toip($message.ip).anonymized)); - set_field("ipv6_anon", tostring(toip("2001:db8::1").anonymized)); + set_field("ip_anon", to_string(to_ip($message.ip).anonymized)); + set_field("ipv6_anon", to_string(to_ip("2001:db8::1").anonymized)); trigger_test(); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt index 1f46a04629c2..e036970bbedc 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt @@ -1,7 +1,7 @@ rule "jsonpath" when true then - let x = parse_json(tostring($message.message)); + let x = parse_json(to_string($message.message)); let new_fields = select_jsonpath(x, { author_first: "$['store']['book'][0]['author']", author_last: "$['store']['book'][-1:]['author']" diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt index 9617b38895c3..c34a6101026a 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt @@ -1,5 +1,5 @@ rule "message field ref" -when tolong(value: $message.responseCode, default: 200) >= 500 +when to_long(value: $message.responseCode, default: 200) >= 500 then set_field(field: "response_category", value: "server_error"); -end \ No newline at end of file +end diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt index 087e9c3d15bf..c22284f1278a 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt @@ -1,5 +1,5 @@ rule "test" -when tostring($message.`@specialfieldname`, "empty") == "string" +when to_string($message.`@specialfieldname`, "empty") == "string" then trigger_test(); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt index 5518b85c7c03..9bec52c605a2 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt @@ -1,6 +1,6 @@ rule "typed field access" when - tolong(customObject("1").id, 0) < 2 + to_long(customObject("1").id, 0) < 2 then trigger_test(); -end \ No newline at end of file +end diff --git a/src/web/rules/RuleHelper.jsx b/src/web/rules/RuleHelper.jsx index e6b9ddfec311..900c6db63b11 100644 --- a/src/web/rules/RuleHelper.jsx +++ b/src/web/rules/RuleHelper.jsx @@ -11,7 +11,7 @@ when has_field("transaction_date") then // the following date format assumes there's no time zone in the string - let new_date = parse_date(tostring($message.transaction_date), "yyyy-MM-dd HH:mm:ss"); + let new_date = parse_date(to_string($message.transaction_date), "yyyy-MM-dd HH:mm:ss"); set_field("transaction_year", new_date.year); end`, @@ -46,19 +46,19 @@ end`, - tobool(any) + to_bool(any) Converts the single parameter to a boolean value using its string value. - todouble(any, [default: double]) + to_double(any, [default: double]) Converts the first parameter to a double floating point value. - tolong(any, [default: long]) + to_long(any, [default: long]) Converts the first parameter to a long integer value. - tostring(any, [default: string]) + to_string(any, [default: string]) Converts the first parameter to its string representation. @@ -103,7 +103,7 @@ end`, Parse a string into a JSON tree. - toip(ip: string) + to_ip(ip: string) Converts the given string to an IP object. From 2ebcfcc99b8a6b1178359c7fca719115954381de Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Tue, 15 Mar 2016 17:41:14 +0100 Subject: [PATCH 190/521] Add additional hash functions (CRC32, CRC32C, Murmur3_32, Murmur3_128) --- .../functions/ProcessorFunctionsModule.java | 8 +++++ .../functions/hashing/CRC32.java | 36 +++++++++++++++++++ .../functions/hashing/CRC32C.java | 36 +++++++++++++++++++ .../functions/hashing/Murmur3_128.java | 36 +++++++++++++++++++ .../functions/hashing/Murmur3_32.java | 36 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 8 +++++ .../pipelineprocessor/functions/digests.txt | 4 +++ 7 files changed, 164 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 4bf621508e30..8fd3349e8c95 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -28,7 +28,11 @@ import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32C; import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_128; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_32; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; @@ -100,7 +104,11 @@ protected void configure() { addMessageProcessorFunction(FormatDate.NAME, FormatDate.class); // hash digest + addMessageProcessorFunction(CRC32.NAME, CRC32.class); + addMessageProcessorFunction(CRC32C.NAME, CRC32C.class); addMessageProcessorFunction(MD5.NAME, MD5.class); + addMessageProcessorFunction(Murmur3_32.NAME, Murmur3_32.class); + addMessageProcessorFunction(Murmur3_128.NAME, Murmur3_128.class); addMessageProcessorFunction(SHA1.NAME, SHA1.class); addMessageProcessorFunction(SHA256.NAME, SHA256.class); addMessageProcessorFunction(SHA512.NAME, SHA512.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java new file mode 100644 index 000000000000..a4f8df956e39 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class CRC32 extends SingleArgStringFunction { + + public static final String NAME = "crc32"; + + @Override + protected String getDigest(String value) { + return Hashing.crc32().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java new file mode 100644 index 000000000000..746551e75882 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class CRC32C extends SingleArgStringFunction { + + public static final String NAME = "crc32c"; + + @Override + protected String getDigest(String value) { + return Hashing.crc32c().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java new file mode 100644 index 000000000000..c58abfd3434b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class Murmur3_128 extends SingleArgStringFunction { + + public static final String NAME = "murmur3_128"; + + @Override + protected String getDigest(String value) { + return Hashing.murmur3_128().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java new file mode 100644 index 000000000000..fc7847fb3565 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class Murmur3_32 extends SingleArgStringFunction { + + public static final String NAME = "murmur3_32"; + + @Override + protected String getDigest(String value) { + return Hashing.murmur3_32().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index d8d7afb50cd0..f76b3018e234 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -31,7 +31,11 @@ import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; import org.graylog.plugins.pipelineprocessor.functions.dates.Now; import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32C; import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_128; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_32; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; @@ -136,7 +140,11 @@ public static void registerFunctions() { functions.put(ParseDate.NAME, new ParseDate()); functions.put(FormatDate.NAME, new FormatDate()); + functions.put(CRC32.NAME, new CRC32()); + functions.put(CRC32C.NAME, new CRC32C()); functions.put(MD5.NAME, new MD5()); + functions.put(Murmur3_32.NAME, new Murmur3_32()); + functions.put(Murmur3_128.NAME, new Murmur3_128()); functions.put(SHA1.NAME, new SHA1()); functions.put(SHA256.NAME, new SHA256()); functions.put(SHA512.NAME, new SHA512()); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt index 3221ef67396d..3e938e60fea0 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt @@ -1,6 +1,10 @@ rule "digests" when + crc32("graylog") == "e3018c57" && + crc32c("graylog") == "82390e89" && md5("graylog") == "6f9efb466e043b9f3635827ce446e13c" && + murmur3_32("graylog") == "67285534" && + murmur3_128("graylog") == "945d5b1aaa8fdfe9b880b31e814972b3" && sha1("graylog") == "6d88bccf40bf65b911fe79d78c7af98e382f0c1a" && sha256("graylog") == "4bbdd5a829dba09d7a7ff4c1367be7d36a017b4267d728d31bd264f63debeaa6" && sha512("graylog") == "f6cb3a96450fb9c9174299a651333c926cd67b6f5c25d8daeede1589ffa006f4dd31da4f0625b7f281051a34c8352b3a9c1a9babf90020360e911a380b5c3f4f" From a7c122c2ecfec16158ff2c92b9142152fdfc3bb3 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 15 Mar 2016 19:27:10 +0100 Subject: [PATCH 191/521] Update plugin name Use same as Java, to fix shadow module.json --- webpack.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webpack.config.js b/webpack.config.js index 1b5effec7f8a..ac1a7cf71326 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -2,7 +2,7 @@ const PluginWebpackConfig = require('graylog-web-plugin').PluginWebpackConfig; const loadBuildConfig = require('graylog-web-plugin').loadBuildConfig; const path = require('path'); -module.exports = new PluginWebpackConfig('org.graylog.plugins.pipelineprocessor.PipelineProcessor', loadBuildConfig(path.resolve(__dirname, './build.config')), { +module.exports = new PluginWebpackConfig('org.graylog.plugins.pipelineprocessor.PipelineProcessorPlugin', loadBuildConfig(path.resolve(__dirname, './build.config')), { loaders: [ { test: /\.ts$/, loader: 'babel-loader!ts-loader', exclude: /node_modules|\.node_cache/ }, ], From 0a02e2d5d6ea0d8964e0158d2638da733d2b9877 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 15 Mar 2016 19:27:26 +0100 Subject: [PATCH 192/521] Bump graylog-web-plugin dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 939281775d07..fd9211a09742 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "eslint-loader": "^1.0.0", "eslint-plugin-react": "^4.1.0", "graylog-web-manifests": "^2.0.0-alpha.3", - "graylog-web-plugin": "~0.0.18", + "graylog-web-plugin": "latest", "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", "style-loader": "^0.13.0", From 6dab87bdb50483c9f14fc9b48e0f29ae045d5f87 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 18 Mar 2016 15:26:31 +0100 Subject: [PATCH 193/521] Make style usage explicit Using `style/useable` to load and unload CSS --- src/web/pipelines/Pipeline.jsx | 12 ++++++++++-- src/web/pipelines/ProcessingTimelineComponent.jsx | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index 68abe683fe25..9df73d08fce3 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -6,8 +6,6 @@ import Stage from './Stage'; import StageForm from './StageForm'; import PipelineDetails from './PipelineDetails'; -import {} from './Pipeline.css'; - const Pipeline = React.createClass({ propTypes: { pipeline: PropTypes.object.isRequired, @@ -15,6 +13,16 @@ const Pipeline = React.createClass({ onPipelineChange: PropTypes.func.isRequired, }, + componentDidMount() { + this.style.use(); + }, + + componentWillUnmount() { + this.style.unuse(); + }, + + style: require('!style/useable!css!./Pipeline.css'), + _saveStage(stage, callback) { const newStages = this.props.pipeline.stages.slice(); newStages.push(stage); diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 0154a5956044..06ce31ebb50f 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -9,15 +9,20 @@ import { DataTable, Spinner } from 'components/common'; import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; -import {} from './ProcessingTimelineComponent.css'; - const ProcessingTimelineComponent = React.createClass({ mixins: [Reflux.connect(PipelinesStore)], componentDidMount() { + this.style.use(); PipelinesActions.list(); }, + componentWillUnmount() { + this.style.unuse(); + }, + + style: require('!style/useable!css!./ProcessingTimelineComponent.css'), + _calculateUsedStages(pipelines) { return pipelines .map(pipeline => pipeline.stages) From adec6627e3820796b1515a487be652088c1c9729 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 18 Mar 2016 16:16:40 +0100 Subject: [PATCH 194/521] Subscribe to server event bus and publish to cluster event bus --- .../pipelineprocessor/processors/PipelineInterpreter.java | 6 +++--- .../pipelineprocessor/rest/PipelineConnectionsResource.java | 2 +- .../plugins/pipelineprocessor/rest/PipelineResource.java | 2 +- .../plugins/pipelineprocessor/rest/RuleResource.java | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 97c7db254f28..abe02ef57f2b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -38,12 +38,12 @@ import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent; import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; -import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; import org.graylog2.plugin.Messages; @@ -94,7 +94,7 @@ public PipelineInterpreter(RuleService ruleService, Journal journal, MetricRegistry metricRegistry, @Named("daemonScheduler") ScheduledExecutorService scheduler, - @ClusterEventBus EventBus clusterBus) { + EventBus serverEventBus) { this.ruleService = ruleService; this.pipelineService = pipelineService; this.pipelineStreamConnectionsService = pipelineStreamConnectionsService; @@ -105,7 +105,7 @@ public PipelineInterpreter(RuleService ruleService, this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); // listens to cluster wide Rule, Pipeline and pipeline stream connection changes - clusterBus.register(this); + serverEventBus.register(this); reload(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index ac62fc4a1468..5130f7b6b775 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -55,7 +55,7 @@ public class PipelineConnectionsResource extends RestResource implements PluginR public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsService, PipelineService pipelineService, StreamService streamService, - @ClusterEventBus EventBus clusterBus) { + ClusterEventBus clusterBus) { this.connectionsService = connectionsService; this.pipelineService = pipelineService; this.streamService = streamService; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index 3b52a88de99c..57d8bc5de87d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -66,7 +66,7 @@ public class PipelineResource extends RestResource implements PluginRestResource @Inject public PipelineResource(PipelineService pipelineService, PipelineRuleParser pipelineRuleParser, - @ClusterEventBus EventBus clusterBus) { + ClusterEventBus clusterBus) { this.pipelineService = pipelineService; this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 04f657a21ff2..67ffae058cd3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -65,7 +65,7 @@ public class RuleResource extends RestResource implements PluginRestResource { @Inject public RuleResource(RuleService ruleService, PipelineRuleParser pipelineRuleParser, - @ClusterEventBus EventBus clusterBus) { + ClusterEventBus clusterBus) { this.ruleService = ruleService; this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; From 9a8a2addb78fb86ea0873ec01d118e18f53f7fda Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 18 Mar 2016 16:21:05 +0100 Subject: [PATCH 195/521] Use "id" instead of "_id" everywhere --- .../pipelineprocessor/rest/PipelineConnections.java | 8 ++++---- .../plugins/pipelineprocessor/rest/PipelineSource.java | 2 +- .../plugins/pipelineprocessor/rest/RuleSource.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java index b49b4ee95197..ee14a6a2ba6e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java @@ -34,7 +34,7 @@ public abstract class PipelineConnections { @Nullable @Id @ObjectId - public abstract String _id(); + public abstract String id(); @JsonProperty public abstract String streamId(); @@ -43,11 +43,11 @@ public abstract class PipelineConnections { public abstract Set pipelineIds(); @JsonCreator - public static PipelineConnections create(@Id @ObjectId @JsonProperty("id") @Nullable String _id, + public static PipelineConnections create(@JsonProperty("id") @Id @ObjectId @Nullable String id, @JsonProperty("stream_id") String streamId, @JsonProperty("pipeline_ids") Set pipelineIds) { return builder() - ._id(_id) + .id(id) .streamId(streamId) .pipelineIds(pipelineIds) .build(); @@ -63,7 +63,7 @@ public static Builder builder() { public abstract static class Builder { public abstract PipelineConnections build(); - public abstract Builder _id(String _id); + public abstract Builder id(String id); public abstract Builder streamId(String streamId); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java index e7b36e4e0057..ad6df5bf902b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -78,7 +78,7 @@ public static Builder builder() { public abstract Builder toBuilder(); @JsonCreator - public static PipelineSource create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + public static PipelineSource create(@JsonProperty("id") @Id @ObjectId @Nullable String id, @JsonProperty("title") String title, @JsonProperty("description") @Nullable String description, @JsonProperty("source") String source, diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index ea6d3e9faf41..a83cb43306e6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -71,7 +71,7 @@ public static Builder builder() { public abstract Builder toBuilder(); @JsonCreator - public static RuleSource create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + public static RuleSource create(@JsonProperty("id") @Id @ObjectId @Nullable String id, @JsonProperty("title") String title, @JsonProperty("description") @Nullable String description, @JsonProperty("source") String source, From a108051d2da556e0a37deb971ff4afe270e4a719 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Fri, 18 Mar 2016 16:21:52 +0100 Subject: [PATCH 196/521] Publish a PipelineConnectionsChangedEvent instead of the db object This fixes deserialization of the cluster event. --- .../PipelineConnectionsChangedEvent.java | 40 +++++++++++++++++++ .../processors/PipelineInterpreter.java | 4 +- .../rest/PipelineConnectionsResource.java | 3 +- 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java b/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java new file mode 100644 index 000000000000..b587d001bee7 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java @@ -0,0 +1,40 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import java.util.Set; + +@JsonAutoDetect +@AutoValue +public abstract class PipelineConnectionsChangedEvent { + @JsonProperty("stream_id") + public abstract String streamId(); + + @JsonProperty("pipeline_ids") + public abstract Set pipelineIds(); + + @JsonCreator + public static PipelineConnectionsChangedEvent create(@JsonProperty("stream_id") String streamId, + @JsonProperty("pipeline_ids") Set pipelineIds) { + return new AutoValue_PipelineConnectionsChangedEvent(streamId, pipelineIds); + } +} \ No newline at end of file diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index abe02ef57f2b..a5772837a11b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -363,8 +363,8 @@ public void handlePipelineChanges(PipelinesChangedEvent event) { } @Subscribe - public void handlePipelineConnectionChanges(PipelineConnections connection) { - log.debug("Pipeline stream connection changed: {}", connection); + public void handlePipelineConnectionChanges(PipelineConnectionsChangedEvent event) { + log.debug("Pipeline stream connection changed: {}", event); scheduler.schedule((Runnable) this::reload, 0, TimeUnit.SECONDS); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index 5130f7b6b775..1df56afe5378 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -22,6 +22,7 @@ import com.wordnik.swagger.annotations.ApiParam; import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent; import org.graylog2.database.NotFoundException; import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.rest.PluginRestResource; @@ -75,7 +76,7 @@ public PipelineConnections connectPipelines(@ApiParam(name = "Json body", requir pipelineService.load(s); } final PipelineConnections save = connectionsService.save(connection); - clusterBus.post(save); + clusterBus.post(PipelineConnectionsChangedEvent.create(save.streamId(), save.pipelineIds())); return save; } From 1edd3d578da90f1f994e852824f8d49767c7ca25 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 17 Mar 2016 15:17:16 +0100 Subject: [PATCH 197/521] add interpreter counter metrics use CounterRate to display per second rates in the pipeline pages: - throughput and error rates - per pipeline - per stage - per rule (globally and per pipeline-stage) --- .../plugins/pipelineprocessor/ast/Rule.java | 11 +++++ .../parser/PipelineRuleParser.java | 8 +++- .../processors/PipelineInterpreter.java | 47 ++++++++++++++----- .../pipelineprocessor/rest/RuleResource.java | 6 +-- .../pipelineprocessor/rest/RuleSource.java | 2 +- src/web/pipelines/Pipeline.css | 2 +- src/web/pipelines/Pipeline.jsx | 2 +- src/web/pipelines/PipelineDetails.jsx | 8 ++++ .../pipelines/ProcessingTimelineComponent.jsx | 7 ++- src/web/pipelines/Stage.jsx | 35 +++++++++++--- src/web/rules/RuleList.jsx | 14 +++++- 11 files changed, 115 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java index 4ac5271c9360..7f40683e75cc 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java @@ -22,12 +22,16 @@ import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import javax.annotation.Nullable; import java.util.Collection; import java.util.Collections; @AutoValue public abstract class Rule { + @Nullable + public abstract String id(); + public abstract String name(); public abstract LogicalExpression when(); @@ -38,12 +42,19 @@ public static Builder builder() { return new AutoValue_Rule.Builder(); } + public abstract Builder toBuilder(); + + public Rule withId(String id) { + return toBuilder().id(id).build(); + } + public static Rule alwaysFalse(String name) { return builder().name(name).when(new BooleanExpression(new CommonToken(-1), false)).then(Collections.emptyList()).build(); } @AutoValue.Builder public abstract static class Builder { + public abstract Builder id(String id); public abstract Builder name(String name); public abstract Builder when(LogicalExpression condition); public abstract Builder then(Collection actions); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index ebacbfaa4d94..53e98c310c5c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -101,7 +101,12 @@ public PipelineRuleParser(FunctionRegistry functionRegistry) { private static final Logger log = LoggerFactory.getLogger(PipelineRuleParser.class); public static final ParseTreeWalker WALKER = ParseTreeWalker.DEFAULT; + public Rule parseRule(String rule, boolean silent) throws ParseException { + return parseRule(null, rule, silent); + } + + public Rule parseRule(String id, String rule, boolean silent) throws ParseException { final ParseContext parseContext = new ParseContext(silent); final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); @@ -128,7 +133,8 @@ public Rule parseRule(String rule, boolean silent) throws ParseException { WALKER.walk(new RuleTypeChecker(parseContext), ruleDeclaration); if (parseContext.getErrors().isEmpty()) { - return parseContext.getRules().get(0); + final Rule parsedRule = parseContext.getRules().get(0); + return parsedRule.withId(id); } throw new ParseException(parseContext.getErrors()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index a5772837a11b..c2256971937b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -80,6 +80,7 @@ public class PipelineInterpreter implements MessageProcessor { private final PipelineStreamConnectionsService pipelineStreamConnectionsService; private final PipelineRuleParser pipelineRuleParser; private final Journal journal; + private final MetricRegistry metricRegistry; private final ScheduledExecutorService scheduler; private final Meter filteredOutMessages; @@ -101,6 +102,7 @@ public PipelineInterpreter(RuleService ruleService, this.pipelineRuleParser = pipelineRuleParser; this.journal = journal; + this.metricRegistry = metricRegistry; this.scheduler = scheduler; this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); @@ -117,7 +119,7 @@ private synchronized void reload() { for (RuleDao ruleDao : ruleService.loadAll()) { Rule rule; try { - rule = pipelineRuleParser.parseRule(ruleDao.source(), false); + rule = pipelineRuleParser.parseRule(ruleDao.id(), ruleDao.source(), false); } catch (ParseException e) { rule = Rule.alwaysFalse("Failed to parse rule: " + ruleDao.id()); } @@ -206,9 +208,11 @@ public Messages process(Messages messages) { } else { // get the default stream pipeline connections for this message pipelinesToRun = streamConnection.get("default"); - log.debug("[{}] running default stream pipelines: [{}]", - msgId, - pipelinesToRun.stream().map(Pipeline::name).toArray()); + if (log.isDebugEnabled()) { + log.debug("[{}] running default stream pipelines: [{}]", + msgId, + pipelinesToRun.stream().map(Pipeline::name).toArray()); + } } } else { // 2. if a message-stream combination has already been processed (is in the set), skip that execution @@ -222,8 +226,11 @@ public Messages process(Messages messages) { log.debug("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); } + // record execution of pipeline in metrics + pipelinesToRun.stream().forEach(pipeline -> metricRegistry.counter(name(Pipeline.class, pipeline.id(), "executed")).inc()); + final StageIterator stages = new StageIterator(pipelinesToRun); - final Set pipelinesToProceedWith = Sets.newHashSet(); + final Set pipelinesToSkip = Sets.newHashSet(); // iterate through all stages for all matching pipelines, per "stage slice" instead of per pipeline. // pipeline execution ordering is not guaranteed @@ -232,13 +239,13 @@ public Messages process(Messages messages) { for (Tuple2 pair : stageSet) { final Stage stage = pair.v1(); final Pipeline pipeline = pair.v2(); - if (!pipelinesToProceedWith.isEmpty() && - !pipelinesToProceedWith.contains(pipeline)) { + if (pipelinesToSkip.contains(pipeline)) { log.debug("[{}] previous stage result prevents further processing of pipeline `{}`", msgId, pipeline.name()); continue; } + metricRegistry.counter(name(Pipeline.class, pipeline.id(), "stage", String.valueOf(stage.stage()), "executed")).inc(); log.debug("[{}] evaluating rule conditions in stage {}: match {}", msgId, stage.stage(), @@ -249,8 +256,12 @@ public Messages process(Messages messages) { // 3. iterate over all the stages in these pipelines and execute them in order final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); + boolean anyRulesMatched = false; for (Rule rule : stage.getRules()) { if (rule.when().evaluateBool(context)) { + anyRulesMatched = true; + countRuleExecution(rule, pipeline, stage, "matched"); + if (context.hasEvaluationErrors()) { final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); appendProcessingError(rule, message, lastError.toString()); @@ -261,10 +272,12 @@ public Messages process(Messages messages) { log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); rulesToRun.add(rule); } else { + countRuleExecution(rule, pipeline, stage, "not-matched"); log.debug("[{}] rule `{}` does not match", msgId, rule.name()); } } for (Rule rule : rulesToRun) { + countRuleExecution(rule, pipeline, stage, "executed"); log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); for (Statement statement : rule.then()) { statement.evaluate(context); @@ -274,6 +287,7 @@ public Messages process(Messages messages) { appendProcessingError(rule, message, lastError.toString()); log.debug("Encountered evaluation error, skipping rest of the rule: {}", lastError); + countRuleExecution(rule, pipeline, stage, "failed"); break; } } @@ -284,10 +298,14 @@ public Messages process(Messages messages) { // any rule could match, but at least one had to, // record that it is ok to proceed with the pipeline if ((stage.matchAll() && (rulesToRun.size() == stage.getRules().size())) - || (rulesToRun.size() > 0)) { - log.debug("[{}] stage for pipeline `{}` required match: {}, ok to proceed with next stage", - msgId, pipeline.name(), stage.matchAll() ? "all" : "either"); - pipelinesToProceedWith.add(pipeline); + || (rulesToRun.size() > 0 && anyRulesMatched)) { + log.debug("[{}] stage {} for pipeline `{}` required match: {}, ok to proceed with next stage", + msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); + } else { + // no longer execute stages from this pipeline, the guard prevents it + log.debug("[{}] stage {} for pipeline `{}` required match: {}, NOT ok to proceed with next stage", + msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); + pipelinesToSkip.add(pipeline); } // 4. after each complete stage run, merge the processing changes, stages are isolated from each other @@ -331,6 +349,11 @@ public Messages process(Messages messages) { return new MessageCollection(fullyProcessed); } + private void countRuleExecution(Rule rule, Pipeline pipeline, Stage stage, String type) { + metricRegistry.counter(name(Rule.class, rule.id(), type)).inc(); + metricRegistry.counter(name(Rule.class, rule.id(), pipeline.id(), String.valueOf(stage.stage()), type)).inc(); + } + private void appendProcessingError(Rule rule, Message message, String errorString) { final String msg = "For rule '" + rule.name() + "': " + errorString; if (message.hasField(GL2_PROCESSING_ERROR)) { @@ -344,6 +367,7 @@ private void appendProcessingError(Rule rule, Message message, String errorStrin public void handleRuleChanges(RulesChangedEvent event) { event.deletedRuleIds().forEach(id -> { log.debug("Invalidated rule {}", id); + metricRegistry.removeMatching((name, metric) -> name.startsWith(name(Pipeline.class, id))); }); event.updatedRuleIds().forEach(id -> { log.debug("Refreshing rule {}", id); @@ -355,6 +379,7 @@ public void handleRuleChanges(RulesChangedEvent event) { public void handlePipelineChanges(PipelinesChangedEvent event) { event.deletedPipelineIds().forEach(id -> { log.debug("Invalidated pipeline {}", id); + metricRegistry.removeMatching((name, metric) -> name.startsWith(name(Pipeline.class, id))); }); event.updatedPipelineIds().forEach(id -> { log.debug("Refreshing pipeline {}", id); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 67ffae058cd3..debf91b258d0 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -78,7 +78,7 @@ public RuleResource(RuleService ruleService, public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { final Rule rule; try { - rule = pipelineRuleParser.parseRule(ruleSource.source(), false); + rule = pipelineRuleParser.parseRule(ruleSource.id(), ruleSource.source(), false); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -103,7 +103,7 @@ public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleS final Rule rule; try { // be silent about parse errors here, many requests will result in invalid syntax - rule = pipelineRuleParser.parseRule(ruleSource.source(), true); + rule = pipelineRuleParser.parseRule(ruleSource.id(), ruleSource.source(), true); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } @@ -152,7 +152,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, final RuleDao ruleDao = ruleService.load(id); final Rule rule; try { - rule = pipelineRuleParser.parseRule(update.source(), false); + rule = pipelineRuleParser.parseRule(id, update.source(), false); } catch (ParseException e) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java index a83cb43306e6..94d45c1c3a0c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -90,7 +90,7 @@ public static RuleSource create(@JsonProperty("id") @Id @ObjectId @Nullable Stri public static RuleSource fromDao(PipelineRuleParser parser, RuleDao dao) { Set errors = null; try { - parser.parseRule(dao.source(), false); + parser.parseRule(dao.id(), dao.source(), false); } catch (ParseException e) { errors = e.getErrors(); diff --git a/src/web/pipelines/Pipeline.css b/src/web/pipelines/Pipeline.css index e7c44ac60d0a..282078ce9fc6 100644 --- a/src/web/pipelines/Pipeline.css +++ b/src/web/pipelines/Pipeline.css @@ -4,7 +4,7 @@ dl.pipeline-dl > dt { text-align: left; - width: 90px; + width: 140px; } dl.pipeline-dl > dt:after { diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx index 9df73d08fce3..5491013cb285 100644 --- a/src/web/pipelines/Pipeline.jsx +++ b/src/web/pipelines/Pipeline.jsx @@ -48,7 +48,7 @@ const Pipeline = React.createClass({ _formatStage(stage, maxStage) { return ( - ); }, diff --git a/src/web/pipelines/PipelineDetails.jsx b/src/web/pipelines/PipelineDetails.jsx index f47b2ed0950e..d54737958275 100644 --- a/src/web/pipelines/PipelineDetails.jsx +++ b/src/web/pipelines/PipelineDetails.jsx @@ -4,6 +4,8 @@ import { Row, Col } from 'react-bootstrap'; import { Timestamp } from 'components/common'; import PipelineForm from './PipelineForm'; +import { MetricContainer, CounterRate } from 'components/metrics'; + const PipelineDetails = React.createClass({ propTypes: { pipeline: React.PropTypes.object, @@ -34,6 +36,12 @@ const PipelineDetails = React.createClass({
    Last modified
    +
    Current throughput
    +
    + + + +
    diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 06ce31ebb50f..15daef3762ad 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -5,6 +5,7 @@ import { LinkContainer } from 'react-router-bootstrap'; import naturalSort from 'javascript-natural-sort'; import { DataTable, Spinner } from 'components/common'; +import { MetricContainer, CounterRate } from 'components/metrics'; import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; @@ -67,6 +68,10 @@ const ProcessingTimelineComponent = React.createClass({ {pipeline.title}
    {pipeline.description} +
    + + + {this._formatStages(pipeline, pipeline.stages)} @@ -114,7 +119,7 @@ const ProcessingTimelineComponent = React.createClass({ this.usedStages = this._calculateUsedStages(this.state.pipelines); - const headers = ['Pipeline', 'ProcessingTimeline', 'Actions']; + const headers = ['Pipeline', 'Processing Timeline', 'Actions']; return (
    {addNewPipelineButton} diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 449789324273..c52f9b6d662c 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -6,10 +6,12 @@ import { LinkContainer } from 'react-router-bootstrap'; import { DataTable, EntityListItem, Spinner } from 'components/common'; import RulesStore from 'rules/RulesStore'; import StageForm from './StageForm'; +import { MetricContainer, CounterRate } from 'components/metrics'; const Stage = React.createClass({ propTypes: { stage: PropTypes.object.isRequired, + pipeline: PropTypes.object.isRequired, isLastStage: PropTypes.bool, onUpdate: PropTypes.func.isRequired, onDelete: PropTypes.func.isRequired, @@ -20,7 +22,7 @@ const Stage = React.createClass({ return {header}; }, - _ruleRowFormatter(rule) { + _ruleRowFormatter(stage, rule) { return ( @@ -29,12 +31,22 @@ const Stage = React.createClass({ {rule.description} + + + + + + + + + + ); }, - _formatRules(rules) { - const headers = ['Title', 'Description']; + _formatRules(stage, rules) { + const headers = ['Title', 'Description', 'Metrics', 'Errors']; return ( this._ruleRowFormatter(stage, rule)} noDataText="This stage has no rules yet. Click on edit to add some." filterLabel="" filterKeys={[]} /> @@ -52,7 +64,11 @@ const Stage = React.createClass({ render() { const stage = this.props.stage; - const suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules`)}`; + let suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules`)}`; + + const throughput = ( + + ); const actions = [ , + , , ]; diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx index 13f83a1dca78..63cd8aa25f6d 100644 --- a/src/web/rules/Rule.jsx +++ b/src/web/rules/Rule.jsx @@ -29,7 +29,7 @@ const Rule = React.createClass({ return (
    - + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list{' '} of actions.{' '} diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 7165e20b071d..bff9f1b490fc 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -24,7 +24,7 @@ const RulesPage = React.createClass({ render() { return ( - + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list of actions. Graylog evaluates the condition against a message and executes the actions if the condition is satisfied. From 5fb34d5778f06aed7f6f3cc5e2f154842891527f Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Thu, 14 Apr 2016 13:25:34 +0200 Subject: [PATCH 230/521] Upgrade to frontend-maven-plugin 1.0, node.js 4.4.3 LTS, and npm 3.8.6 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 242589297a66..64dd24f1eabc 100644 --- a/pom.xml +++ b/pom.xml @@ -384,7 +384,7 @@ com.github.eirslett frontend-maven-plugin - 0.0.29 + 1.0 @@ -393,8 +393,8 @@ install-node-and-npm - v4.3.1 - 3.7.3 + v4.4.3 + 3.8.6 From 6f84a0d9fbcae36668a97b1a94386f1a6f553b39 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 19 Apr 2016 18:45:47 +0200 Subject: [PATCH 231/521] Adapt to changes in PageHeader --- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 2 +- src/web/pipelines/PipelineDetailsPage.jsx | 2 +- src/web/pipelines/PipelinesOverviewPage.jsx | 2 +- src/web/rules/Rule.jsx | 2 +- src/web/rules/RulesPage.jsx | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 504b152381ac..9b3cfbef7217 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -61,7 +61,7 @@ const PipelineConnectionsPage = React.createClass({ } return ( - + Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} as input for the different pipelines you configure. diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index a06be2bc4e14..6a72bd972f6c 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -91,7 +91,7 @@ const PipelineDetailsPage = React.createClass({ return (
    - + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} rules are evaluated and applied. Messages can go through one or more stages. diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx index 1b2dde42b13f..d62d3bcc4f20 100644 --- a/src/web/pipelines/PipelinesOverviewPage.jsx +++ b/src/web/pipelines/PipelinesOverviewPage.jsx @@ -9,7 +9,7 @@ const PipelinesOverviewPage = React.createClass({ render() { return (
    - + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} rules are evaluated and applied. Messages can go through one or more stages. diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx index 63cd8aa25f6d..99d8da09493b 100644 --- a/src/web/rules/Rule.jsx +++ b/src/web/rules/Rule.jsx @@ -29,7 +29,7 @@ const Rule = React.createClass({ return (
    - + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list{' '} of actions.{' '} diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index bff9f1b490fc..61d2209d573a 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -24,7 +24,7 @@ const RulesPage = React.createClass({ render() { return ( - + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list of actions. Graylog evaluates the condition against a message and executes the actions if the condition is satisfied. From bd8dde3c7e73d4ed4dc2fcf9088c19c29cac6970 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 20 Apr 2016 14:44:17 +0200 Subject: [PATCH 232/521] when a rule could not be resolved, fall back to the rule reference name without a link DataTable and DataTableElement pass the current iteration index into the header and row formatter functions now (as the second parameter) This allows formatters to pick render the number or to pick an alternative from a secondary source fixes Graylog2/graylog-plugin-pipeline-processor#11 --- src/web/pipelines/Stage.jsx | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index a138656dd9be..1c64eedc050e 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -22,13 +22,28 @@ const Stage = React.createClass({ return {header}; }, - _ruleRowFormatter(stage, rule) { + _ruleRowFormatter(stage, ruleArg, ruleIdx) { + let rule = ruleArg; + + let ruleTitle; + // this can happen when a rule has been renamed, but not all references are updated + if (!rule) { + rule = { + id: `invalid-${ruleIdx}`, + description: `Rule ${stage.rules[ruleIdx]} has been renamed or removed. This rule will be skipped.`, + }; + ruleTitle = {stage.rules[ruleIdx]}; + } else { + ruleTitle = ( + {rule.title} + + ); + + } return ( - - {rule.title} - + {ruleTitle} {rule.description} @@ -54,7 +69,7 @@ const Stage = React.createClass({ headers={headers} headerCellFormatter={this._ruleHeaderFormatter} rows={rules} - dataRowFormatter={(rule) => this._ruleRowFormatter(stage, rule)} + dataRowFormatter={(rule, i) => this._ruleRowFormatter(stage, rule, i)} noDataText="This stage has no rules yet. Click on edit to add some." filterLabel="" filterKeys={[]} /> From a1522179ba7ebd55fc9b58dd6091c48ed35ee43d Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 20 Apr 2016 16:12:21 +0200 Subject: [PATCH 233/521] require authenticated user in rest resources issue #14 --- .../pipelineprocessor/rest/PipelineConnectionsResource.java | 2 ++ .../plugins/pipelineprocessor/rest/PipelineResource.java | 2 ++ .../graylog/plugins/pipelineprocessor/rest/RuleResource.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index 6224710ecb18..872c3a2c50a5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -20,6 +20,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent; @@ -45,6 +46,7 @@ @Path("/system/pipelines/connections") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication public class PipelineConnectionsResource extends RestResource implements PluginRestResource { private final PipelineStreamConnectionsService connectionsService; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index a523635605d2..2cf74f147b3f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -21,6 +21,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.db.PipelineService; @@ -55,6 +56,7 @@ @Path("/system/pipelines/pipeline") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication public class PipelineResource extends RestResource implements PluginRestResource { private static final Logger log = LoggerFactory.getLogger(PipelineResource.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 1382718fc512..f261259586ff 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -20,6 +20,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; @@ -54,6 +55,7 @@ @Path("/system/pipelines/rule") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication public class RuleResource extends RestResource implements PluginRestResource { private static final Logger log = LoggerFactory.getLogger(RuleResource.class); From 8db1bece4a66edee3de5ff817b40ba8acfeeedfc Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 21 Apr 2016 14:35:33 +0200 Subject: [PATCH 234/521] add experimental warning to overview page --- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 9b3cfbef7217..1b8747040417 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -64,7 +64,8 @@ const PipelineConnectionsPage = React.createClass({ Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} - as input for the different pipelines you configure. + as input for the different pipelines you configure.

    + This feature of Graylog is new and should be considered experimental. There may be missing functionality or incompatible changes in the future.     Read more about Graylog pipelines in the . From ac73447e9c50114919be4ac7fa196f96bdcea003 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Thu, 21 Apr 2016 14:52:54 +0200 Subject: [PATCH 235/521] Add REST API authentication and permissions (#15) Fixes #14 --- .../PipelineProcessorModule.java | 2 + .../rest/PipelineConnectionsResource.java | 49 +++++++++++++++-- .../rest/PipelineResource.java | 11 +++- .../rest/PipelineRestPermissions.java | 53 +++++++++++++++++++ .../pipelineprocessor/rest/RuleResource.java | 8 +++ 5 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 0043f838a7ca..31efdc13929d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -20,6 +20,7 @@ import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.rest.PipelineConnectionsResource; import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineRestPermissions; import org.graylog.plugins.pipelineprocessor.rest.RuleResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; @@ -40,6 +41,7 @@ protected void configure() { addRestResource(RuleResource.class); addRestResource(PipelineResource.class); addRestResource(PipelineConnectionsResource.class); + addPermissions(PipelineRestPermissions.class); install(new ProcessorFunctionsModule()); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java index 872c3a2c50a5..031cce09677e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -16,11 +16,13 @@ */ package org.graylog.plugins.pipelineprocessor.rest; +import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent; @@ -28,6 +30,7 @@ import org.graylog2.events.ClusterEventBus; import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.shared.security.RestPermissions; import org.graylog2.streams.StreamService; import javax.inject.Inject; @@ -41,6 +44,7 @@ import javax.ws.rs.core.MediaType; import java.util.Collections; import java.util.Set; +import java.util.stream.Collectors; @Api(value = "Pipelines/Connections", description = "Stream connections of processing pipelines") @Path("/system/pipelines/connections") @@ -67,14 +71,17 @@ public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsS @ApiOperation(value = "Connect processing pipelines to a stream", notes = "") @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_EDIT) public PipelineConnections connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineConnections connection) throws NotFoundException { final String streamId = connection.streamId(); // the default stream doesn't exist as an entity if (!streamId.equalsIgnoreCase("default")) { + checkPermission(RestPermissions.STREAMS_READ, streamId); streamService.load(streamId); } // verify the pipelines exist for (String s : connection.pipelineIds()) { + checkPermission(PipelineRestPermissions.PIPELINE_READ, s); pipelineService.load(s); } final PipelineConnections save = connectionsService.save(connection); @@ -85,19 +92,51 @@ public PipelineConnections connectPipelines(@ApiParam(name = "Json body", requir @ApiOperation("Get pipeline connections for the given stream") @GET @Path("/{streamId}") + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_READ) public PipelineConnections getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { - return connectionsService.load(streamId); + // the user needs to at least be able to read the stream + checkPermission(RestPermissions.STREAMS_READ, streamId); + + final PipelineConnections connections = connectionsService.load(streamId); + // filter out all pipelines the user does not have enough permissions to see + return PipelineConnections.create( + connections.id(), + connections.streamId(), + connections.pipelineIds() + .stream() + .filter(id -> isPermitted(PipelineRestPermissions.PIPELINE_READ, id)) + .collect(Collectors.toSet()) + ); } @ApiOperation("Get all pipeline connections") @GET + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_READ) public Set getAll() throws NotFoundException { - Set pipelineConnections = connectionsService.loadAll(); + final Set pipelineConnections = connectionsService.loadAll(); + + final Set filteredConnections = Sets.newHashSetWithExpectedSize(pipelineConnections.size()); + for (PipelineConnections pc : pipelineConnections) { + // only include the streams the user can see + if (isPermitted(RestPermissions.STREAMS_READ, pc.streamId())) { + // filter out all pipelines the user does not have enough permissions to see + filteredConnections.add(PipelineConnections.create( + pc.id(), + pc.streamId(), + pc.pipelineIds() + .stream() + .filter(id -> isPermitted(PipelineRestPermissions.PIPELINE_READ, id)) + .collect(Collectors.toSet())) + ); + } + } + + // to simplify clients, we always return the default stream, until we have it as a true entity - if (!pipelineConnections.stream().anyMatch(pc -> pc.streamId().equals("default"))) { - pipelineConnections.add(PipelineConnections.create(null, "default", Collections.emptySet())); + if (!filteredConnections.stream().anyMatch(pc -> pc.streamId().equals("default"))) { + filteredConnections.add(PipelineConnections.create(null, "default", Collections.emptySet())); } - return pipelineConnections; + return filteredConnections; } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java index 2cf74f147b3f..bb9d9270558c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.db.PipelineService; @@ -76,6 +77,7 @@ public PipelineResource(PipelineService pipelineService, @ApiOperation(value = "Create a processing pipeline from source", notes = "") @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CREATE) public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { final Pipeline pipeline; try { @@ -121,7 +123,9 @@ public Collection getAll() { final Collection daos = pipelineService.loadAll(); final ArrayList results = Lists.newArrayList(); for (PipelineDao dao : daos) { - results.add(PipelineSource.fromDao(pipelineRuleParser, dao)); + if (isPermitted(PipelineRestPermissions.PIPELINE_READ, dao.id())) { + results.add(PipelineSource.fromDao(pipelineRuleParser, dao)); + } } return results; @@ -131,6 +135,7 @@ public Collection getAll() { @Path("/{id}") @GET public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_READ, id); final PipelineDao dao = pipelineService.load(id); return PipelineSource.fromDao(pipelineRuleParser, dao); } @@ -140,6 +145,8 @@ public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) thr @PUT public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_EDIT, id); + final PipelineDao dao = pipelineService.load(id); final Pipeline pipeline; try { @@ -163,6 +170,8 @@ public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, @Path("/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_DELETE, id); + pipelineService.load(id); pipelineService.delete(id); clusterBus.post(PipelinesChangedEvent.deletedPipelineId(id)); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java new file mode 100644 index 000000000000..96a3330c990e --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java @@ -0,0 +1,53 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.collect.ImmutableSet; +import org.graylog2.plugin.security.Permission; +import org.graylog2.plugin.security.PluginPermissions; + +import java.util.Collections; +import java.util.Set; + +import static org.graylog2.plugin.security.Permission.create; + +public class PipelineRestPermissions implements PluginPermissions { + + /* pipelines */ + public static final String PIPELINE_CREATE = "pipeline:create"; + public static final String PIPELINE_READ = "pipeline:read"; + public static final String PIPELINE_EDIT = "pipeline:edit"; + public static final String PIPELINE_DELETE = "pipeline:delete"; + + /* rules */ + public static final String PIPELINE_RULE_CREATE = "pipeline_rule:create"; + public static final String PIPELINE_RULE_READ = "pipeline_rule:read"; + public static final String PIPELINE_RULE_EDIT = "pipeline_rule:edit"; + public static final String PIPELINE_RULE_DELETE = "pipeline_rule:delete"; + + /* connections */ + public static final String PIPELINE_CONNECTION_READ = "pipeline_connection:read"; + public static final String PIPELINE_CONNECTION_EDIT = "pipeline_connection:edit"; + + + @Override + public Set permissions() { + return ImmutableSet.of( + create(PIPELINE_CREATE, "Create new processing pipeline"), + create(PIPELINE_READ, "Read a processing pipeline"), + create(PIPELINE_EDIT, "Update a processing pipeline"), + create(PIPELINE_DELETE, "Delete a processing pipeline"), + + create(PIPELINE_RULE_CREATE, "Create new processing rule"), + create(PIPELINE_RULE_READ, "Read a processing rule"), + create(PIPELINE_RULE_EDIT, "Update a processing rule"), + create(PIPELINE_RULE_DELETE, "Delete a processing rule"), + + create(PIPELINE_CONNECTION_READ, "Read a pipeline stream connection"), + create(PIPELINE_CONNECTION_EDIT, "Update a pipeline stream connections") + ); + } + + @Override + public Set readerBasePermissions() { + return Collections.emptySet(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index f261259586ff..0cae111382e1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.pipelineprocessor.ast.Rule; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; @@ -76,6 +77,7 @@ public RuleResource(RuleService ruleService, @ApiOperation(value = "Create a processing rule from source", notes = "") @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_CREATE) public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { final Rule rule; try { @@ -119,6 +121,7 @@ public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleS @ApiOperation(value = "Get all processing rules") @GET + @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_READ) public Collection getAll() { final Collection ruleDaos = ruleService.loadAll(); return ruleDaos.stream() @@ -130,6 +133,7 @@ public Collection getAll() { @Path("/{id}") @GET public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_RULE_READ, id); return RuleSource.fromDao(pipelineRuleParser, ruleService.load(id)); } @@ -141,6 +145,7 @@ public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) return ruleDaos.stream() .map(ruleDao -> RuleSource.fromDao(pipelineRuleParser, ruleDao)) + .filter(rule -> isPermitted(PipelineRestPermissions.PIPELINE_RULE_READ, rule.id())) .collect(Collectors.toList()); } @@ -149,6 +154,8 @@ public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) @PUT public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_RULE_EDIT, id); + final RuleDao ruleDao = ruleService.load(id); final Rule rule; try { @@ -174,6 +181,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, @Path("/{id}") @DELETE public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_RULE_DELETE, id); ruleService.load(id); ruleService.delete(id); From 7ed3e9357eae42d0d94ec6282d14f5c87725c42b Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 21 Apr 2016 15:15:47 +0200 Subject: [PATCH 236/521] Add experimental label to pipeline pages --- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 4 ++-- src/web/pipelines/PipelineDetailsPage.jsx | 2 +- src/web/pipelines/PipelinesOverviewPage.jsx | 2 +- src/web/rules/Rule.jsx | 2 +- src/web/rules/RulesPage.jsx | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 1b8747040417..d63f89199e8f 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -1,4 +1,4 @@ -import React, { PropTypes } from 'react'; +import React from 'react'; import Reflux from 'reflux'; import { Button, Row, Col } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; @@ -61,7 +61,7 @@ const PipelineConnectionsPage = React.createClass({ } return ( - + Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} as input for the different pipelines you configure.

    diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index 6a72bd972f6c..5aa7d65b5dd7 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -91,7 +91,7 @@ const PipelineDetailsPage = React.createClass({ return (
    - + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} rules are evaluated and applied. Messages can go through one or more stages. diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx index d62d3bcc4f20..1a16547e3261 100644 --- a/src/web/pipelines/PipelinesOverviewPage.jsx +++ b/src/web/pipelines/PipelinesOverviewPage.jsx @@ -9,7 +9,7 @@ const PipelinesOverviewPage = React.createClass({ render() { return (
    - + Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where{' '} rules are evaluated and applied. Messages can go through one or more stages. diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx index 99d8da09493b..355d9da52e30 100644 --- a/src/web/rules/Rule.jsx +++ b/src/web/rules/Rule.jsx @@ -29,7 +29,7 @@ const Rule = React.createClass({ return (
    - + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list{' '} of actions.{' '} diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 61d2209d573a..58642af23c97 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -24,7 +24,7 @@ const RulesPage = React.createClass({ render() { return ( - + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list of actions. Graylog evaluates the condition against a message and executes the actions if the condition is satisfied. From 58f791d1d3841ab7dbf42ac7f373912bcfca97b5 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 21 Apr 2016 15:34:05 +0200 Subject: [PATCH 237/521] Use PageHeader to show experimental information --- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index d63f89199e8f..643c61677f8f 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -64,8 +64,7 @@ const PipelineConnectionsPage = React.createClass({ Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '} - as input for the different pipelines you configure.

    - This feature of Graylog is new and should be considered experimental. There may be missing functionality or incompatible changes in the future. + as input for the different pipelines you configure.     Read more about Graylog pipelines in the . From 6d037c85d82939fd3baa969892631099a708015c Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Thu, 21 Apr 2016 16:30:06 +0200 Subject: [PATCH 238/521] Bump Graylog server version to 2.0.0-rc.1 --- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 64dd24f1eabc..daae1e7e19b9 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-beta.4-SNAPSHOT + 2.0.0-rc.1 /usr/share/graylog-server/plugin 4.5.1 1.7.13 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index 0d2d7947b6f3..da16f6de414b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 0, 0, "alpha.8"); + return new Version(1, 0, 0, "beta.1"); } @Override From dcc3ae4eb3657132ce5f99b67dd4583c507f0cc3 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 21 Apr 2016 14:34:52 +0000 Subject: [PATCH 239/521] [graylog-plugin-pipeline-processor-release] prepare release 1.0.0-beta.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index daae1e7e19b9..440fde175174 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-alpha.9-SNAPSHOT + 1.0.0-beta.1 jar ${project.artifactId} From 25877ce374c556e2c1538cde8911e2816def67e6 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 21 Apr 2016 14:35:13 +0000 Subject: [PATCH 240/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 440fde175174..5f5896e3fdaf 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-beta.1 + 1.0.0-beta.2-SNAPSHOT jar ${project.artifactId} From 0ec7031d1dab4987646c5b6cbe1cdd074ca17a92 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Thu, 21 Apr 2016 17:13:15 +0200 Subject: [PATCH 241/521] Bump Graylog server dependency to 2.0.0-rc.2-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f5896e3fdaf..ab8b700ec386 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-rc.1 + 2.0.0-rc.2-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From b79d561faab6d172ce2830c4cc7cd0aaf41c29dc Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Tue, 26 Apr 2016 15:38:15 +0200 Subject: [PATCH 242/521] Bump Graylog server version to 2.0.0 --- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ab8b700ec386..ae4285ee0fed 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0-rc.2-SNAPSHOT + 2.0.0 /usr/share/graylog-server/plugin 4.5.1 1.7.13 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index da16f6de414b..6814e8df0541 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 0, 0, "beta.1"); + return new Version(1, 0, 0, "beta.2"); } @Override From 9527eb84755dc2fe79aa5d8b71a061c1dee84867 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Tue, 26 Apr 2016 13:45:43 +0000 Subject: [PATCH 243/521] [graylog-plugin-pipeline-processor-release] prepare release 1.0.0-beta.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae4285ee0fed..69d746449611 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-beta.2-SNAPSHOT + 1.0.0-beta.2 jar ${project.artifactId} From df23bd7568f989bd5ed25a3572e9d644961cf5df Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Tue, 26 Apr 2016 13:45:58 +0000 Subject: [PATCH 244/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 69d746449611..5d8b0086dfb0 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-beta.2 + 1.0.0-beta.3-SNAPSHOT jar ${project.artifactId} From 808856c87cd424bac6668ba54856d2f743bbed56 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Mon, 2 May 2016 14:42:03 +0200 Subject: [PATCH 245/521] Bump Graylog server version to 2.0.1-SNAPSHO --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d8b0086dfb0..43ed483b2f03 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ true true true - 2.0.0 + 2.0.1-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From 76c364847e82c29274e123bd722234f99637854a Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Tue, 3 May 2016 15:31:01 +0200 Subject: [PATCH 246/521] Bump version to 1.1.0-beta.3-SNAPSHOT and graylog server to 2.1.0-SNAPSHOT --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 43ed483b2f03..206a268d61d3 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.0.0-beta.3-SNAPSHOT + 1.1.0-beta.3-SNAPSHOT jar ${project.artifactId} @@ -39,7 +39,7 @@ true true true - 2.0.1-SNAPSHOT + 2.1.0-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From c5d406b63d66b2543e48ec3b436a6e08bf05a0f4 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 4 May 2016 11:35:42 +0200 Subject: [PATCH 247/521] add issue template --- .github/ISSUE_TEMPLATE.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000000..9e8a64bb0017 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,13 @@ +### Problem description + +### Steps to reproduce the problem + +1. ... + +### Environment + +* Graylog Version: +* Elasticsearch Version: +* MongoDB Version: +* Operating System: +* Browser version: From db7185c9a8bdeab6ff989fac1160a41cd95064b0 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 4 May 2016 11:38:03 +0200 Subject: [PATCH 248/521] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 9e8a64bb0017..69cf48e4143d 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -7,6 +7,7 @@ ### Environment * Graylog Version: +* Pipeline Processor plugin version: * Elasticsearch Version: * MongoDB Version: * Operating System: From 1ba3c6343f1cbf22e56ca36a5301adea12f580f2 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 9 May 2016 12:30:00 +0200 Subject: [PATCH 249/521] Bind the GrokMatch function so it is available in rules (#26) Fixes #24 (cherry picked from commit 446826c3f2b7cd79760b1a942528884987f6a8a9) --- .../pipelineprocessor/functions/ProcessorFunctionsModule.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 7cdcf105e084..0dfc22302484 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -50,6 +50,7 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; +import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; @@ -83,6 +84,7 @@ protected void configure() { // generic functions addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class); + addMessageProcessorFunction(GrokMatch.NAME, GrokMatch.class); // string functions addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class); From 2055a87fdd228fcdd5290369412c5174cf8c4091 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Tue, 17 May 2016 12:00:10 +0200 Subject: [PATCH 250/521] Add syslog-related functions (#19) --- .../syslog/SyslogFacilityConversion.java | 50 ++++++++ .../syslog/SyslogLevelConversion.java | 50 ++++++++ .../functions/syslog/SyslogPriority.java | 30 +++++ .../syslog/SyslogPriorityAsString.java | 30 +++++ .../syslog/SyslogPriorityConversion.java | 50 ++++++++ .../SyslogPriorityToStringConversion.java | 52 ++++++++ .../functions/syslog/SyslogUtils.java | 119 ++++++++++++++++++ .../rest/PipelineRestPermissions.java | 16 +++ .../functions/FunctionsSnippetsTest.java | 61 +++++++++ .../parser/PrecedenceTest.java | 16 +++ .../pipelineprocessor/functions/syslog.txt | 50 ++++++++ 11 files changed, 524 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java new file mode 100644 index 000000000000..54fae2afadbd --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java @@ -0,0 +1,50 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.common.primitives.Ints; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogFacilityConversion extends AbstractFunction { + public static final String NAME = "syslog_facility"; + + private final ParameterDescriptor valueParam = object("value").build(); + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final Integer facility = firstNonNull(Ints.tryParse(s), -1); + + return SyslogUtils.facilityToString(facility); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(valueParam) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java new file mode 100644 index 000000000000..c60044bbe365 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java @@ -0,0 +1,50 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.common.primitives.Ints; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogLevelConversion extends AbstractFunction { + public static final String NAME = "syslog_level"; + + private final ParameterDescriptor valueParam = object("value").build(); + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final Integer level = firstNonNull(Ints.tryParse(s), -1); + + return SyslogUtils.levelToString(level); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(valueParam) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java new file mode 100644 index 000000000000..07f2fd94200b --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java @@ -0,0 +1,30 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class SyslogPriority { + public abstract int getLevel(); + + public abstract int getFacility(); + + public static SyslogPriority create(int level, int facility) { + return new AutoValue_SyslogPriority(level, facility); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java new file mode 100644 index 000000000000..387fe1743600 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java @@ -0,0 +1,30 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class SyslogPriorityAsString { + public abstract String getLevel(); + + public abstract String getFacility(); + + public static SyslogPriorityAsString create(String level, String facility) { + return new AutoValue_SyslogPriorityAsString(level, facility); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java new file mode 100644 index 000000000000..3337bb70594d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java @@ -0,0 +1,50 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogPriorityConversion extends AbstractFunction { + public static final String NAME = "expand_syslog_priority"; + + private final ParameterDescriptor valueParam = object("value").build(); + + @Override + public SyslogPriority evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final int priority = Integer.parseInt(s); + final int facility = SyslogUtils.facilityFromPriority(priority); + final int level = SyslogUtils.levelFromPriority(priority); + + return SyslogPriority.create(level, facility); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(SyslogPriority.class) + .params(valueParam) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java new file mode 100644 index 000000000000..8da7dcfc93b9 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java @@ -0,0 +1,52 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogPriorityToStringConversion extends AbstractFunction { + public static final String NAME = "expand_syslog_priority_as_string"; + + private final ParameterDescriptor valueParam = object("value").build(); + + @Override + public SyslogPriorityAsString evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final int priority = Integer.parseInt(s); + final int facility = SyslogUtils.facilityFromPriority(priority); + final String facilityString = SyslogUtils.facilityToString(facility); + final int level = SyslogUtils.levelFromPriority(priority); + final String levelString = SyslogUtils.levelToString(level); + + return SyslogPriorityAsString.create(levelString, facilityString); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(SyslogPriorityAsString.class) + .params(valueParam) + .build(); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java new file mode 100644 index 000000000000..94490c863284 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java @@ -0,0 +1,119 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +public final class SyslogUtils { + /** + * Converts integer syslog loglevel to human readable string + * + * @param level The level to convert + * @return The human readable level + * @see RFC 5424, Section 6.2.1 + */ + public static String levelToString(int level) { + switch (level) { + case 0: + return "Emergency"; + case 1: + return "Alert"; + case 2: + return "Critical"; + case 3: + return "Error"; + case 4: + return "Warning"; + case 5: + return "Notice"; + case 6: + return "Informational"; + case 7: + return "Debug"; + } + + return "Unknown"; + } + + /** + * Converts integer syslog facility to human readable string + * + * @param facility The facility to convert + * @return The human readable facility + * @see RFC 5424, Section 6.2.1 + */ + public static String facilityToString(int facility) { + switch (facility) { + case 0: + return "kern"; + case 1: + return "user"; + case 2: + return "mail"; + case 3: + return "daemon"; + case 4: + return "auth"; + case 5: + return "syslog"; + case 6: + return "lpr"; + case 7: + return "news"; + case 8: + return "uucp"; + case 9: + return "clock"; + case 10: + return "authpriv"; + case 11: + return "ftp"; + case 12: + return "ntp"; + case 13: + return "log audit"; + case 14: + return "log alert"; + case 15: + return "cron"; + case 16: + return "local0"; + case 17: + return "local1"; + case 18: + return "local2"; + case 19: + return "local3"; + case 20: + return "local4"; + case 21: + return "local5"; + case 22: + return "local6"; + case 23: + return "local7"; + default: + return "Unknown"; + } + } + + public static int levelFromPriority(int priority) { + return priority - (facilityFromPriority(priority) << 3); + } + + public static int facilityFromPriority(int priority) { + return priority >> 3; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java index 96a3330c990e..255869da5240 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import com.google.common.collect.ImmutableSet; diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 3d5e410177ee..75bafda357cf 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -62,6 +62,10 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Swapcase; import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogFacilityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogLevelConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityToStringConversion; import org.graylog.plugins.pipelineprocessor.functions.urls.UrlConversion; import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.ParseException; @@ -164,6 +168,11 @@ public static void registerFunctions() { functions.put(IsNull.NAME, new IsNull()); functions.put(IsNotNull.NAME, new IsNotNull()); + functions.put(SyslogPriorityConversion.NAME, new SyslogPriorityConversion()); + functions.put(SyslogPriorityToStringConversion.NAME, new SyslogPriorityToStringConversion()); + functions.put(SyslogFacilityConversion.NAME, new SyslogFacilityConversion()); + functions.put(SyslogLevelConversion.NAME, new SyslogLevelConversion()); + functions.put(UrlConversion.NAME, new UrlConversion()); final GrokPatternService grokPatternService = mock(GrokPatternService.class); @@ -378,4 +387,56 @@ public void urls() { assertThat(message.getField("equal")).isEqualTo("can=containanotherone"); assertThat(message.getField("authority")).isEqualTo("admin:s3cr31@some.host.with.lots.of.subdomains.com:9999"); } + + @Test + public void syslog() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + + assertThat(message.getField("level0")).isEqualTo("Emergency"); + assertThat(message.getField("level1")).isEqualTo("Alert"); + assertThat(message.getField("level2")).isEqualTo("Critical"); + assertThat(message.getField("level3")).isEqualTo("Error"); + assertThat(message.getField("level4")).isEqualTo("Warning"); + assertThat(message.getField("level5")).isEqualTo("Notice"); + assertThat(message.getField("level6")).isEqualTo("Informational"); + assertThat(message.getField("level7")).isEqualTo("Debug"); + + assertThat(message.getField("facility0")).isEqualTo("kern"); + assertThat(message.getField("facility1")).isEqualTo("user"); + assertThat(message.getField("facility2")).isEqualTo("mail"); + assertThat(message.getField("facility3")).isEqualTo("daemon"); + assertThat(message.getField("facility4")).isEqualTo("auth"); + assertThat(message.getField("facility5")).isEqualTo("syslog"); + assertThat(message.getField("facility6")).isEqualTo("lpr"); + assertThat(message.getField("facility7")).isEqualTo("news"); + assertThat(message.getField("facility8")).isEqualTo("uucp"); + assertThat(message.getField("facility9")).isEqualTo("clock"); + assertThat(message.getField("facility10")).isEqualTo("authpriv"); + assertThat(message.getField("facility11")).isEqualTo("ftp"); + assertThat(message.getField("facility12")).isEqualTo("ntp"); + assertThat(message.getField("facility13")).isEqualTo("log audit"); + assertThat(message.getField("facility14")).isEqualTo("log alert"); + assertThat(message.getField("facility15")).isEqualTo("cron"); + assertThat(message.getField("facility16")).isEqualTo("local0"); + assertThat(message.getField("facility17")).isEqualTo("local1"); + assertThat(message.getField("facility18")).isEqualTo("local2"); + assertThat(message.getField("facility19")).isEqualTo("local3"); + assertThat(message.getField("facility20")).isEqualTo("local4"); + assertThat(message.getField("facility21")).isEqualTo("local5"); + assertThat(message.getField("facility22")).isEqualTo("local6"); + assertThat(message.getField("facility23")).isEqualTo("local7"); + + assertThat(message.getField("prio1_facility")).isEqualTo(0); + assertThat(message.getField("prio1_level")).isEqualTo(0); + assertThat(message.getField("prio2_facility")).isEqualTo(20); + assertThat(message.getField("prio2_level")).isEqualTo(5); + assertThat(message.getField("prio3_facility")).isEqualTo("kern"); + assertThat(message.getField("prio3_level")).isEqualTo("Emergency"); + assertThat(message.getField("prio4_facility")).isEqualTo("local4"); + assertThat(message.getField("prio4_level")).isEqualTo("Notice"); + } } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java index c290929fe3c2..be824a9004c9 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.parser; import org.graylog.plugins.pipelineprocessor.BaseParserTest; diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt new file mode 100644 index 000000000000..c013f59a88f3 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt @@ -0,0 +1,50 @@ +rule "syslog tests" +when + true +then + set_field("level0", syslog_level(0)); + set_field("level1", syslog_level(1)); + set_field("level2", syslog_level(2)); + set_field("level3", syslog_level(3)); + set_field("level4", syslog_level(4)); + set_field("level5", syslog_level(5)); + set_field("level6", syslog_level(6)); + set_field("level7", syslog_level(7)); + + set_field("facility0", syslog_facility(0)); + set_field("facility1", syslog_facility(1)); + set_field("facility2", syslog_facility(2)); + set_field("facility3", syslog_facility(3)); + set_field("facility4", syslog_facility(4)); + set_field("facility5", syslog_facility(5)); + set_field("facility6", syslog_facility(6)); + set_field("facility7", syslog_facility(7)); + set_field("facility8", syslog_facility(8)); + set_field("facility9", syslog_facility(9)); + set_field("facility10", syslog_facility(10)); + set_field("facility11", syslog_facility(11)); + set_field("facility12", syslog_facility(12)); + set_field("facility13", syslog_facility(13)); + set_field("facility14", syslog_facility(14)); + set_field("facility15", syslog_facility(15)); + set_field("facility16", syslog_facility(16)); + set_field("facility17", syslog_facility(17)); + set_field("facility18", syslog_facility(18)); + set_field("facility19", syslog_facility(19)); + set_field("facility20", syslog_facility(20)); + set_field("facility21", syslog_facility(21)); + set_field("facility22", syslog_facility(22)); + set_field("facility23", syslog_facility(23)); + + let priority1 = expand_syslog_priority(0); + set_fields({prio1_facility: priority1.facility, prio1_level: priority1.level }); + let priority2 = expand_syslog_priority(165); + set_fields({prio2_facility: priority2.facility, prio2_level: priority2.level }); + + let priority3 = expand_syslog_priority_as_string(0); + set_fields({prio3_facility: priority3.facility, prio3_level: priority3.level }); + let priority4 = expand_syslog_priority_as_string(165); + set_fields({prio4_facility: priority4.facility, prio4_level: priority4.level }); + + trigger_test(); +end \ No newline at end of file From 4fcaf8dec0437e4ce02108f0cae1ff1ebb18d647 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Tue, 17 May 2016 12:00:27 +0200 Subject: [PATCH 251/521] Add concat() function (#20) --- .../functions/strings/Concat.java | 54 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 2 + .../pipelineprocessor/functions/strings.txt | 3 +- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java new file mode 100644 index 000000000000..2086fee45b61 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.base.Strings; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Concat extends AbstractFunction { + public static final String NAME = "concat"; + private final ParameterDescriptor firstParam; + private final ParameterDescriptor secondParam; + + public Concat() { + firstParam = ParameterDescriptor.string("first").build(); + secondParam = ParameterDescriptor.string("second").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String first = Strings.nullToEmpty(firstParam.required(args, context)); + final String second = Strings.nullToEmpty(secondParam.required(args, context)); + + return first.concat(second); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of(firstParam, secondParam)) + .build(); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 75bafda357cf..aae745e294f6 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -54,6 +54,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Concat; import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; @@ -137,6 +138,7 @@ public static void registerFunctions() { // string functions functions.put(Abbreviate.NAME, new Abbreviate()); functions.put(Capitalize.NAME, new Capitalize()); + functions.put(Concat.NAME, new Concat()); functions.put(Contains.NAME, new Contains()); functions.put(Lowercase.NAME, new Lowercase()); functions.put(Substring.NAME, new Substring()); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt index ee74d12c70b1..5c4c974883c7 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt @@ -13,7 +13,8 @@ when abbreviate("abcdefg", 6) == "abc..." && abbreviate("abcdefg", 7) == "abcdefg" && abbreviate("abcdefg", 8) == "abcdefg" && - abbreviate("abcdefg", 4) == "a..." + abbreviate("abcdefg", 4) == "a..." && + concat("foo", "bar") == "foobar" then set_field("has_xyz", contains("abcdef", "xyz")); trigger_test(); From adf53e23cea6cc26ac2b40ba924a87869fec200e Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 17 May 2016 12:07:34 +0200 Subject: [PATCH 252/521] add missing bindings for syslog functinos and concat (cherry picked from commit 20a5f056a57594cab24090a7e42c383fa5978123) --- .../functions/ProcessorFunctionsModule.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 0dfc22302484..1a9a3cea622f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -49,6 +49,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Concat; import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; @@ -57,6 +58,10 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Swapcase; import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogFacilityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogLevelConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityToStringConversion; import org.graylog.plugins.pipelineprocessor.functions.urls.UrlConversion; import org.graylog2.plugin.PluginModule; @@ -95,6 +100,7 @@ protected void configure() { addMessageProcessorFunction(Swapcase.NAME, Swapcase.class); addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); + addMessageProcessorFunction(Concat.NAME, Concat.class); // json addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); @@ -126,6 +132,12 @@ protected void configure() { // URL parsing addMessageProcessorFunction(UrlConversion.NAME, UrlConversion.class); + + // Syslog support + addMessageProcessorFunction(SyslogFacilityConversion.NAME, SyslogFacilityConversion.class); + addMessageProcessorFunction(SyslogLevelConversion.NAME, SyslogLevelConversion.class); + addMessageProcessorFunction(SyslogPriorityConversion.NAME, SyslogPriorityConversion.class); + addMessageProcessorFunction(SyslogPriorityToStringConversion.NAME, SyslogPriorityToStringConversion.class); } protected void addMessageProcessorFunction(String name, Class> functionClass) { From 90c00f150b447e146a51576b067bce70c022712f Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 25 May 2016 14:31:12 +0200 Subject: [PATCH 253/521] Change installation process in the readme Fixes #33 --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b8b789342fbb..5dc6a4bfe5df 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,17 @@ __Use this paragraph to enter a description of your plugin.__ Installation ------------ -[Download the plugin](https://github.com/graylog-plugin-pipeline-processor/releases) -and place the `.jar` file in your Graylog plugin directory. The plugin directory -is the `plugins/` folder relative from your `graylog-server` directory by default -and can be configured in your `graylog.conf` file. +This plugin is included by default in Graylog 2.0 tarballs and packages, +so you do not need to install it by hand. + +For now we do not provide individual releases of the plugin, but you can still +find it inside the Graylog tarball available in the [downloads page](https://www.graylog.org/download/). +The plugin is located in the `plugin` directory of the tarball. + +Once you get the `.jar` file from the tarball, place it in your Graylog plugin +directory. The plugin directory is the `plugins/` folder relative from your +`graylog-server` directory by default and can be configured in your +`graylog.conf` file. Restart `graylog-server` and you are done. From 9d46a02edb78d056fd7aad8b592835b0df767388 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 1 Jun 2016 15:53:11 +0200 Subject: [PATCH 254/521] Do not display streams without pipelines connected Fixes Graylog2/graylog2-server#2322 --- src/web/pipeline-connections/PipelineConnections.jsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index fecfc6c7e9c3..54a838689ce7 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -27,10 +27,14 @@ const PipelineConnections = React.createClass({ this.props.onConnectionsChange(newConnection, callback); }, + _isConnectionWithPipelines(connection) { + return connection.pipeline_ids.length > 0; + }, + render() { // TODO: Sort this list by stream title const formattedConnections = this.props.connections - .filter(c => this.state.filteredStreams.some(s => s.id === c.stream_id)) + .filter(c => this.state.filteredStreams.some(s => s.id === c.stream_id && this._isConnectionWithPipelines(c))) .map(c => { return ( s.id === c.stream_id)[0]} @@ -39,7 +43,7 @@ const PipelineConnections = React.createClass({ ); }); - const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id)); + const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id && this._isConnectionWithPipelines(c))); return (
    From e625d909e53e5a552e61dbe410c6702ba0a8f041 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 1 Jun 2016 15:53:46 +0200 Subject: [PATCH 255/521] Sort streams by title --- src/web/pipeline-connections/PipelineConnections.jsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index 54a838689ce7..88d9b6910a05 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -1,5 +1,6 @@ import React from 'react'; import { Row, Col } from 'react-bootstrap'; +import naturalSort from 'javascript-natural-sort'; import { EntityList, TypeAheadDataFilter } from 'components/common'; import Connection from './Connection'; @@ -32,9 +33,14 @@ const PipelineConnections = React.createClass({ }, render() { - // TODO: Sort this list by stream title + const filteredStreamsMap = {}; + this.state.filteredStreams.forEach(s => { + filteredStreamsMap[s.id] = s; + }); + const formattedConnections = this.props.connections .filter(c => this.state.filteredStreams.some(s => s.id === c.stream_id && this._isConnectionWithPipelines(c))) + .sort((c1, c2) => naturalSort(filteredStreamsMap[c1.stream_id].title, filteredStreamsMap[c2.stream_id].title)) .map(c => { return ( s.id === c.stream_id)[0]} From 6b35cec24e97486f84e6d3c61ed8d250dadfc539 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 9 Jun 2016 15:26:05 +0200 Subject: [PATCH 256/521] Pipeline simulator v1 (#34) * Add REST resource to simulate pipeline processing * Adapt default stream title and description Reflects better which messages are routed in there * Add first version of pipeline simulator UI * Move simulator message comparison to a component * Display errors simulating processing inline * Allow simulating messages in default stream * Change action buttons on simulation page * Display text if message is dropped in processing * Adapt to changes in server PR --- .../PipelineProcessorModule.java | 2 + .../rest/SimulationRequest.java | 45 +++++++ .../rest/SimulationResponse.java | 34 ++++++ .../rest/SimulatorResource.java | 80 +++++++++++++ src/web/index.jsx | 2 + src/web/pipeline-connections/Connection.jsx | 14 ++- .../PipelineConnectionsPage.jsx | 4 +- src/web/simulator/ProcessorSimulator.jsx | 58 +++++++++ src/web/simulator/SimulationPreview.css | 9 ++ src/web/simulator/SimulationPreview.jsx | 111 ++++++++++++++++++ src/web/simulator/SimulatorActions.js | 7 ++ src/web/simulator/SimulatorPage.jsx | 101 ++++++++++++++++ src/web/simulator/SimulatorStore.js | 31 +++++ 13 files changed, 491 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java create mode 100644 src/web/simulator/ProcessorSimulator.jsx create mode 100644 src/web/simulator/SimulationPreview.css create mode 100644 src/web/simulator/SimulationPreview.jsx create mode 100644 src/web/simulator/SimulatorActions.js create mode 100644 src/web/simulator/SimulatorPage.jsx create mode 100644 src/web/simulator/SimulatorStore.js diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 31efdc13929d..4ad14f0236fa 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -22,6 +22,7 @@ import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; import org.graylog.plugins.pipelineprocessor.rest.PipelineRestPermissions; import org.graylog.plugins.pipelineprocessor.rest.RuleResource; +import org.graylog.plugins.pipelineprocessor.rest.SimulatorResource; import org.graylog2.plugin.PluginConfigBean; import org.graylog2.plugin.PluginModule; @@ -41,6 +42,7 @@ protected void configure() { addRestResource(RuleResource.class); addRestResource(PipelineResource.class); addRestResource(PipelineConnectionsResource.class); + addRestResource(SimulatorResource.class); addPermissions(PipelineRestPermissions.class); install(new ProcessorFunctionsModule()); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java new file mode 100644 index 000000000000..a05f9422881a --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java @@ -0,0 +1,45 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +@AutoValue +@JsonAutoDetect +public abstract class SimulationRequest { + @JsonProperty + public abstract String streamId(); + + @JsonProperty + public abstract String index(); + + @JsonProperty + public abstract String messageId(); + + public static Builder builder() { + return new AutoValue_SimulationRequest.Builder(); + } + + @JsonCreator + public static SimulationRequest create (@JsonProperty("stream_id") String streamId, + @JsonProperty("index") String index, + @JsonProperty("message_id") String messageId) { + return builder() + .streamId(streamId) + .index(index) + .messageId(messageId) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract SimulationRequest build(); + + public abstract Builder streamId(String streamId); + + public abstract Builder index(String index); + + public abstract Builder messageId(String messageId); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java new file mode 100644 index 000000000000..43cdee61b729 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java @@ -0,0 +1,34 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; + +import java.util.List; + +@AutoValue +@JsonAutoDetect +public abstract class SimulationResponse { + @JsonProperty + public abstract List messages(); + + public static SimulationResponse.Builder builder() { + return new AutoValue_SimulationResponse.Builder(); + } + + @JsonCreator + public static SimulationResponse create (@JsonProperty("messages") List messages) { + return builder() + .messages(messages) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract SimulationResponse build(); + + public abstract SimulationResponse.Builder messages(List messages); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java new file mode 100644 index 000000000000..5fa776510b73 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -0,0 +1,80 @@ +package org.graylog.plugins.pipelineprocessor.rest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog2.database.NotFoundException; +import org.graylog2.indexer.messages.DocumentNotFoundException; +import org.graylog2.indexer.messages.Messages; +import org.graylog2.indexer.results.ResultMessage; +import org.graylog2.messageprocessors.OrderedMessageProcessors; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.messageprocessors.MessageProcessor; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; +import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.shared.security.RestPermissions; +import org.graylog2.streams.StreamService; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.ArrayList; +import java.util.List; + +@Api(value = "Pipelines/Simulator", description = "Simulate pipeline message processor") +@Path("/system/pipelines/simulate") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication +public class SimulatorResource extends RestResource implements PluginRestResource { + private final OrderedMessageProcessors orderedMessageProcessors; + private final Messages messages; + private final StreamService streamService; + + @Inject + public SimulatorResource(OrderedMessageProcessors orderedMessageProcessors, Messages messages, StreamService streamService) { + this.orderedMessageProcessors = orderedMessageProcessors; + this.messages = messages; + this.streamService = streamService; + } + + @ApiOperation(value = "Simulate the execution of the pipeline message processor") + @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_READ) + public SimulationResponse simulate(@ApiParam(name = "simulation", required = true) @NotNull SimulationRequest request) throws NotFoundException { + checkPermission(RestPermissions.MESSAGES_READ, request.messageId()); + checkPermission(RestPermissions.STREAMS_READ, request.streamId()); + try { + final ResultMessage resultMessage = messages.get(request.messageId(), request.index()); + final Message message = resultMessage.getMessage(); + if (!request.streamId().equals("default")) { + final Stream stream = streamService.load(request.streamId()); + message.addStream(stream); + } + + List simulationResults = new ArrayList<>(); + + for (MessageProcessor messageProcessor : orderedMessageProcessors) { + if (messageProcessor instanceof PipelineInterpreter) { + org.graylog2.plugin.Messages processedMessages = messageProcessor.process(message); + for (Message processedMessage : processedMessages) { + simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); + } + } + } + + return SimulationResponse.create(simulationResults); + } catch (DocumentNotFoundException e) { + throw new NotFoundException(e); + } + } +} diff --git a/src/web/index.jsx b/src/web/index.jsx index 648660784b11..ef349c062969 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -3,6 +3,7 @@ import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage'; import PipelineDetailsPage from 'pipelines/PipelineDetailsPage'; import PipelineConnectionsPage from 'pipeline-connections/PipelineConnectionsPage'; +import SimulatorPage from 'simulator/SimulatorPage'; import RulesPage from 'rules/RulesPage'; import RuleDetailsPage from 'rules/RuleDetailsPage'; @@ -12,6 +13,7 @@ PluginStore.register(new PluginManifest(packageJson, { { path: '/system/pipelines/overview', component: PipelinesOverviewPage }, { path: '/system/pipelines/rules', component: RulesPage }, { path: '/system/pipelines/rules/:ruleId', component: RuleDetailsPage }, + { path: '/system/pipelines/simulate/:streamId', component: SimulatorPage }, { path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage }, ], diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx index 6443b2d1b4b8..8643c330a4e4 100644 --- a/src/web/pipeline-connections/Connection.jsx +++ b/src/web/pipeline-connections/Connection.jsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Col } from 'react-bootstrap'; +import { Button, Col } from 'react-bootstrap'; import { LinkContainer } from 'react-router-bootstrap'; import { DataTable, EntityListItem, Timestamp } from 'components/common'; @@ -46,10 +46,14 @@ const Connection = React.createClass({ }, render() { - const actions = ( - - ); + const actions = [ + + + , + , + ]; const content = ( diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 643c61677f8f..71cbe32a6217 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -33,8 +33,8 @@ const PipelineConnectionsPage = React.createClass({ StreamsStore.listStreams().then((streams) => { streams.push({ id: 'default', - title: 'Incoming messages', - description: 'Default stream of all incoming messages.', + title: 'Default', + description: 'Stream used by default for messages not matching another stream.', }); this.setState({ streams }); }); diff --git a/src/web/simulator/ProcessorSimulator.jsx b/src/web/simulator/ProcessorSimulator.jsx new file mode 100644 index 000000000000..099de5a02acd --- /dev/null +++ b/src/web/simulator/ProcessorSimulator.jsx @@ -0,0 +1,58 @@ +import React from 'react'; +import { Col, Row } from 'react-bootstrap'; + +import LoaderTabs from 'components/messageloaders/LoaderTabs'; +import SimulationPreview from './SimulationPreview'; + +import SimulatorActions from './SimulatorActions'; +import SimulatorStore from './SimulatorStore'; + +const ProcessorSimulator = React.createClass({ + propTypes: { + stream: React.PropTypes.object.isRequired, + }, + + getInitialState() { + return { + message: undefined, + simulation: undefined, + loading: false, + error: undefined, + }; + }, + + _onMessageLoad(message) { + this.setState({ message: message, simulation: undefined, loading: true, error: undefined }); + + SimulatorActions.simulate.triggerPromise(this.props.stream, message.index, message.id).then( + messages => { + this.setState({ simulation: messages, loading: false }); + }, + error => { + this.setState({ loading: false, error: error }); + } + ); + }, + + render() { + return ( +
    + + +

    Load a message

    +

    Load a message to be used in the simulation. No changes will be done in your stored + messages.

    + + +     + +
    + ); + }, +}); + +export default ProcessorSimulator; diff --git a/src/web/simulator/SimulationPreview.css b/src/web/simulator/SimulationPreview.css new file mode 100644 index 000000000000..52900ce52664 --- /dev/null +++ b/src/web/simulator/SimulationPreview.css @@ -0,0 +1,9 @@ +.message-preview-wrapper { + margin-left: 15px; + margin-right: 15px; +} + +.message-preview-wrapper dl { + margin-top: 5px; + margin-bottom: 0; +} \ No newline at end of file diff --git a/src/web/simulator/SimulationPreview.jsx b/src/web/simulator/SimulationPreview.jsx new file mode 100644 index 000000000000..318fda46a43e --- /dev/null +++ b/src/web/simulator/SimulationPreview.jsx @@ -0,0 +1,111 @@ +import React from 'react'; +import { Alert, Col, Row } from 'react-bootstrap'; + +import { Spinner } from 'components/common'; +import MessageShow from 'components/search/MessageShow'; + +const SimulationPreview = React.createClass({ + propTypes: { + stream: React.PropTypes.object.isRequired, + originalMessage: React.PropTypes.object, + simulationResults: React.PropTypes.array, + isLoading: React.PropTypes.bool, + error: React.PropTypes.object, + }, + + componentDidMount() { + this.style.use(); + }, + + componentWillUnmount() { + this.style.unuse(); + }, + + style: require('!style/useable!css!./SimulationPreview.css'), + + render() { + if (!this.props.originalMessage && !this.props.simulationResults) { + return null; + } + + const streams = {}; + streams[this.props.stream.id] = this.props.stream; + + let originalMessagePreview = (this.props.isLoading ? : null); + if (this.props.originalMessage) { + originalMessagePreview = ( + + ); + } + + let simulationPreview = (this.props.isLoading ? : null); + if (this.props.simulationResults && Array.isArray(this.props.simulationResults)) { + if (this.props.simulationResults.length === 0) { + simulationPreview = ( + +

    Message would be dropped

    +

    + Processing the loaded message would drop it from the system. That means that the message would + not be stored, and would not be available on searches, alerts, or dashboards. +

    +     + ); + } else { + const messages = this.props.simulationResults.map(message => { + return ( + + ); + }); + simulationPreview =
    {messages}
    ; + } + } + + let errorMessage; + if (this.props.error) { + errorMessage = ( + +

    Error simulating message processing

    +

    + Could not simulate processing of message {this.props.originalMessage.id} in stream{' '} + {this.props.stream.title}. +
    + Please try loading the message again, or use another message for the simulation. +

    +     + ); + } + + return ( + + +
    + + +

    Original message

    +

    This is the original message loaded from Graylog.

    +
    + {originalMessagePreview} +
    + + +

    Simulation results

    +

    This is the result of processing the loaded message:

    + {errorMessage} + {simulationPreview} + +     + ); + }, +}); + +export default SimulationPreview; diff --git a/src/web/simulator/SimulatorActions.js b/src/web/simulator/SimulatorActions.js new file mode 100644 index 000000000000..19f374d7e818 --- /dev/null +++ b/src/web/simulator/SimulatorActions.js @@ -0,0 +1,7 @@ +import Reflux from 'reflux'; + +const SimulatorActions = Reflux.createActions({ + simulate: { asyncResult: true }, +}); + +export default SimulatorActions; diff --git a/src/web/simulator/SimulatorPage.jsx b/src/web/simulator/SimulatorPage.jsx new file mode 100644 index 000000000000..175d0fa6d451 --- /dev/null +++ b/src/web/simulator/SimulatorPage.jsx @@ -0,0 +1,101 @@ +import React from 'react'; +import Reflux from 'reflux'; +import { Button, Row, Col } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { PageHeader, Spinner } from 'components/common'; +import DocumentationLink from 'components/support/DocumentationLink'; +import ProcessorSimulator from './ProcessorSimulator'; + +import PipelinesActions from 'pipelines/PipelinesActions'; +import PipelinesStore from 'pipelines/PipelinesStore'; +import PipelineConnectionsActions from 'pipeline-connections/PipelineConnectionsActions'; +import PipelineConnectionsStore from 'pipeline-connections/PipelineConnectionsStore'; + +import StoreProvider from 'injection/StoreProvider'; +const StreamsStore = StoreProvider.getStore('Streams'); + +import DocsHelper from 'util/DocsHelper'; + +const SimulatorPage = React.createClass({ + propTypes: { + params: React.PropTypes.object.isRequired, + }, + + mixins: [Reflux.connect(PipelinesStore), Reflux.connect(PipelineConnectionsStore)], + + getInitialState() { + return { + stream: this.props.params.streamId === 'default' ? this._getDefaultStream() : undefined, + }; + }, + + componentDidMount() { + PipelinesActions.list(); + PipelineConnectionsActions.list(); + + if (!this.state.stream) { + StreamsStore.get(this.props.params.streamId, stream => this.setState({ stream: stream })); + } + }, + + _getDefaultStream() { + return { + id: 'default', + title: 'Default', + description: 'Stream used by default for messages not matching another stream.', + }; + }, + + _isLoading() { + return !this.state.pipelines || !this.state.stream || !this.state.connections; + }, + + render() { + let content; + if (this._isLoading()) { + content = ; + } else { + content = ; + } + + let title; + if (this.state.stream) { + title = Simulate processing in stream {this.state.stream.title}; + } else { + title = 'Simulate processing'; + } + + return ( +
    + + + Processing messages can be complex. Use this page to simulate the result of processing an incoming{' '} + message using your current set of pipelines and rules for this stream. + + + Read more about Graylog pipelines in the . + + + + + + +   + + + + + + + + + {content} + + +
    + ); + }, +}); + +export default SimulatorPage; diff --git a/src/web/simulator/SimulatorStore.js b/src/web/simulator/SimulatorStore.js new file mode 100644 index 000000000000..3b031a1cd2e6 --- /dev/null +++ b/src/web/simulator/SimulatorStore.js @@ -0,0 +1,31 @@ +import Reflux from 'reflux'; +import URLUtils from 'util/URLUtils'; +import fetch from 'logic/rest/FetchProvider'; + +import MessageFormatter from 'logic/message/MessageFormatter'; + +import SimulatorActions from './SimulatorActions'; + +const urlPrefix = '/plugins/org.graylog.plugins.pipelineprocessor'; + +const SimulatorStore = Reflux.createStore({ + listenables: [SimulatorActions], + + simulate(stream, index, messageId) { + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/simulate`); + const simulation = { + stream_id: stream.id, + index: index, + message_id: messageId, + }; + + let promise = fetch('POST', url, simulation); + promise = promise.then(response => { + return response.messages.map(MessageFormatter.formatMessageSummary); + }); + + SimulatorActions.simulate.promise(promise); + }, +}); + +export default SimulatorStore; From 6f7e5c1edacbad3837044c96e4cec2169c5fb192 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 14 Jun 2016 18:09:02 +0200 Subject: [PATCH 257/521] Add simulation details (#36) * Add listener parameter to pipeline interpreter Makes it possible to hook into different pipeline processing steps. * Include simulation trace when simulating process In that way it's more straightforward to see what happened during the process if something unexpected happened. * Rename SimulationPreview to SimulationResults * Adapt UI to response with simulation trace * Add time that took processing the message * Move simulated messages to their own component * Add dropdown to select simulation results view This dropdown will let the user switch among: - Preview of processed messages - Changes summary - Simulation trace * First version of simulator trace * Generate simulation view options from a function * Add changes summary view option to simulator Provides a summary of changes done to the message, including added, removed, and mutated fields in the original message. --- .../pipelineprocessor/ast/Pipeline.java | 7 + .../plugins/pipelineprocessor/ast/Rule.java | 7 + .../plugins/pipelineprocessor/ast/Stage.java | 4 + .../processors/PipelineInterpreter.java | 21 ++ .../listeners/InterpreterListener.java | 25 +++ .../listeners/NoopInterpreterListener.java | 80 +++++++ .../rest/SimulationResponse.java | 17 +- .../rest/SimulatorResource.java | 8 +- .../simulator/PipelineInterpreterTrace.java | 22 ++ .../simulator/PipelineInterpreterTracer.java | 45 ++++ .../SimulatorInterpreterListener.java | 87 ++++++++ src/web/simulator/ProcessorSimulator.jsx | 8 +- src/web/simulator/SimulationChanges.css | 39 ++++ src/web/simulator/SimulationChanges.jsx | 211 ++++++++++++++++++ src/web/simulator/SimulationPreview.jsx | 114 ++-------- ...ationPreview.css => SimulationResults.css} | 0 src/web/simulator/SimulationResults.jsx | 153 +++++++++++++ src/web/simulator/SimulationTrace.css | 12 + src/web/simulator/SimulationTrace.jsx | 38 ++++ src/web/simulator/SimulatorStore.js | 6 +- 20 files changed, 804 insertions(+), 100 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java create mode 100644 src/web/simulator/SimulationChanges.css create mode 100644 src/web/simulator/SimulationChanges.jsx rename src/web/simulator/{SimulationPreview.css => SimulationResults.css} (100%) create mode 100644 src/web/simulator/SimulationResults.jsx create mode 100644 src/web/simulator/SimulationTrace.css create mode 100644 src/web/simulator/SimulationTrace.jsx diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java index 08c620d3ec5b..0eff5ff1b236 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java @@ -54,4 +54,11 @@ public abstract static class Builder { public abstract Builder stages(SortedSet stages); } + + public String toString() { + final StringBuilder sb = new StringBuilder("Pipeline "); + sb.append("'").append(name()).append("'"); + sb.append(" (").append(id()).append(")"); + return sb.toString(); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java index 7f40683e75cc..f8d06e8d3a22 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java @@ -62,4 +62,11 @@ public abstract static class Builder { public abstract Rule build(); } + + public String toString() { + final StringBuilder sb = new StringBuilder("Rule "); + sb.append("'").append(name()).append("'"); + sb.append(" (").append(id()).append(")"); + return sb.toString(); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java index b063b211713d..c56d0a7e2216 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java @@ -57,4 +57,8 @@ public abstract static class Builder { public abstract Builder ruleReferences(List ruleRefs); } + + public String toString() { + return "Stage " + stage(); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 56dbe03dd437..a78def740b96 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -43,6 +43,8 @@ import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.processors.listeners.InterpreterListener; +import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; import org.graylog2.plugin.Message; import org.graylog2.plugin.MessageCollection; @@ -177,6 +179,11 @@ private synchronized void reload() { */ @Override public Messages process(Messages messages) { + return process(messages, new NoopInterpreterListener()); + } + + public Messages process(Messages messages, InterpreterListener interpreterListener) { + interpreterListener.startProcessing(); // message id + stream id final Set> processingBlacklist = Sets.newHashSet(); @@ -208,6 +215,7 @@ public Messages process(Messages messages) { } else { // get the default stream pipeline connections for this message pipelinesToRun = streamConnection.get("default"); + interpreterListener.processDefaultStream(message, pipelinesToRun); if (log.isDebugEnabled()) { log.debug("[{}] running default stream pipelines: [{}]", msgId, @@ -223,6 +231,7 @@ public Messages process(Messages messages) { pipelinesToRun = ImmutableSet.copyOf(streamsIds.stream() .flatMap(streamId -> streamConnection.get(streamId).stream()) .collect(Collectors.toSet())); + interpreterListener.processStreams(message, pipelinesToRun, streamsIds); log.debug("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); } @@ -246,6 +255,7 @@ public Messages process(Messages messages) { continue; } metricRegistry.counter(name(Pipeline.class, pipeline.id(), "stage", String.valueOf(stage.stage()), "executed")).inc(); + interpreterListener.enterStage(stage); log.debug("[{}] evaluating rule conditions in stage {}: match {}", msgId, stage.stage(), @@ -258,6 +268,7 @@ public Messages process(Messages messages) { final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); boolean anyRulesMatched = false; for (Rule rule : stage.getRules()) { + interpreterListener.evaluateRule(rule, pipeline); if (rule.when().evaluateBool(context)) { anyRulesMatched = true; countRuleExecution(rule, pipeline, stage, "matched"); @@ -265,20 +276,24 @@ public Messages process(Messages messages) { if (context.hasEvaluationErrors()) { final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); appendProcessingError(rule, message, lastError.toString()); + interpreterListener.failEvaluateRule(rule, pipeline); log.debug("Encountered evaluation error during condition, skipping rule actions: {}", lastError); continue; } + interpreterListener.satisfyRule(rule, pipeline); log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); rulesToRun.add(rule); } else { countRuleExecution(rule, pipeline, stage, "not-matched"); + interpreterListener.dissatisfyRule(rule, pipeline); log.debug("[{}] rule `{}` does not match", msgId, rule.name()); } } RULES: for (Rule rule : rulesToRun) { countRuleExecution(rule, pipeline, stage, "executed"); + interpreterListener.executeRule(rule, pipeline); log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); for (Statement statement : rule.then()) { statement.evaluate(context); @@ -286,6 +301,7 @@ public Messages process(Messages messages) { // if the last statement resulted in an error, do not continue to execute this rules final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); appendProcessingError(rule, message, lastError.toString()); + interpreterListener.failExecuteRule(rule, pipeline); log.debug("Encountered evaluation error, skipping rest of the rule: {}", lastError); countRuleExecution(rule, pipeline, stage, "failed"); @@ -300,10 +316,12 @@ public Messages process(Messages messages) { // record that it is ok to proceed with the pipeline if ((stage.matchAll() && (rulesToRun.size() == stage.getRules().size())) || (rulesToRun.size() > 0 && anyRulesMatched)) { + interpreterListener.continuePipelineExecution(pipeline, stage); log.debug("[{}] stage {} for pipeline `{}` required match: {}, ok to proceed with next stage", msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); } else { // no longer execute stages from this pipeline, the guard prevents it + interpreterListener.stopPipelineExecution(pipeline, stage); log.debug("[{}] stage {} for pipeline `{}` required match: {}, NOT ok to proceed with next stage", msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); pipelinesToSkip.add(pipeline); @@ -315,6 +333,7 @@ public Messages process(Messages messages) { // 4a. also add all new messages from the context to the toProcess work list Iterables.addAll(toProcess, context.createdMessages()); context.clearCreatedMessages(); + interpreterListener.exitStage(stage); } } @@ -346,6 +365,8 @@ public Messages process(Messages messages) { } } } + + interpreterListener.finishProcessing(); // 7. return the processed messages return new MessageCollection(fullyProcessed); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java new file mode 100644 index 000000000000..517d37d9793d --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java @@ -0,0 +1,25 @@ +package org.graylog.plugins.pipelineprocessor.processors.listeners; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog2.plugin.Message; + +import java.util.Set; + +public interface InterpreterListener { + void startProcessing(); + void finishProcessing(); + void processDefaultStream(Message message, Set pipelines); + void processStreams(Message message, Set pipelines, Set streams); + void enterStage(Stage stage); + void exitStage(Stage stage); + void evaluateRule(Rule rule, Pipeline pipeline); + void failEvaluateRule(Rule rule, Pipeline pipeline); + void satisfyRule(Rule rule, Pipeline pipeline); + void dissatisfyRule(Rule rule, Pipeline pipeline); + void executeRule(Rule rule, Pipeline pipeline); + void failExecuteRule(Rule rule, Pipeline pipeline); + void continuePipelineExecution(Pipeline pipeline, Stage stage); + void stopPipelineExecution(Pipeline pipeline, Stage stage); +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java new file mode 100644 index 000000000000..11aab111db6f --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java @@ -0,0 +1,80 @@ +package org.graylog.plugins.pipelineprocessor.processors.listeners; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog2.plugin.Message; + +import java.util.Set; + +public class NoopInterpreterListener implements InterpreterListener { + @Override + public void startProcessing() { + + } + + @Override + public void finishProcessing() { + + } + + @Override + public void processDefaultStream(Message messageId, Set pipelines) { + + } + + @Override + public void processStreams(Message messageId, Set pipelines, Set streams) { + + } + + @Override + public void enterStage(Stage stage) { + + } + + @Override + public void exitStage(Stage stage) { + + } + + @Override + public void evaluateRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void failEvaluateRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void satisfyRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void dissatisfyRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void executeRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void failExecuteRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void continuePipelineExecution(Pipeline pipeline, Stage stage) { + + } + + @Override + public void stopPipelineExecution(Pipeline pipeline, Stage stage) { + + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java index 43cdee61b729..d88857493126 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTrace; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; import java.util.List; @@ -14,14 +15,24 @@ public abstract class SimulationResponse { @JsonProperty public abstract List messages(); + @JsonProperty + public abstract List simulationTrace(); + + @JsonProperty + public abstract long tookMicroseconds(); + public static SimulationResponse.Builder builder() { return new AutoValue_SimulationResponse.Builder(); } @JsonCreator - public static SimulationResponse create (@JsonProperty("messages") List messages) { + public static SimulationResponse create (@JsonProperty("messages") List messages, + @JsonProperty("simulation_trace") List simulationTrace, + @JsonProperty("took_microseconds") long tookMicroseconds) { return builder() .messages(messages) + .simulationTrace(simulationTrace) + .tookMicroseconds(tookMicroseconds) .build(); } @@ -30,5 +41,9 @@ public abstract static class Builder { public abstract SimulationResponse build(); public abstract SimulationResponse.Builder messages(List messages); + + public abstract SimulationResponse.Builder simulationTrace(List trace); + + public abstract SimulationResponse.Builder tookMicroseconds(long tookMicroseconds); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java index 5fa776510b73..aacf812c84ab 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -6,6 +6,7 @@ import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTracer; import org.graylog2.database.NotFoundException; import org.graylog2.indexer.messages.DocumentNotFoundException; import org.graylog2.indexer.messages.Messages; @@ -61,18 +62,19 @@ public SimulationResponse simulate(@ApiParam(name = "simulation", required = tru message.addStream(stream); } - List simulationResults = new ArrayList<>(); + final List simulationResults = new ArrayList<>(); + final PipelineInterpreterTracer pipelineInterpreterTracer = new PipelineInterpreterTracer(); for (MessageProcessor messageProcessor : orderedMessageProcessors) { if (messageProcessor instanceof PipelineInterpreter) { - org.graylog2.plugin.Messages processedMessages = messageProcessor.process(message); + org.graylog2.plugin.Messages processedMessages = ((PipelineInterpreter)messageProcessor).process(message, pipelineInterpreterTracer.getSimulatorInterpreterListener()); for (Message processedMessage : processedMessages) { simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); } } } - return SimulationResponse.create(simulationResults); + return SimulationResponse.create(simulationResults, pipelineInterpreterTracer.getExecutionTrace(), pipelineInterpreterTracer.took()); } catch (DocumentNotFoundException e) { throw new NotFoundException(e); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java new file mode 100644 index 000000000000..5981cc0c270c --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java @@ -0,0 +1,22 @@ +package org.graylog.plugins.pipelineprocessor.simulator; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineInterpreterTrace { + @JsonProperty + public abstract long time(); + + @JsonProperty + public abstract String message(); + + @JsonCreator + public static PipelineInterpreterTrace create (@JsonProperty("time") long time, + @JsonProperty("message") String message) { + return new AutoValue_PipelineInterpreterTrace(time, message); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java new file mode 100644 index 000000000000..fcccbfc89f24 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java @@ -0,0 +1,45 @@ +package org.graylog.plugins.pipelineprocessor.simulator; + +import com.google.common.base.Stopwatch; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class PipelineInterpreterTracer { + private final List executionTrace; + private final Stopwatch timer; + private final SimulatorInterpreterListener simulatorInterpreterListener; + + public PipelineInterpreterTracer() { + executionTrace = new ArrayList<>(); + timer = Stopwatch.createUnstarted(); + simulatorInterpreterListener = new SimulatorInterpreterListener(this); + } + + public SimulatorInterpreterListener getSimulatorInterpreterListener() { + return simulatorInterpreterListener; + } + + public List getExecutionTrace() { + return executionTrace; + } + + public long took() { + return timer.elapsed(TimeUnit.MICROSECONDS); + } + + public void addTrace(String message) { + executionTrace.add(PipelineInterpreterTrace.create(timer.elapsed(TimeUnit.MICROSECONDS), message)); + } + + public void startProcessing(String message) { + timer.start(); + addTrace(message); + } + + public void finishProcessing(String message) { + timer.stop(); + addTrace(message); + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java new file mode 100644 index 000000000000..02b3a606d780 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java @@ -0,0 +1,87 @@ +package org.graylog.plugins.pipelineprocessor.simulator; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.processors.listeners.InterpreterListener; +import org.graylog2.plugin.Message; + +import java.util.Set; + +class SimulatorInterpreterListener implements InterpreterListener { + private final PipelineInterpreterTracer executionTrace; + + SimulatorInterpreterListener(PipelineInterpreterTracer executionTrace) { + this.executionTrace = executionTrace; + } + + @Override + public void startProcessing() { + executionTrace.startProcessing("Starting message processing"); + } + + @Override + public void finishProcessing() { + executionTrace.finishProcessing("Finished message processing"); + } + + @Override + public void processDefaultStream(Message message, Set pipelines) { + executionTrace.addTrace("Message " + message.getId() + " running " + pipelines + " for default stream"); + } + + @Override + public void processStreams(Message message, Set pipelines, Set streams) { + executionTrace.addTrace("Message " + message.getId() + " running " + pipelines + " for streams " + streams); + } + + @Override + public void enterStage(Stage stage) { + executionTrace.addTrace("Enter " + stage); + } + + @Override + public void exitStage(Stage stage) { + executionTrace.addTrace("Exit " + stage); + } + + @Override + public void evaluateRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Evaluate " + rule + " in " + pipeline); + } + + @Override + public void failEvaluateRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Failed evaluation " + rule + " in " + pipeline); + } + + @Override + public void satisfyRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Evaluation satisfied " + rule + " in " + pipeline); + } + + @Override + public void dissatisfyRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Evaluation not satisfied " + rule + " in " + pipeline); + } + + @Override + public void executeRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Execute " + rule + " in " + pipeline); + } + + @Override + public void failExecuteRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Failed execution " + rule + " in " + pipeline); + } + + @Override + public void continuePipelineExecution(Pipeline pipeline, Stage stage) { + executionTrace.addTrace("Completed " + stage + " for " + pipeline + ", continuing to next stage"); + } + + @Override + public void stopPipelineExecution(Pipeline pipeline, Stage stage) { + executionTrace.addTrace("Completed " + stage + " for " + pipeline + ", NOT continuing to next stage"); + } +} diff --git a/src/web/simulator/ProcessorSimulator.jsx b/src/web/simulator/ProcessorSimulator.jsx index 099de5a02acd..a7aa59cefebf 100644 --- a/src/web/simulator/ProcessorSimulator.jsx +++ b/src/web/simulator/ProcessorSimulator.jsx @@ -2,7 +2,7 @@ import React from 'react'; import { Col, Row } from 'react-bootstrap'; import LoaderTabs from 'components/messageloaders/LoaderTabs'; -import SimulationPreview from './SimulationPreview'; +import SimulationResults from './SimulationResults'; import SimulatorActions from './SimulatorActions'; import SimulatorStore from './SimulatorStore'; @@ -25,8 +25,8 @@ const ProcessorSimulator = React.createClass({ this.setState({ message: message, simulation: undefined, loading: true, error: undefined }); SimulatorActions.simulate.triggerPromise(this.props.stream, message.index, message.id).then( - messages => { - this.setState({ simulation: messages, loading: false }); + response => { + this.setState({ simulation: response, loading: false }); }, error => { this.setState({ loading: false, error: error }); @@ -45,7 +45,7 @@ const ProcessorSimulator = React.createClass({ - message.id === originalMessage.id); + }, + + _formatFieldTitle(field) { + return
    {field}
    ; + }, + + _formatFieldValue(field, value, isAdded, isRemoved) { + const className = (isAdded ? 'added-field' : (isRemoved ? 'removed-field' : '')); + return
    {value}
    ; + }, + + _formatAddedFields(originalMessage, processedMessage) { + const originalFields = Object.keys(originalMessage.fields); + const processedFields = Object.keys(processedMessage.fields); + + const addedFields = processedFields.filter(field => originalFields.indexOf(field) === -1); + + if (addedFields.length === 0) { + return null; + } + + const formattedFields = []; + + addedFields.sort().forEach(field => { + formattedFields.push(this._formatFieldTitle(field)); + formattedFields.push(this._formatFieldValue(field, processedMessage.fields[field], true, false)); + }); + + return ( +
    +

    Added fields

    +
    + {formattedFields} +
    +
    + ); + }, + + _formatRemovedFields(originalMessage, processedMessage) { + const originalFields = Object.keys(originalMessage.fields); + const processedFields = Object.keys(processedMessage.fields); + + const removedFields = originalFields.filter(field => processedFields.indexOf(field) === -1); + + if (removedFields.length === 0) { + return null; + } + + const formattedFields = []; + + removedFields.sort().forEach(field => { + formattedFields.push(this._formatFieldTitle(field)); + formattedFields.push(this._formatFieldValue(field, originalMessage.fields[field], false, true)); + }); + + return ( +
    +

    Removed fields

    +
    + {formattedFields} +
    +
    + ); + }, + + _formatMutatedFields(originalMessage, processedMessage) { + const originalFields = Object.keys(originalMessage.fields); + const processedFields = Object.keys(processedMessage.fields); + + const mutatedFields = []; + + originalFields.forEach(field => { + if (processedFields.indexOf(field) === -1) { + return; + } + const originalValue = originalMessage.fields[field]; + const processedValue = processedMessage.fields[field]; + + if (typeof originalValue !== typeof processedValue) { + mutatedFields.push(field); + return; + } + + // Convert to JSON to avoid problems comparing objects or arrays. Yes, this sucks :/ + if (JSON.stringify(originalValue) !== JSON.stringify(processedValue)) { + mutatedFields.push(field); + } + }); + + if (mutatedFields.length === 0) { + return null; + } + + const formattedFields = []; + + mutatedFields.sort().forEach(field => { + formattedFields.push(this._formatFieldTitle(field)); + formattedFields.push(this._formatFieldValue(`${field}-original`, originalMessage.fields[field], false, true)); + formattedFields.push(this._formatFieldValue(field, processedMessage.fields[field], true, false)); + }); + + return ( +
    +

    Mutated fields

    +
    + {formattedFields} +
    +
    + ); + }, + + _getOriginalMessageChanges() { + const originalMessage = this.props.originalMessage; + const processedMessages = this.props.simulationResults.messages; + + if (this._isOriginalMessageRemoved(originalMessage, processedMessages)) { + return

    Original message would be dropped during processing.

    ; + } + + const processedMessage = processedMessages.find(message => message.id === originalMessage.id); + + const formattedAddedFields = this._formatAddedFields(originalMessage, processedMessage); + const formattedRemovedFields = this._formatRemovedFields(originalMessage, processedMessage); + const formattedMutatedFields = this._formatMutatedFields(originalMessage, processedMessage); + + if (!formattedAddedFields && !formattedRemovedFields && !formattedMutatedFields) { + return

    Original message would be not be modified during processing.

    ; + } + + return ( +
    + {formattedAddedFields} + {formattedRemovedFields} + {formattedMutatedFields} +
    + ); + }, + + _formatOriginalMessageChanges() { + return ( + + +

    + Changes in original message{' '} + {this.props.originalMessage.id} +

    + {this._getOriginalMessageChanges()} + +     + ); + }, + + _formatOtherChanges() { + const originalMessageId = this.props.originalMessage.id; + const simulatedMessages = this.props.simulationResults.messages; + + const createdMessages = simulatedMessages.filter(message => message.id !== originalMessageId); + + if (createdMessages.length === 0) { + return null; + } + + return ( + + +

    Other changes

    +

    + There would be {createdMessages.length}{' '} + created.{' '} + Switch to the Results preview view option to see{' '} + . +

    + +     + ); + }, + + render() { + return ( +
    + {this._formatOriginalMessageChanges()} + {this._formatOtherChanges()} +
    + ); + }, +}); + +export default SimulationChanges; diff --git a/src/web/simulator/SimulationPreview.jsx b/src/web/simulator/SimulationPreview.jsx index 318fda46a43e..1658dcb54639 100644 --- a/src/web/simulator/SimulationPreview.jsx +++ b/src/web/simulator/SimulationPreview.jsx @@ -1,110 +1,42 @@ import React from 'react'; -import { Alert, Col, Row } from 'react-bootstrap'; +import { Alert } from 'react-bootstrap'; -import { Spinner } from 'components/common'; import MessageShow from 'components/search/MessageShow'; const SimulationPreview = React.createClass({ propTypes: { - stream: React.PropTypes.object.isRequired, - originalMessage: React.PropTypes.object, - simulationResults: React.PropTypes.array, - isLoading: React.PropTypes.bool, - error: React.PropTypes.object, + simulationResults: React.PropTypes.object.isRequired, + streams: React.PropTypes.object.isRequired, }, - componentDidMount() { - this.style.use(); - }, - - componentWillUnmount() { - this.style.unuse(); - }, - - style: require('!style/useable!css!./SimulationPreview.css'), - render() { - if (!this.props.originalMessage && !this.props.simulationResults) { - return null; - } - - const streams = {}; - streams[this.props.stream.id] = this.props.stream; - - let originalMessagePreview = (this.props.isLoading ? : null); - if (this.props.originalMessage) { - originalMessagePreview = ( - - ); - } - - let simulationPreview = (this.props.isLoading ? : null); - if (this.props.simulationResults && Array.isArray(this.props.simulationResults)) { - if (this.props.simulationResults.length === 0) { - simulationPreview = ( - -

    Message would be dropped

    -

    - Processing the loaded message would drop it from the system. That means that the message would - not be stored, and would not be available on searches, alerts, or dashboards. -

    -     - ); - } else { - const messages = this.props.simulationResults.map(message => { - return ( - - ); - }); - simulationPreview =
    {messages}
    ; - } - } + const messages = this.props.simulationResults.messages; - let errorMessage; - if (this.props.error) { - errorMessage = ( - -

    Error simulating message processing

    + if (messages.length === 0) { + return ( + +

    Message would be dropped

    - Could not simulate processing of message {this.props.originalMessage.id} in stream{' '} - {this.props.stream.title}. -
    - Please try loading the message again, or use another message for the simulation. + Processing the loaded message would drop it from the system. That means that the message would + not be stored, and would not be available on searches, alerts, or dashboards.

    ); } - return ( - - -
    - - -

    Original message

    -

    This is the original message loaded from Graylog.

    -
    - {originalMessagePreview} -
    - - -

    Simulation results

    -

    This is the result of processing the loaded message:

    - {errorMessage} - {simulationPreview} - -     - ); + const formattedMessages = messages.map(message => { + return ( + + ); + }); + + return
    {formattedMessages}
    ; }, }); diff --git a/src/web/simulator/SimulationPreview.css b/src/web/simulator/SimulationResults.css similarity index 100% rename from src/web/simulator/SimulationPreview.css rename to src/web/simulator/SimulationResults.css diff --git a/src/web/simulator/SimulationResults.jsx b/src/web/simulator/SimulationResults.jsx new file mode 100644 index 000000000000..5f068d8fa5e0 --- /dev/null +++ b/src/web/simulator/SimulationResults.jsx @@ -0,0 +1,153 @@ +import React from 'react'; +import { Alert, Col, DropdownButton, MenuItem, Row } from 'react-bootstrap'; + +import { Spinner } from 'components/common'; +import MessageShow from 'components/search/MessageShow'; + +import SimulationChanges from './SimulationChanges'; +import SimulationPreview from './SimulationPreview'; +import SimulationTrace from './SimulationTrace'; + +import NumberUtils from 'util/NumberUtils'; + +const SimulationResults = React.createClass({ + propTypes: { + stream: React.PropTypes.object.isRequired, + originalMessage: React.PropTypes.object, + simulationResults: React.PropTypes.object, + isLoading: React.PropTypes.bool, + error: React.PropTypes.object, + }, + + getInitialState() { + return { + viewOption: this.VIEW_OPTIONS.SIMULATION_PREVIEW, + }; + }, + + componentDidMount() { + this.style.use(); + }, + + componentWillUnmount() { + this.style.unuse(); + }, + + VIEW_OPTIONS: { + SIMULATION_PREVIEW: 1, + SIMULATION_SUMMARY: 2, + SIMULATION_TRACE: 3, + }, + + style: require('!style/useable!css!./SimulationResults.css'), + + _changeViewOptions(_, eventKey) { + const selectedOption = Object.keys(this.VIEW_OPTIONS).find(key => this.VIEW_OPTIONS[key] === eventKey); + this.setState({ viewOption: this.VIEW_OPTIONS[selectedOption] }); + }, + + _getViewOptionsMenuItems() { + const viewOptionsMenuItems = []; + + viewOptionsMenuItems.push(this._getViewOptionsMenuItem(this.VIEW_OPTIONS.SIMULATION_SUMMARY, 'Changes summary')); + viewOptionsMenuItems.push(this._getViewOptionsMenuItem(this.VIEW_OPTIONS.SIMULATION_PREVIEW, 'Results preview')); + viewOptionsMenuItems.push(this._getViewOptionsMenuItem(this.VIEW_OPTIONS.SIMULATION_TRACE, 'Simulation trace')); + + return viewOptionsMenuItems; + }, + + _getViewOptionsMenuItem(option, text) { + return ( + + {text} + + ); + }, + + _getViewComponent(streams) { + if (this.props.isLoading || !this.props.simulationResults) { + return ; + } + + switch (this.state.viewOption) { + case this.VIEW_OPTIONS.SIMULATION_PREVIEW: + return ; + case this.VIEW_OPTIONS.SIMULATION_SUMMARY: + return ; + case this.VIEW_OPTIONS.SIMULATION_TRACE: + return ; + default: + // it should never happen™ + } + + return null; + }, + + render() { + if (!this.props.originalMessage && !this.props.simulationResults) { + return null; + } + + const streams = {}; + streams[this.props.stream.id] = this.props.stream; + + let originalMessagePreview = (this.props.isLoading ? : null); + if (this.props.originalMessage) { + originalMessagePreview = ( + + ); + } + + let errorMessage; + if (this.props.error) { + errorMessage = ( + +

    Error simulating message processing

    +

    + Could not simulate processing of message {this.props.originalMessage.id} in stream{' '} + {this.props.stream.title}. +
    + Please try loading the message again, or use another message for the simulation. +

    +     + ); + } + + return ( + + +
    + + +

    Original message

    +

    This is the original message loaded from Graylog.

    +
    + {originalMessagePreview} +
    + + +
    + + {this._getViewOptionsMenuItems()} + +
    +

    Simulation results

    +

    + {this.props.isLoading ? + 'Simulating message processing, please wait a moment.' : + `These are the results of processing the loaded message. Processing took ${NumberUtils.formatNumber(this.props.simulationResults.took_microseconds)} µs.`} +

    + {errorMessage} + {this._getViewComponent(streams)} + +     + ); + }, +}); + +export default SimulationResults; diff --git a/src/web/simulator/SimulationTrace.css b/src/web/simulator/SimulationTrace.css new file mode 100644 index 000000000000..5dde5eb2fb51 --- /dev/null +++ b/src/web/simulator/SimulationTrace.css @@ -0,0 +1,12 @@ +.dl-simulation-trace { + padding-top: 15px; +} + +.dl-simulation-trace dt { + width: 80px; +} + +.dl-simulation-trace dd { + margin-left: 100px; + margin-bottom: 5px; +} diff --git a/src/web/simulator/SimulationTrace.jsx b/src/web/simulator/SimulationTrace.jsx new file mode 100644 index 000000000000..f62cd9e41cd5 --- /dev/null +++ b/src/web/simulator/SimulationTrace.jsx @@ -0,0 +1,38 @@ +import React from 'react'; + +import NumberUtils from 'util/NumberUtils'; + +const SimulationTrace = React.createClass({ + propTypes: { + simulationResults: React.PropTypes.object.isRequired, + }, + + componentDidMount() { + this.style.use(); + }, + + componentWillUnmount() { + this.style.unuse(); + }, + + style: require('!style/useable!css!./SimulationTrace.css'), + + render() { + const simulationTrace = this.props.simulationResults.simulation_trace; + + const traceEntries = []; + + simulationTrace.forEach((trace, idx) => { + traceEntries.push(
    {NumberUtils.formatNumber(trace.time)} μs
    ); + traceEntries.push(
    {trace.message}
    ); + }); + + return ( +
    + {traceEntries} +
    + ); + }, +}); + +export default SimulationTrace; diff --git a/src/web/simulator/SimulatorStore.js b/src/web/simulator/SimulatorStore.js index 3b031a1cd2e6..7f5b85b299cd 100644 --- a/src/web/simulator/SimulatorStore.js +++ b/src/web/simulator/SimulatorStore.js @@ -3,6 +3,7 @@ import URLUtils from 'util/URLUtils'; import fetch from 'logic/rest/FetchProvider'; import MessageFormatter from 'logic/message/MessageFormatter'; +import ObjectUtils from 'util/ObjectUtils'; import SimulatorActions from './SimulatorActions'; @@ -21,7 +22,10 @@ const SimulatorStore = Reflux.createStore({ let promise = fetch('POST', url, simulation); promise = promise.then(response => { - return response.messages.map(MessageFormatter.formatMessageSummary); + const formattedResponse = ObjectUtils.clone(response); + formattedResponse.messages = response.messages.map(MessageFormatter.formatMessageSummary); + + return formattedResponse; }); SimulatorActions.simulate.promise(promise); From e8ba75f4e8a37900e6a9bdce24a94e014f541ac5 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 20 Jun 2016 15:50:11 +0200 Subject: [PATCH 258/521] Add license headers --- .../listeners/InterpreterListener.java | 16 ++++++++++++++++ .../listeners/NoopInterpreterListener.java | 16 ++++++++++++++++ .../rest/SimulationRequest.java | 16 ++++++++++++++++ .../rest/SimulationResponse.java | 16 ++++++++++++++++ .../rest/SimulatorResource.java | 16 ++++++++++++++++ .../simulator/PipelineInterpreterTrace.java | 16 ++++++++++++++++ .../simulator/PipelineInterpreterTracer.java | 16 ++++++++++++++++ .../simulator/SimulatorInterpreterListener.java | 16 ++++++++++++++++ 8 files changed, 128 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java index 517d37d9793d..6e2bd90848ed 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.processors.listeners; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java index 11aab111db6f..793d4d020f27 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.processors.listeners; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java index a05f9422881a..cbf1e8963d0c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import com.fasterxml.jackson.annotation.JsonAutoDetect; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java index d88857493126..e6f803d89ea6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import com.fasterxml.jackson.annotation.JsonAutoDetect; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java index aacf812c84ab..8e7be6ec966c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.rest; import io.swagger.annotations.Api; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java index 5981cc0c270c..cfcc35147586 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.simulator; import com.fasterxml.jackson.annotation.JsonAutoDetect; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java index fcccbfc89f24..027f10729040 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.simulator; import com.google.common.base.Stopwatch; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java index 02b3a606d780..a02e810906ab 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.simulator; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; From 3495bd9f8dd03f80e3d9ba06116300b3bc513ca3 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 1 Jul 2016 13:43:42 +0200 Subject: [PATCH 259/521] Remove shrinkwrap during development Make changes in package.json easier to do. We should add it back when we are getting closer to the release. --- npm-shrinkwrap.json | 3365 ------------------------------------------- 1 file changed, 3365 deletions(-) delete mode 100644 npm-shrinkwrap.json diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json deleted file mode 100644 index ff397ab7ec68..000000000000 --- a/npm-shrinkwrap.json +++ /dev/null @@ -1,3365 +0,0 @@ -{ - "name": "PipelineProcessor", - "version": "1.0.0-alpha.7-SNAPSHOT", - "dependencies": { - "acorn": { - "version": "1.2.2", - "from": "acorn@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-1.2.2.tgz" - }, - "acorn-jsx": { - "version": "2.0.1", - "from": "acorn-jsx@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-2.0.1.tgz", - "dependencies": { - "acorn": { - "version": "2.7.0", - "from": "acorn@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-2.7.0.tgz" - } - } - }, - "align-text": { - "version": "0.1.4", - "from": "align-text@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz" - }, - "alphanum-sort": { - "version": "1.0.2", - "from": "alphanum-sort@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz" - }, - "alter": { - "version": "0.2.0", - "from": "alter@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/alter/-/alter-0.2.0.tgz" - }, - "amdefine": { - "version": "1.0.0", - "from": "amdefine@>=0.0.4", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.0.tgz" - }, - "ansi-escapes": { - "version": "1.3.0", - "from": "ansi-escapes@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.3.0.tgz" - }, - "ansi-regex": { - "version": "2.0.0", - "from": "ansi-regex@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" - }, - "ansi-styles": { - "version": "2.2.1", - "from": "ansi-styles@>=2.2.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz" - }, - "anymatch": { - "version": "1.3.0", - "from": "anymatch@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-1.3.0.tgz" - }, - "argparse": { - "version": "1.0.7", - "from": "argparse@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.7.tgz" - }, - "arr-diff": { - "version": "2.0.0", - "from": "arr-diff@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-2.0.0.tgz" - }, - "arr-flatten": { - "version": "1.0.1", - "from": "arr-flatten@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.0.1.tgz" - }, - "array-union": { - "version": "1.0.1", - "from": "array-union@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.1.tgz" - }, - "array-uniq": { - "version": "1.0.2", - "from": "array-uniq@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.2.tgz" - }, - "array-unique": { - "version": "0.2.1", - "from": "array-unique@>=0.2.1 <0.3.0", - "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.2.1.tgz" - }, - "arrify": { - "version": "1.0.1", - "from": "arrify@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" - }, - "asap": { - "version": "2.0.3", - "from": "asap@>=2.0.3 <2.1.0", - "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.3.tgz" - }, - "assert": { - "version": "1.3.0", - "from": "assert@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/assert/-/assert-1.3.0.tgz" - }, - "ast-traverse": { - "version": "0.1.1", - "from": "ast-traverse@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/ast-traverse/-/ast-traverse-0.1.1.tgz" - }, - "ast-types": { - "version": "0.8.12", - "from": "ast-types@0.8.12", - "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.8.12.tgz" - }, - "async": { - "version": "0.2.10", - "from": "async@>=0.2.6 <0.3.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" - }, - "async-each": { - "version": "1.0.0", - "from": "async-each@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.0.tgz" - }, - "autoprefixer": { - "version": "6.3.5", - "from": "autoprefixer@>=6.3.1 <7.0.0", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-6.3.5.tgz" - }, - "babel-core": { - "version": "5.8.38", - "from": "babel-core@>=5.8.25 <6.0.0", - "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-5.8.38.tgz", - "dependencies": { - "js-tokens": { - "version": "1.0.1", - "from": "js-tokens@1.0.1", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-1.0.1.tgz" - }, - "source-map": { - "version": "0.5.3", - "from": "source-map@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" - } - } - }, - "babel-loader": { - "version": "5.4.0", - "from": "babel-loader@>=5.3.2 <6.0.0", - "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-5.4.0.tgz" - }, - "babel-plugin-constant-folding": { - "version": "1.0.1", - "from": "babel-plugin-constant-folding@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-constant-folding/-/babel-plugin-constant-folding-1.0.1.tgz" - }, - "babel-plugin-dead-code-elimination": { - "version": "1.0.2", - "from": "babel-plugin-dead-code-elimination@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-dead-code-elimination/-/babel-plugin-dead-code-elimination-1.0.2.tgz" - }, - "babel-plugin-eval": { - "version": "1.0.1", - "from": "babel-plugin-eval@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-eval/-/babel-plugin-eval-1.0.1.tgz" - }, - "babel-plugin-inline-environment-variables": { - "version": "1.0.1", - "from": "babel-plugin-inline-environment-variables@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-inline-environment-variables/-/babel-plugin-inline-environment-variables-1.0.1.tgz" - }, - "babel-plugin-jscript": { - "version": "1.0.4", - "from": "babel-plugin-jscript@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-jscript/-/babel-plugin-jscript-1.0.4.tgz" - }, - "babel-plugin-member-expression-literals": { - "version": "1.0.1", - "from": "babel-plugin-member-expression-literals@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-member-expression-literals/-/babel-plugin-member-expression-literals-1.0.1.tgz" - }, - "babel-plugin-property-literals": { - "version": "1.0.1", - "from": "babel-plugin-property-literals@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-property-literals/-/babel-plugin-property-literals-1.0.1.tgz" - }, - "babel-plugin-proto-to-assign": { - "version": "1.0.4", - "from": "babel-plugin-proto-to-assign@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-proto-to-assign/-/babel-plugin-proto-to-assign-1.0.4.tgz" - }, - "babel-plugin-react-constant-elements": { - "version": "1.0.3", - "from": "babel-plugin-react-constant-elements@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-react-constant-elements/-/babel-plugin-react-constant-elements-1.0.3.tgz" - }, - "babel-plugin-react-display-name": { - "version": "1.0.3", - "from": "babel-plugin-react-display-name@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-react-display-name/-/babel-plugin-react-display-name-1.0.3.tgz" - }, - "babel-plugin-remove-console": { - "version": "1.0.1", - "from": "babel-plugin-remove-console@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-remove-console/-/babel-plugin-remove-console-1.0.1.tgz" - }, - "babel-plugin-remove-debugger": { - "version": "1.0.1", - "from": "babel-plugin-remove-debugger@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-remove-debugger/-/babel-plugin-remove-debugger-1.0.1.tgz" - }, - "babel-plugin-runtime": { - "version": "1.0.7", - "from": "babel-plugin-runtime@>=1.0.7 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-runtime/-/babel-plugin-runtime-1.0.7.tgz" - }, - "babel-plugin-undeclared-variables-check": { - "version": "1.0.2", - "from": "babel-plugin-undeclared-variables-check@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-undeclared-variables-check/-/babel-plugin-undeclared-variables-check-1.0.2.tgz" - }, - "babel-plugin-undefined-to-void": { - "version": "1.1.6", - "from": "babel-plugin-undefined-to-void@>=1.1.6 <2.0.0", - "resolved": "https://registry.npmjs.org/babel-plugin-undefined-to-void/-/babel-plugin-undefined-to-void-1.1.6.tgz" - }, - "babel-runtime": { - "version": "5.8.38", - "from": "babel-runtime@>=5.8.25 <6.0.0", - "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-5.8.38.tgz" - }, - "babylon": { - "version": "5.8.38", - "from": "babylon@>=5.8.38 <6.0.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-5.8.38.tgz" - }, - "balanced-match": { - "version": "0.3.0", - "from": "balanced-match@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.3.0.tgz" - }, - "base62": { - "version": "0.1.1", - "from": "base62@0.1.1", - "resolved": "https://registry.npmjs.org/base62/-/base62-0.1.1.tgz" - }, - "Base64": { - "version": "0.2.1", - "from": "Base64@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/Base64/-/Base64-0.2.1.tgz" - }, - "base64-js": { - "version": "0.0.8", - "from": "base64-js@0.0.8", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-0.0.8.tgz" - }, - "big.js": { - "version": "3.1.3", - "from": "big.js@>=3.0.2 <4.0.0", - "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.1.3.tgz" - }, - "binary-extensions": { - "version": "1.4.0", - "from": "binary-extensions@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.4.0.tgz" - }, - "bluebird": { - "version": "2.10.2", - "from": "bluebird@>=2.9.33 <3.0.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.10.2.tgz" - }, - "blueimp-tmpl": { - "version": "2.5.7", - "from": "blueimp-tmpl@>=2.5.5 <3.0.0", - "resolved": "https://registry.npmjs.org/blueimp-tmpl/-/blueimp-tmpl-2.5.7.tgz" - }, - "brace": { - "version": "0.7.0", - "from": "brace@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/brace/-/brace-0.7.0.tgz" - }, - "brace-expansion": { - "version": "1.1.3", - "from": "brace-expansion@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.3.tgz" - }, - "braces": { - "version": "1.8.3", - "from": "braces@>=1.8.2 <2.0.0", - "resolved": "https://registry.npmjs.org/braces/-/braces-1.8.3.tgz" - }, - "breakable": { - "version": "1.0.0", - "from": "breakable@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/breakable/-/breakable-1.0.0.tgz" - }, - "browserify-zlib": { - "version": "0.1.4", - "from": "browserify-zlib@>=0.1.4 <0.2.0", - "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz" - }, - "browserslist": { - "version": "1.3.0", - "from": "browserslist@>=1.3.0 <1.4.0", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-1.3.0.tgz" - }, - "buffer": { - "version": "3.6.0", - "from": "buffer@>=3.0.3 <4.0.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-3.6.0.tgz" - }, - "caller-path": { - "version": "0.1.0", - "from": "caller-path@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz" - }, - "callsites": { - "version": "0.2.0", - "from": "callsites@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz" - }, - "camel-case": { - "version": "1.2.2", - "from": "camel-case@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-1.2.2.tgz" - }, - "camelcase": { - "version": "1.2.1", - "from": "camelcase@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz" - }, - "caniuse-db": { - "version": "1.0.30000443", - "from": "caniuse-db@>=1.0.30000436 <2.0.0", - "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000443.tgz" - }, - "center-align": { - "version": "0.1.3", - "from": "center-align@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz" - }, - "chalk": { - "version": "1.1.3", - "from": "chalk@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz" - }, - "change-case": { - "version": "2.3.1", - "from": "change-case@>=2.3.0 <2.4.0", - "resolved": "https://registry.npmjs.org/change-case/-/change-case-2.3.1.tgz" - }, - "chokidar": { - "version": "1.4.3", - "from": "chokidar@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.4.3.tgz" - }, - "clap": { - "version": "1.1.0", - "from": "clap@>=1.0.9 <2.0.0", - "resolved": "https://registry.npmjs.org/clap/-/clap-1.1.0.tgz", - "dependencies": { - "chalk": { - "version": "1.1.1", - "from": "chalk@1.1.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.1.tgz" - } - } - }, - "classnames": { - "version": "2.2.3", - "from": "classnames@>=2.1.5 <3.0.0", - "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.2.3.tgz" - }, - "clean-css": { - "version": "3.4.10", - "from": "clean-css@>=3.4.0 <3.5.0", - "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-3.4.10.tgz", - "dependencies": { - "commander": { - "version": "2.8.1", - "from": "commander@>=2.8.0 <2.9.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz" - }, - "source-map": { - "version": "0.4.4", - "from": "source-map@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz" - } - } - }, - "cli": { - "version": "0.11.2", - "from": "cli@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/cli/-/cli-0.11.2.tgz" - }, - "cli-cursor": { - "version": "1.0.2", - "from": "cli-cursor@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz" - }, - "cli-width": { - "version": "2.1.0", - "from": "cli-width@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.1.0.tgz" - }, - "cliui": { - "version": "2.1.0", - "from": "cliui@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz" - }, - "clone": { - "version": "1.0.2", - "from": "clone@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.2.tgz" - }, - "coa": { - "version": "1.0.1", - "from": "coa@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/coa/-/coa-1.0.1.tgz" - }, - "code-point-at": { - "version": "1.0.0", - "from": "code-point-at@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.0.0.tgz" - }, - "color": { - "version": "0.11.1", - "from": "color@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/color/-/color-0.11.1.tgz" - }, - "color-convert": { - "version": "0.5.3", - "from": "color-convert@>=0.5.3 <0.6.0", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-0.5.3.tgz" - }, - "color-name": { - "version": "1.1.1", - "from": "color-name@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.1.tgz" - }, - "color-string": { - "version": "0.3.0", - "from": "color-string@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-0.3.0.tgz" - }, - "colormin": { - "version": "1.1.0", - "from": "colormin@>=1.0.5 <2.0.0", - "resolved": "https://registry.npmjs.org/colormin/-/colormin-1.1.0.tgz" - }, - "colors": { - "version": "1.1.2", - "from": "colors@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz" - }, - "commander": { - "version": "2.9.0", - "from": "commander@>=2.5.0 <3.0.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz" - }, - "commoner": { - "version": "0.10.4", - "from": "commoner@>=0.10.3 <0.11.0", - "resolved": "https://registry.npmjs.org/commoner/-/commoner-0.10.4.tgz" - }, - "concat-map": { - "version": "0.0.1", - "from": "concat-map@0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" - }, - "concat-stream": { - "version": "1.5.1", - "from": "concat-stream@>=1.4.6 <2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.1.tgz" - }, - "console-browserify": { - "version": "1.1.0", - "from": "console-browserify@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz" - }, - "constant-case": { - "version": "1.1.2", - "from": "constant-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/constant-case/-/constant-case-1.1.2.tgz" - }, - "constants-browserify": { - "version": "0.0.1", - "from": "constants-browserify@0.0.1", - "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-0.0.1.tgz" - }, - "convert-source-map": { - "version": "1.2.0", - "from": "convert-source-map@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.2.0.tgz" - }, - "core-js": { - "version": "1.2.6", - "from": "core-js@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.6.tgz" - }, - "core-util-is": { - "version": "1.0.2", - "from": "core-util-is@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" - }, - "crypto-browserify": { - "version": "3.2.8", - "from": "crypto-browserify@>=3.2.6 <3.3.0", - "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.2.8.tgz" - }, - "css-color-names": { - "version": "0.0.3", - "from": "css-color-names@0.0.3", - "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.3.tgz" - }, - "css-loader": { - "version": "0.23.1", - "from": "css-loader@>=0.23.1 <0.24.0", - "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-0.23.1.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "css-selector-tokenizer": { - "version": "0.5.4", - "from": "css-selector-tokenizer@>=0.5.1 <0.6.0", - "resolved": "https://registry.npmjs.org/css-selector-tokenizer/-/css-selector-tokenizer-0.5.4.tgz" - }, - "cssesc": { - "version": "0.1.0", - "from": "cssesc@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-0.1.0.tgz" - }, - "cssnano": { - "version": "3.5.2", - "from": "cssnano@>=2.6.1 <4.0.0", - "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-3.5.2.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "csso": { - "version": "1.6.4", - "from": "csso@>=1.6.4 <1.7.0", - "resolved": "https://registry.npmjs.org/csso/-/csso-1.6.4.tgz", - "dependencies": { - "source-map": { - "version": "0.5.3", - "from": "source-map@>=0.5.3 <0.6.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" - } - } - }, - "d": { - "version": "0.1.1", - "from": "d@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/d/-/d-0.1.1.tgz" - }, - "date-now": { - "version": "0.1.4", - "from": "date-now@>=0.1.4 <0.2.0", - "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz" - }, - "debug": { - "version": "2.2.0", - "from": "debug@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz" - }, - "decamelize": { - "version": "1.2.0", - "from": "decamelize@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz" - }, - "deep-equal": { - "version": "1.0.1", - "from": "deep-equal@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz" - }, - "deep-is": { - "version": "0.1.3", - "from": "deep-is@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz" - }, - "defined": { - "version": "1.0.0", - "from": "defined@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz" - }, - "defs": { - "version": "1.1.1", - "from": "defs@>=1.1.0 <1.2.0", - "resolved": "https://registry.npmjs.org/defs/-/defs-1.1.1.tgz", - "dependencies": { - "esprima-fb": { - "version": "15001.1001.0-dev-harmony-fb", - "from": "esprima-fb@>=15001.1001.0-dev-harmony-fb <15001.1002.0", - "resolved": "https://registry.npmjs.org/esprima-fb/-/esprima-fb-15001.1001.0-dev-harmony-fb.tgz" - } - } - }, - "del": { - "version": "2.2.0", - "from": "del@>=2.0.2 <3.0.0", - "resolved": "https://registry.npmjs.org/del/-/del-2.2.0.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "detect-indent": { - "version": "3.0.1", - "from": "detect-indent@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-3.0.1.tgz" - }, - "detective": { - "version": "4.3.1", - "from": "detective@>=4.3.1 <5.0.0", - "resolved": "https://registry.npmjs.org/detective/-/detective-4.3.1.tgz" - }, - "doctrine": { - "version": "1.2.1", - "from": "doctrine@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.2.1.tgz", - "dependencies": { - "esutils": { - "version": "1.1.6", - "from": "esutils@>=1.1.6 <2.0.0", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-1.1.6.tgz" - } - } - }, - "dom-helpers": { - "version": "2.4.0", - "from": "dom-helpers@>=2.4.0 <3.0.0", - "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-2.4.0.tgz" - }, - "domain-browser": { - "version": "1.1.7", - "from": "domain-browser@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.1.7.tgz" - }, - "dot-case": { - "version": "1.1.2", - "from": "dot-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-1.1.2.tgz" - }, - "enhanced-resolve": { - "version": "0.9.1", - "from": "enhanced-resolve@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-0.9.1.tgz", - "dependencies": { - "memory-fs": { - "version": "0.2.0", - "from": "memory-fs@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.2.0.tgz" - } - } - }, - "envify": { - "version": "3.4.0", - "from": "envify@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/envify/-/envify-3.4.0.tgz" - }, - "errno": { - "version": "0.1.4", - "from": "errno@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.4.tgz" - }, - "es5-ext": { - "version": "0.10.11", - "from": "es5-ext@>=0.10.8 <0.11.0", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.11.tgz" - }, - "es6-iterator": { - "version": "2.0.0", - "from": "es6-iterator@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.0.tgz" - }, - "es6-map": { - "version": "0.1.3", - "from": "es6-map@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/es6-map/-/es6-map-0.1.3.tgz" - }, - "es6-set": { - "version": "0.1.4", - "from": "es6-set@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/es6-set/-/es6-set-0.1.4.tgz" - }, - "es6-symbol": { - "version": "3.0.2", - "from": "es6-symbol@>=3.0.1 <3.1.0", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.0.2.tgz" - }, - "es6-weak-map": { - "version": "2.0.1", - "from": "es6-weak-map@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.1.tgz" - }, - "escape-string-regexp": { - "version": "1.0.5", - "from": "escape-string-regexp@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" - }, - "escope": { - "version": "3.6.0", - "from": "escope@>=3.6.0 <4.0.0", - "resolved": "https://registry.npmjs.org/escope/-/escope-3.6.0.tgz" - }, - "eslint": { - "version": "2.5.3", - "from": "eslint@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.5.3.tgz", - "dependencies": { - "glob": { - "version": "7.0.3", - "from": "glob@>=7.0.3 <8.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.0.3.tgz" - }, - "globals": { - "version": "9.2.0", - "from": "globals@>=9.2.0 <10.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-9.2.0.tgz" - }, - "lodash": { - "version": "4.7.0", - "from": "lodash@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.7.0.tgz" - }, - "user-home": { - "version": "2.0.0", - "from": "user-home@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/user-home/-/user-home-2.0.0.tgz" - } - } - }, - "eslint-config-airbnb": { - "version": "6.0.2", - "from": "eslint-config-airbnb@6.0.2", - "resolved": "https://registry.npmjs.org/eslint-config-airbnb/-/eslint-config-airbnb-6.0.2.tgz" - }, - "eslint-loader": { - "version": "1.3.0", - "from": "eslint-loader@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/eslint-loader/-/eslint-loader-1.3.0.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "eslint-plugin-react": { - "version": "4.2.3", - "from": "eslint-plugin-react@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-4.2.3.tgz" - }, - "espree": { - "version": "3.1.3", - "from": "espree@3.1.3", - "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.3.tgz", - "dependencies": { - "acorn": { - "version": "3.0.4", - "from": "acorn@>=3.0.4 <4.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.0.4.tgz" - } - } - }, - "esprima-fb": { - "version": "13001.1001.0-dev-harmony-fb", - "from": "esprima-fb@13001.1001.0-dev-harmony-fb", - "resolved": "https://registry.npmjs.org/esprima-fb/-/esprima-fb-13001.1001.0-dev-harmony-fb.tgz" - }, - "esrecurse": { - "version": "4.1.0", - "from": "esrecurse@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.1.0.tgz", - "dependencies": { - "estraverse": { - "version": "4.1.1", - "from": "estraverse@>=4.1.0 <4.2.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.1.1.tgz" - }, - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "estraverse": { - "version": "4.2.0", - "from": "estraverse@>=4.2.0 <5.0.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz" - }, - "esutils": { - "version": "2.0.2", - "from": "esutils@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz" - }, - "event-emitter": { - "version": "0.3.4", - "from": "event-emitter@>=0.3.4 <0.4.0", - "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.4.tgz" - }, - "eventemitter3": { - "version": "1.2.0", - "from": "eventemitter3@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-1.2.0.tgz" - }, - "events": { - "version": "1.1.0", - "from": "events@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.0.tgz" - }, - "exit": { - "version": "0.1.2", - "from": "exit@0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz" - }, - "exit-hook": { - "version": "1.1.1", - "from": "exit-hook@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz" - }, - "expand-brackets": { - "version": "0.1.4", - "from": "expand-brackets@>=0.1.4 <0.2.0", - "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-0.1.4.tgz" - }, - "expand-range": { - "version": "1.8.1", - "from": "expand-range@>=1.8.1 <2.0.0", - "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-1.8.1.tgz" - }, - "extglob": { - "version": "0.3.2", - "from": "extglob@>=0.3.1 <0.4.0", - "resolved": "https://registry.npmjs.org/extglob/-/extglob-0.3.2.tgz" - }, - "fast-levenshtein": { - "version": "1.1.3", - "from": "fast-levenshtein@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-1.1.3.tgz" - }, - "fastparse": { - "version": "1.1.1", - "from": "fastparse@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/fastparse/-/fastparse-1.1.1.tgz" - }, - "fbjs": { - "version": "0.6.1", - "from": "fbjs@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.6.1.tgz" - }, - "figures": { - "version": "1.5.0", - "from": "figures@>=1.3.5 <2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-1.5.0.tgz" - }, - "file-entry-cache": { - "version": "1.2.4", - "from": "file-entry-cache@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-1.2.4.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "filename-regex": { - "version": "2.0.0", - "from": "filename-regex@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.0.tgz" - }, - "fill-range": { - "version": "2.2.3", - "from": "fill-range@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-2.2.3.tgz" - }, - "flat-cache": { - "version": "1.0.10", - "from": "flat-cache@>=1.0.9 <2.0.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.0.10.tgz" - }, - "flatten": { - "version": "1.0.2", - "from": "flatten@1.0.2", - "resolved": "https://registry.npmjs.org/flatten/-/flatten-1.0.2.tgz" - }, - "for-in": { - "version": "0.1.5", - "from": "for-in@>=0.1.5 <0.2.0", - "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.5.tgz" - }, - "for-own": { - "version": "0.1.4", - "from": "for-own@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.4.tgz" - }, - "fs-readdir-recursive": { - "version": "0.1.2", - "from": "fs-readdir-recursive@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-0.1.2.tgz" - }, - "fsevents": { - "version": "1.0.11", - "from": "fsevents@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.0.11.tgz", - "dependencies": { - "ansi": { - "version": "0.3.1", - "from": "ansi@~0.3.1", - "resolved": "https://registry.npmjs.org/ansi/-/ansi-0.3.1.tgz" - }, - "ansi-regex": { - "version": "2.0.0", - "from": "ansi-regex@^2.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" - }, - "ansi-styles": { - "version": "2.2.1", - "from": "ansi-styles@^2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz" - }, - "are-we-there-yet": { - "version": "1.1.2", - "from": "are-we-there-yet@~1.1.2", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz" - }, - "asn1": { - "version": "0.2.3", - "from": "asn1@>=0.2.3 <0.3.0", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz" - }, - "assert-plus": { - "version": "0.2.0", - "from": "assert-plus@^0.2.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz" - }, - "async": { - "version": "1.5.2", - "from": "async@^1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" - }, - "aws-sign2": { - "version": "0.6.0", - "from": "aws-sign2@~0.6.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz" - }, - "aws4": { - "version": "1.3.2", - "from": "aws4@^1.2.1", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz", - "dependencies": { - "lru-cache": { - "version": "4.0.1", - "from": "lru-cache@^4.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.1.tgz", - "dependencies": { - "pseudomap": { - "version": "1.0.2", - "from": "pseudomap@^1.0.1", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" - }, - "yallist": { - "version": "2.0.0", - "from": "yallist@^2.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz" - } - } - } - } - }, - "bl": { - "version": "1.0.3", - "from": "bl@~1.0.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.0.3.tgz" - }, - "block-stream": { - "version": "0.0.8", - "from": "block-stream@*", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.8.tgz" - }, - "boom": { - "version": "2.10.1", - "from": "boom@2.x.x", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz" - }, - "caseless": { - "version": "0.11.0", - "from": "caseless@~0.11.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" - }, - "chalk": { - "version": "1.1.3", - "from": "chalk@^1.1.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz" - }, - "combined-stream": { - "version": "1.0.5", - "from": "combined-stream@~1.0.5", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz" - }, - "commander": { - "version": "2.9.0", - "from": "commander@^2.9.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz" - }, - "core-util-is": { - "version": "1.0.2", - "from": "core-util-is@~1.0.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" - }, - "cryptiles": { - "version": "2.0.5", - "from": "cryptiles@2.x.x", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz" - }, - "dashdash": { - "version": "1.13.0", - "from": "dashdash@>=1.10.1 <2.0.0", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.13.0.tgz", - "dependencies": { - "assert-plus": { - "version": "1.0.0", - "from": "assert-plus@^1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" - } - } - }, - "debug": { - "version": "2.2.0", - "from": "debug@~2.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz" - }, - "deep-extend": { - "version": "0.4.1", - "from": "deep-extend@~0.4.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.1.tgz" - }, - "delayed-stream": { - "version": "1.0.0", - "from": "delayed-stream@~1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" - }, - "delegates": { - "version": "1.0.0", - "from": "delegates@^1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz" - }, - "ecc-jsbn": { - "version": "0.1.1", - "from": "ecc-jsbn@>=0.0.1 <1.0.0", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz" - }, - "escape-string-regexp": { - "version": "1.0.5", - "from": "escape-string-regexp@^1.0.2", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" - }, - "extend": { - "version": "3.0.0", - "from": "extend@~3.0.0", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" - }, - "extsprintf": { - "version": "1.0.2", - "from": "extsprintf@1.0.2", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz" - }, - "forever-agent": { - "version": "0.6.1", - "from": "forever-agent@~0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" - }, - "form-data": { - "version": "1.0.0-rc4", - "from": "form-data@~1.0.0-rc3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc4.tgz" - }, - "fstream": { - "version": "1.0.8", - "from": "fstream@^1.0.2", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.8.tgz" - }, - "fstream-ignore": { - "version": "1.0.3", - "from": "fstream-ignore@~1.0.3", - "resolved": "https://registry.npmjs.org/fstream-ignore/-/fstream-ignore-1.0.3.tgz", - "dependencies": { - "minimatch": { - "version": "3.0.0", - "from": "minimatch@^3.0.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.0.tgz", - "dependencies": { - "brace-expansion": { - "version": "1.1.3", - "from": "brace-expansion@^1.0.0", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.3.tgz", - "dependencies": { - "balanced-match": { - "version": "0.3.0", - "from": "balanced-match@^0.3.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.3.0.tgz" - }, - "concat-map": { - "version": "0.0.1", - "from": "concat-map@0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" - } - } - } - } - } - } - }, - "gauge": { - "version": "1.2.7", - "from": "gauge@~1.2.5", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-1.2.7.tgz" - }, - "generate-function": { - "version": "2.0.0", - "from": "generate-function@^2.0.0", - "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" - }, - "generate-object-property": { - "version": "1.2.0", - "from": "generate-object-property@^1.1.0", - "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz" - }, - "graceful-fs": { - "version": "4.1.3", - "from": "graceful-fs@^4.1.2", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz" - }, - "graceful-readlink": { - "version": "1.0.1", - "from": "graceful-readlink@>= 1.0.0", - "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz" - }, - "har-validator": { - "version": "2.0.6", - "from": "har-validator@~2.0.6", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz" - }, - "has-ansi": { - "version": "2.0.0", - "from": "has-ansi@^2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz" - }, - "has-unicode": { - "version": "2.0.0", - "from": "has-unicode@^2.0.0", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.0.tgz" - }, - "hawk": { - "version": "3.1.3", - "from": "hawk@~3.1.0", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz" - }, - "hoek": { - "version": "2.16.3", - "from": "hoek@2.x.x", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz" - }, - "http-signature": { - "version": "1.1.1", - "from": "http-signature@~1.1.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz" - }, - "inherits": { - "version": "2.0.1", - "from": "inherits@*", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" - }, - "ini": { - "version": "1.3.4", - "from": "ini@~1.3.0", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.4.tgz" - }, - "is-my-json-valid": { - "version": "2.13.1", - "from": "is-my-json-valid@^2.12.4", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz" - }, - "is-property": { - "version": "1.0.2", - "from": "is-property@^1.0.0", - "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" - }, - "is-typedarray": { - "version": "1.0.0", - "from": "is-typedarray@~1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" - }, - "isarray": { - "version": "1.0.0", - "from": "isarray@~1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" - }, - "isstream": { - "version": "0.1.2", - "from": "isstream@~0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" - }, - "jodid25519": { - "version": "1.0.2", - "from": "jodid25519@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz" - }, - "jsbn": { - "version": "0.1.0", - "from": "jsbn@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" - }, - "json-schema": { - "version": "0.2.2", - "from": "json-schema@0.2.2", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz" - }, - "json-stringify-safe": { - "version": "5.0.1", - "from": "json-stringify-safe@~5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" - }, - "jsonpointer": { - "version": "2.0.0", - "from": "jsonpointer@2.0.0", - "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz" - }, - "jsprim": { - "version": "1.2.2", - "from": "jsprim@^1.2.2", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz" - }, - "lodash.pad": { - "version": "4.1.0", - "from": "lodash.pad@^4.1.0", - "resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.1.0.tgz" - }, - "lodash.padend": { - "version": "4.2.0", - "from": "lodash.padend@^4.1.0", - "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.2.0.tgz" - }, - "lodash.padstart": { - "version": "4.2.0", - "from": "lodash.padstart@^4.1.0", - "resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.2.0.tgz" - }, - "lodash.repeat": { - "version": "4.0.0", - "from": "lodash.repeat@^4.0.0", - "resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.0.0.tgz" - }, - "lodash.tostring": { - "version": "4.1.2", - "from": "lodash.tostring@^4.0.0", - "resolved": "https://registry.npmjs.org/lodash.tostring/-/lodash.tostring-4.1.2.tgz" - }, - "mime-db": { - "version": "1.22.0", - "from": "mime-db@~1.22.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" - }, - "mime-types": { - "version": "2.1.10", - "from": "mime-types@~2.1.7", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz" - }, - "minimist": { - "version": "0.0.8", - "from": "minimist@0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" - }, - "mkdirp": { - "version": "0.5.1", - "from": "mkdirp@>=0.3.0 <0.4.0||>=0.4.0 <0.5.0||>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz" - }, - "ms": { - "version": "0.7.1", - "from": "ms@0.7.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" - }, - "node-pre-gyp": { - "version": "0.6.25", - "from": "node-pre-gyp@0.6.25", - "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.6.25.tgz", - "dependencies": { - "nopt": { - "version": "3.0.6", - "from": "nopt@~3.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "dependencies": { - "abbrev": { - "version": "1.0.7", - "from": "abbrev@1", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.7.tgz" - } - } - } - } - }, - "node-uuid": { - "version": "1.4.7", - "from": "node-uuid@~1.4.7", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" - }, - "npmlog": { - "version": "2.0.3", - "from": "npmlog@~2.0.0", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-2.0.3.tgz" - }, - "oauth-sign": { - "version": "0.8.1", - "from": "oauth-sign@~0.8.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.1.tgz" - }, - "once": { - "version": "1.3.3", - "from": "once@~1.3.3", - "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz" - }, - "pinkie": { - "version": "2.0.4", - "from": "pinkie@^2.0.0", - "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" - }, - "pinkie-promise": { - "version": "2.0.0", - "from": "pinkie-promise@^2.0.0", - "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.0.tgz" - }, - "process-nextick-args": { - "version": "1.0.6", - "from": "process-nextick-args@~1.0.6", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" - }, - "qs": { - "version": "6.0.2", - "from": "qs@~6.0.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.0.2.tgz" - }, - "rc": { - "version": "1.1.6", - "from": "rc@~1.1.0", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.1.6.tgz", - "dependencies": { - "minimist": { - "version": "1.2.0", - "from": "minimist@^1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz" - } - } - }, - "readable-stream": { - "version": "2.0.6", - "from": "readable-stream@^2.0.0 || ^1.1.13", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz" - }, - "request": { - "version": "2.69.0", - "from": "request@2.x", - "resolved": "https://registry.npmjs.org/request/-/request-2.69.0.tgz" - }, - "rimraf": { - "version": "2.5.2", - "from": "rimraf@~2.5.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.2.tgz", - "dependencies": { - "glob": { - "version": "7.0.3", - "from": "glob@^7.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.0.3.tgz", - "dependencies": { - "inflight": { - "version": "1.0.4", - "from": "inflight@^1.0.4", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.4.tgz", - "dependencies": { - "wrappy": { - "version": "1.0.1", - "from": "wrappy@1", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" - } - } - }, - "inherits": { - "version": "2.0.1", - "from": "inherits@2", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" - }, - "minimatch": { - "version": "3.0.0", - "from": "minimatch@2 || 3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.0.tgz", - "dependencies": { - "brace-expansion": { - "version": "1.1.3", - "from": "brace-expansion@^1.0.0", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.3.tgz", - "dependencies": { - "balanced-match": { - "version": "0.3.0", - "from": "balanced-match@^0.3.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.3.0.tgz" - }, - "concat-map": { - "version": "0.0.1", - "from": "concat-map@0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" - } - } - } - } - }, - "once": { - "version": "1.3.3", - "from": "once@^1.3.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", - "dependencies": { - "wrappy": { - "version": "1.0.1", - "from": "wrappy@1", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" - } - } - }, - "path-is-absolute": { - "version": "1.0.0", - "from": "path-is-absolute@^1.0.0", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.0.tgz" - } - } - } - } - }, - "semver": { - "version": "5.1.0", - "from": "semver@~5.1.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz" - }, - "sntp": { - "version": "1.0.9", - "from": "sntp@1.x.x", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" - }, - "sshpk": { - "version": "1.7.4", - "from": "sshpk@^1.7.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.7.4.tgz" - }, - "string_decoder": { - "version": "0.10.31", - "from": "string_decoder@~0.10.x", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" - }, - "stringstream": { - "version": "0.0.5", - "from": "stringstream@~0.0.4", - "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" - }, - "strip-ansi": { - "version": "3.0.1", - "from": "strip-ansi@^3.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz" - }, - "strip-json-comments": { - "version": "1.0.4", - "from": "strip-json-comments@~1.0.4", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz" - }, - "supports-color": { - "version": "2.0.0", - "from": "supports-color@^2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" - }, - "tar": { - "version": "2.2.1", - "from": "tar@~2.2.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz" - }, - "tar-pack": { - "version": "3.1.3", - "from": "tar-pack@~3.1.0", - "resolved": "https://registry.npmjs.org/tar-pack/-/tar-pack-3.1.3.tgz" - }, - "tough-cookie": { - "version": "2.2.2", - "from": "tough-cookie@~2.2.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz" - }, - "tunnel-agent": { - "version": "0.4.2", - "from": "tunnel-agent@~0.4.1", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.2.tgz" - }, - "tweetnacl": { - "version": "0.14.3", - "from": "tweetnacl@>=0.13.0 <1.0.0", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.3.tgz" - }, - "uid-number": { - "version": "0.0.6", - "from": "uid-number@~0.0.6", - "resolved": "https://registry.npmjs.org/uid-number/-/uid-number-0.0.6.tgz" - }, - "util-deprecate": { - "version": "1.0.2", - "from": "util-deprecate@~1.0.1", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" - }, - "verror": { - "version": "1.3.6", - "from": "verror@1.3.6", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz" - }, - "wrappy": { - "version": "1.0.1", - "from": "wrappy@1", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" - }, - "xtend": { - "version": "4.0.1", - "from": "xtend@^4.0.0", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" - } - } - }, - "generate-function": { - "version": "2.0.0", - "from": "generate-function@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" - }, - "generate-object-property": { - "version": "1.2.0", - "from": "generate-object-property@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz" - }, - "get-stdin": { - "version": "4.0.1", - "from": "get-stdin@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz" - }, - "glob": { - "version": "5.0.15", - "from": "glob@>=5.0.15 <6.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz" - }, - "glob-base": { - "version": "0.3.0", - "from": "glob-base@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz" - }, - "glob-parent": { - "version": "2.0.0", - "from": "glob-parent@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz" - }, - "globals": { - "version": "6.4.1", - "from": "globals@>=6.4.0 <7.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-6.4.1.tgz" - }, - "globby": { - "version": "4.0.0", - "from": "globby@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-4.0.0.tgz", - "dependencies": { - "glob": { - "version": "6.0.4", - "from": "glob@>=6.0.1 <7.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz" - }, - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "graceful-fs": { - "version": "4.1.3", - "from": "graceful-fs@>=4.1.2 <5.0.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz" - }, - "graceful-readlink": { - "version": "1.0.1", - "from": "graceful-readlink@>=1.0.0", - "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz" - }, - "graylog-web-manifests": { - "version": "2.0.0-alpha.3", - "from": "graylog-web-manifests@>=2.0.0-alpha.3 <3.0.0", - "resolved": "https://registry.npmjs.org/graylog-web-manifests/-/graylog-web-manifests-2.0.0-alpha.3.tgz" - }, - "graylog-web-plugin": { - "version": "0.0.20", - "from": "graylog-web-plugin@latest" - }, - "has-ansi": { - "version": "2.0.0", - "from": "has-ansi@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz" - }, - "has-flag": { - "version": "1.0.0", - "from": "has-flag@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz" - }, - "has-own": { - "version": "1.0.0", - "from": "has-own@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/has-own/-/has-own-1.0.0.tgz" - }, - "history": { - "version": "1.17.0", - "from": "history@>=1.17.0 <2.0.0", - "resolved": "https://registry.npmjs.org/history/-/history-1.17.0.tgz" - }, - "home-or-tmp": { - "version": "1.0.0", - "from": "home-or-tmp@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-1.0.0.tgz" - }, - "html-minifier": { - "version": "1.4.0", - "from": "html-minifier@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-1.4.0.tgz" - }, - "html-webpack-plugin": { - "version": "1.7.0", - "from": "html-webpack-plugin@>=1.7.0 <2.0.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-1.7.0.tgz", - "dependencies": { - "bluebird": { - "version": "3.3.4", - "from": "bluebird@>=3.0.5 <4.0.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.3.4.tgz" - } - } - }, - "http-browserify": { - "version": "1.7.0", - "from": "http-browserify@>=1.3.2 <2.0.0", - "resolved": "https://registry.npmjs.org/http-browserify/-/http-browserify-1.7.0.tgz" - }, - "https-browserify": { - "version": "0.0.0", - "from": "https-browserify@0.0.0", - "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-0.0.0.tgz" - }, - "iconv-lite": { - "version": "0.4.13", - "from": "iconv-lite@>=0.4.5 <0.5.0", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.13.tgz" - }, - "icss-replace-symbols": { - "version": "1.0.2", - "from": "icss-replace-symbols@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.0.2.tgz" - }, - "ieee754": { - "version": "1.1.6", - "from": "ieee754@>=1.1.4 <2.0.0", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.6.tgz" - }, - "ignore": { - "version": "3.0.14", - "from": "ignore@>=3.0.10 <4.0.0", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.0.14.tgz" - }, - "imurmurhash": { - "version": "0.1.4", - "from": "imurmurhash@>=0.1.4 <0.2.0", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz" - }, - "indexes-of": { - "version": "1.0.1", - "from": "indexes-of@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/indexes-of/-/indexes-of-1.0.1.tgz" - }, - "indexof": { - "version": "0.0.1", - "from": "indexof@0.0.1", - "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz" - }, - "inflight": { - "version": "1.0.4", - "from": "inflight@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.4.tgz" - }, - "inherits": { - "version": "2.0.1", - "from": "inherits@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" - }, - "inquirer": { - "version": "0.12.0", - "from": "inquirer@>=0.12.0 <0.13.0", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-0.12.0.tgz", - "dependencies": { - "lodash": { - "version": "4.7.0", - "from": "lodash@>=4.3.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.7.0.tgz" - } - } - }, - "interpret": { - "version": "0.6.6", - "from": "interpret@>=0.6.4 <0.7.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-0.6.6.tgz" - }, - "invariant": { - "version": "2.2.1", - "from": "invariant@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.1.tgz" - }, - "invert-kv": { - "version": "1.0.0", - "from": "invert-kv@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz" - }, - "is-absolute-url": { - "version": "2.0.0", - "from": "is-absolute-url@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-2.0.0.tgz" - }, - "is-binary-path": { - "version": "1.0.1", - "from": "is-binary-path@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz" - }, - "is-buffer": { - "version": "1.1.3", - "from": "is-buffer@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.3.tgz" - }, - "is-dotfile": { - "version": "1.0.2", - "from": "is-dotfile@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-dotfile/-/is-dotfile-1.0.2.tgz" - }, - "is-equal-shallow": { - "version": "0.1.3", - "from": "is-equal-shallow@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz" - }, - "is-extendable": { - "version": "0.1.1", - "from": "is-extendable@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz" - }, - "is-extglob": { - "version": "1.0.0", - "from": "is-extglob@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz" - }, - "is-finite": { - "version": "1.0.1", - "from": "is-finite@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.1.tgz" - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "from": "is-fullwidth-code-point@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz" - }, - "is-glob": { - "version": "2.0.1", - "from": "is-glob@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz" - }, - "is-integer": { - "version": "1.0.6", - "from": "is-integer@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/is-integer/-/is-integer-1.0.6.tgz" - }, - "is-lower-case": { - "version": "1.1.3", - "from": "is-lower-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-lower-case/-/is-lower-case-1.1.3.tgz" - }, - "is-my-json-valid": { - "version": "2.13.1", - "from": "is-my-json-valid@>=2.10.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz" - }, - "is-number": { - "version": "2.1.0", - "from": "is-number@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-2.1.0.tgz" - }, - "is-path-cwd": { - "version": "1.0.0", - "from": "is-path-cwd@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz" - }, - "is-path-in-cwd": { - "version": "1.0.0", - "from": "is-path-in-cwd@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.0.tgz" - }, - "is-path-inside": { - "version": "1.0.0", - "from": "is-path-inside@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.0.tgz" - }, - "is-plain-obj": { - "version": "1.1.0", - "from": "is-plain-obj@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz" - }, - "is-primitive": { - "version": "2.0.0", - "from": "is-primitive@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-2.0.0.tgz" - }, - "is-property": { - "version": "1.0.2", - "from": "is-property@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" - }, - "is-resolvable": { - "version": "1.0.0", - "from": "is-resolvable@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.0.0.tgz" - }, - "is-svg": { - "version": "1.1.1", - "from": "is-svg@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/is-svg/-/is-svg-1.1.1.tgz" - }, - "is-upper-case": { - "version": "1.1.2", - "from": "is-upper-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/is-upper-case/-/is-upper-case-1.1.2.tgz" - }, - "isarray": { - "version": "1.0.0", - "from": "isarray@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" - }, - "isobject": { - "version": "2.0.0", - "from": "isobject@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.0.0.tgz", - "dependencies": { - "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" - } - } - }, - "javascript-natural-sort": { - "version": "0.7.1", - "from": "javascript-natural-sort@>=0.7.1 <0.8.0", - "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz" - }, - "js-base64": { - "version": "2.1.9", - "from": "js-base64@>=2.1.9 <3.0.0", - "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.1.9.tgz" - }, - "js-tokens": { - "version": "1.0.3", - "from": "js-tokens@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-1.0.3.tgz" - }, - "js-yaml": { - "version": "3.5.5", - "from": "js-yaml@>=3.5.3 <3.6.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.5.5.tgz", - "dependencies": { - "esprima": { - "version": "2.7.2", - "from": "esprima@>=2.6.0 <3.0.0" - } - } - }, - "jsesc": { - "version": "0.5.0", - "from": "jsesc@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz" - }, - "json-loader": { - "version": "0.5.4", - "from": "json-loader@>=0.5.4 <0.6.0", - "resolved": "https://registry.npmjs.org/json-loader/-/json-loader-0.5.4.tgz" - }, - "json-stable-stringify": { - "version": "1.0.1", - "from": "json-stable-stringify@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz" - }, - "json5": { - "version": "0.4.0", - "from": "json5@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.4.0.tgz" - }, - "jsonify": { - "version": "0.0.0", - "from": "jsonify@>=0.0.0 <0.1.0", - "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz" - }, - "jsonpointer": { - "version": "2.0.0", - "from": "jsonpointer@2.0.0", - "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz" - }, - "jstransform": { - "version": "10.1.0", - "from": "jstransform@>=10.0.1 <11.0.0", - "resolved": "https://registry.npmjs.org/jstransform/-/jstransform-10.1.0.tgz" - }, - "keycode": { - "version": "2.1.1", - "from": "keycode@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/keycode/-/keycode-2.1.1.tgz" - }, - "kind-of": { - "version": "3.0.2", - "from": "kind-of@>=3.0.2 <4.0.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.0.2.tgz" - }, - "lazy-cache": { - "version": "1.0.3", - "from": "lazy-cache@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.3.tgz" - }, - "lcid": { - "version": "1.0.0", - "from": "lcid@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz" - }, - "leven": { - "version": "1.0.2", - "from": "leven@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-1.0.2.tgz" - }, - "levn": { - "version": "0.3.0", - "from": "levn@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz" - }, - "loader-utils": { - "version": "0.2.13", - "from": "loader-utils@>=0.2.9 <0.3.0", - "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.13.tgz" - }, - "lodash": { - "version": "3.10.1", - "from": "lodash@>=3.10.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz" - }, - "lodash-compat": { - "version": "3.10.2", - "from": "lodash-compat@>=3.10.1 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash-compat/-/lodash-compat-3.10.2.tgz" - }, - "lodash._arraycopy": { - "version": "3.0.0", - "from": "lodash._arraycopy@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._arraycopy/-/lodash._arraycopy-3.0.0.tgz" - }, - "lodash._arrayeach": { - "version": "3.0.0", - "from": "lodash._arrayeach@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._arrayeach/-/lodash._arrayeach-3.0.0.tgz" - }, - "lodash._basecallback": { - "version": "3.3.1", - "from": "lodash._basecallback@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basecallback/-/lodash._basecallback-3.3.1.tgz" - }, - "lodash._basecopy": { - "version": "3.0.1", - "from": "lodash._basecopy@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz" - }, - "lodash._baseeach": { - "version": "3.0.4", - "from": "lodash._baseeach@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._baseeach/-/lodash._baseeach-3.0.4.tgz" - }, - "lodash._basefind": { - "version": "3.0.0", - "from": "lodash._basefind@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basefind/-/lodash._basefind-3.0.0.tgz" - }, - "lodash._basefindindex": { - "version": "3.6.0", - "from": "lodash._basefindindex@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basefindindex/-/lodash._basefindindex-3.6.0.tgz" - }, - "lodash._basefor": { - "version": "3.0.3", - "from": "lodash._basefor@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._basefor/-/lodash._basefor-3.0.3.tgz" - }, - "lodash._baseisequal": { - "version": "3.0.7", - "from": "lodash._baseisequal@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._baseisequal/-/lodash._baseisequal-3.0.7.tgz" - }, - "lodash._bindcallback": { - "version": "3.0.1", - "from": "lodash._bindcallback@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz" - }, - "lodash._createassigner": { - "version": "3.1.1", - "from": "lodash._createassigner@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._createassigner/-/lodash._createassigner-3.1.1.tgz" - }, - "lodash._createcompounder": { - "version": "3.0.0", - "from": "lodash._createcompounder@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._createcompounder/-/lodash._createcompounder-3.0.0.tgz" - }, - "lodash._getnative": { - "version": "3.9.1", - "from": "lodash._getnative@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._getnative/-/lodash._getnative-3.9.1.tgz" - }, - "lodash._isiterateecall": { - "version": "3.0.9", - "from": "lodash._isiterateecall@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz" - }, - "lodash._root": { - "version": "3.0.1", - "from": "lodash._root@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz" - }, - "lodash.camelcase": { - "version": "3.0.1", - "from": "lodash.camelcase@>=3.0.1 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-3.0.1.tgz" - }, - "lodash.deburr": { - "version": "3.2.0", - "from": "lodash.deburr@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-3.2.0.tgz" - }, - "lodash.find": { - "version": "3.2.1", - "from": "lodash.find@>=3.2.1 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-3.2.1.tgz" - }, - "lodash.isarguments": { - "version": "3.0.8", - "from": "lodash.isarguments@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.0.8.tgz" - }, - "lodash.isarray": { - "version": "3.0.4", - "from": "lodash.isarray@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz" - }, - "lodash.isplainobject": { - "version": "3.2.0", - "from": "lodash.isplainobject@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-3.2.0.tgz" - }, - "lodash.istypedarray": { - "version": "3.0.5", - "from": "lodash.istypedarray@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.istypedarray/-/lodash.istypedarray-3.0.5.tgz" - }, - "lodash.keys": { - "version": "3.1.2", - "from": "lodash.keys@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz" - }, - "lodash.keysin": { - "version": "3.0.8", - "from": "lodash.keysin@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.keysin/-/lodash.keysin-3.0.8.tgz" - }, - "lodash.merge": { - "version": "3.3.2", - "from": "lodash.merge@>=3.3.2 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-3.3.2.tgz" - }, - "lodash.pairs": { - "version": "3.0.1", - "from": "lodash.pairs@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.pairs/-/lodash.pairs-3.0.1.tgz" - }, - "lodash.restparam": { - "version": "3.6.1", - "from": "lodash.restparam@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.restparam/-/lodash.restparam-3.6.1.tgz" - }, - "lodash.toplainobject": { - "version": "3.0.0", - "from": "lodash.toplainobject@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.toplainobject/-/lodash.toplainobject-3.0.0.tgz" - }, - "lodash.words": { - "version": "3.2.0", - "from": "lodash.words@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/lodash.words/-/lodash.words-3.2.0.tgz" - }, - "longest": { - "version": "1.0.1", - "from": "longest@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz" - }, - "loose-envify": { - "version": "1.1.0", - "from": "loose-envify@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.1.0.tgz" - }, - "lower-case": { - "version": "1.1.3", - "from": "lower-case@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-1.1.3.tgz" - }, - "lower-case-first": { - "version": "1.0.2", - "from": "lower-case-first@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/lower-case-first/-/lower-case-first-1.0.2.tgz" - }, - "memory-fs": { - "version": "0.3.0", - "from": "memory-fs@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.3.0.tgz" - }, - "micromatch": { - "version": "2.3.7", - "from": "micromatch@>=2.1.5 <3.0.0", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-2.3.7.tgz" - }, - "minimatch": { - "version": "2.0.10", - "from": "minimatch@>=2.0.3 <3.0.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-2.0.10.tgz" - }, - "minimist": { - "version": "1.2.0", - "from": "minimist@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz" - }, - "mkdirp": { - "version": "0.5.1", - "from": "mkdirp@>=0.5.1 <0.6.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "dependencies": { - "minimist": { - "version": "0.0.8", - "from": "minimist@0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" - } - } - }, - "ms": { - "version": "0.7.1", - "from": "ms@0.7.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" - }, - "mute-stream": { - "version": "0.0.5", - "from": "mute-stream@0.0.5", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.5.tgz" - }, - "nan": { - "version": "2.2.1", - "from": "nan@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.2.1.tgz" - }, - "ncname": { - "version": "1.0.0", - "from": "ncname@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/ncname/-/ncname-1.0.0.tgz" - }, - "node-libs-browser": { - "version": "0.5.3", - "from": "node-libs-browser@>=0.4.0 <=0.6.0", - "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-0.5.3.tgz", - "dependencies": { - "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" - }, - "readable-stream": { - "version": "1.1.13", - "from": "readable-stream@>=1.1.13 <2.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.13.tgz" - } - } - }, - "normalize-path": { - "version": "2.0.1", - "from": "normalize-path@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.0.1.tgz" - }, - "normalize-range": { - "version": "0.1.2", - "from": "normalize-range@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz" - }, - "normalize-url": { - "version": "1.4.1", - "from": "normalize-url@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.4.1.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "num2fraction": { - "version": "1.2.2", - "from": "num2fraction@>=1.2.2 <2.0.0", - "resolved": "https://registry.npmjs.org/num2fraction/-/num2fraction-1.2.2.tgz" - }, - "number-is-nan": { - "version": "1.0.0", - "from": "number-is-nan@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.0.tgz" - }, - "object-assign": { - "version": "3.0.0", - "from": "object-assign@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-3.0.0.tgz" - }, - "object.omit": { - "version": "2.0.0", - "from": "object.omit@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/object.omit/-/object.omit-2.0.0.tgz" - }, - "once": { - "version": "1.3.3", - "from": "once@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz" - }, - "onetime": { - "version": "1.1.0", - "from": "onetime@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz" - }, - "optimist": { - "version": "0.6.1", - "from": "optimist@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", - "dependencies": { - "minimist": { - "version": "0.0.10", - "from": "minimist@>=0.0.1 <0.1.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz" - } - } - }, - "optionator": { - "version": "0.8.1", - "from": "optionator@>=0.8.1 <0.9.0", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.1.tgz", - "dependencies": { - "wordwrap": { - "version": "1.0.0", - "from": "wordwrap@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz" - } - } - }, - "os-browserify": { - "version": "0.1.2", - "from": "os-browserify@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.1.2.tgz" - }, - "os-homedir": { - "version": "1.0.1", - "from": "os-homedir@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.1.tgz" - }, - "os-locale": { - "version": "1.4.0", - "from": "os-locale@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz" - }, - "os-tmpdir": { - "version": "1.0.1", - "from": "os-tmpdir@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.1.tgz" - }, - "output-file-sync": { - "version": "1.1.1", - "from": "output-file-sync@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/output-file-sync/-/output-file-sync-1.1.1.tgz" - }, - "pako": { - "version": "0.2.8", - "from": "pako@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.8.tgz" - }, - "param-case": { - "version": "1.1.2", - "from": "param-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/param-case/-/param-case-1.1.2.tgz" - }, - "parse-glob": { - "version": "3.0.4", - "from": "parse-glob@>=3.0.4 <4.0.0", - "resolved": "https://registry.npmjs.org/parse-glob/-/parse-glob-3.0.4.tgz" - }, - "pascal-case": { - "version": "1.1.2", - "from": "pascal-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-1.1.2.tgz" - }, - "path-browserify": { - "version": "0.0.0", - "from": "path-browserify@0.0.0", - "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz" - }, - "path-case": { - "version": "1.1.2", - "from": "path-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/path-case/-/path-case-1.1.2.tgz" - }, - "path-exists": { - "version": "1.0.0", - "from": "path-exists@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-1.0.0.tgz" - }, - "path-is-absolute": { - "version": "1.0.0", - "from": "path-is-absolute@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.0.tgz" - }, - "path-is-inside": { - "version": "1.0.1", - "from": "path-is-inside@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.1.tgz" - }, - "pbkdf2-compat": { - "version": "2.0.1", - "from": "pbkdf2-compat@2.0.1", - "resolved": "https://registry.npmjs.org/pbkdf2-compat/-/pbkdf2-compat-2.0.1.tgz" - }, - "pify": { - "version": "2.3.0", - "from": "pify@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" - }, - "pinkie": { - "version": "2.0.4", - "from": "pinkie@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" - }, - "pinkie-promise": { - "version": "2.0.0", - "from": "pinkie-promise@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.0.tgz" - }, - "pluralize": { - "version": "1.2.1", - "from": "pluralize@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-1.2.1.tgz" - }, - "postcss": { - "version": "5.0.19", - "from": "postcss@>=5.0.6 <6.0.0", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.0.19.tgz", - "dependencies": { - "source-map": { - "version": "0.5.3", - "from": "source-map@>=0.5.1 <0.6.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" - }, - "supports-color": { - "version": "3.1.2", - "from": "supports-color@>=3.1.2 <4.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz" - } - } - }, - "postcss-calc": { - "version": "5.2.0", - "from": "postcss-calc@>=5.2.0 <6.0.0", - "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-5.2.0.tgz" - }, - "postcss-colormin": { - "version": "2.2.0", - "from": "postcss-colormin@>=2.1.8 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-2.2.0.tgz" - }, - "postcss-convert-values": { - "version": "2.3.4", - "from": "postcss-convert-values@>=2.3.4 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-2.3.4.tgz" - }, - "postcss-discard-comments": { - "version": "2.0.4", - "from": "postcss-discard-comments@>=2.0.4 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-2.0.4.tgz" - }, - "postcss-discard-duplicates": { - "version": "2.0.1", - "from": "postcss-discard-duplicates@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-2.0.1.tgz" - }, - "postcss-discard-empty": { - "version": "2.0.1", - "from": "postcss-discard-empty@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-2.0.1.tgz" - }, - "postcss-discard-unused": { - "version": "2.2.1", - "from": "postcss-discard-unused@>=2.2.1 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-discard-unused/-/postcss-discard-unused-2.2.1.tgz" - }, - "postcss-filter-plugins": { - "version": "2.0.0", - "from": "postcss-filter-plugins@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-filter-plugins/-/postcss-filter-plugins-2.0.0.tgz" - }, - "postcss-merge-idents": { - "version": "2.1.5", - "from": "postcss-merge-idents@>=2.1.5 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-merge-idents/-/postcss-merge-idents-2.1.5.tgz" - }, - "postcss-merge-longhand": { - "version": "2.0.1", - "from": "postcss-merge-longhand@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-2.0.1.tgz" - }, - "postcss-merge-rules": { - "version": "2.0.6", - "from": "postcss-merge-rules@>=2.0.3 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-2.0.6.tgz" - }, - "postcss-message-helpers": { - "version": "2.0.0", - "from": "postcss-message-helpers@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-message-helpers/-/postcss-message-helpers-2.0.0.tgz" - }, - "postcss-minify-font-values": { - "version": "1.0.3", - "from": "postcss-minify-font-values@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-1.0.3.tgz", - "dependencies": { - "object-assign": { - "version": "4.0.1", - "from": "object-assign@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.0.1.tgz" - } - } - }, - "postcss-minify-gradients": { - "version": "1.0.1", - "from": "postcss-minify-gradients@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-1.0.1.tgz" - }, - "postcss-minify-params": { - "version": "1.0.4", - "from": "postcss-minify-params@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-1.0.4.tgz" - }, - "postcss-minify-selectors": { - "version": "2.0.4", - "from": "postcss-minify-selectors@>=2.0.4 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-2.0.4.tgz" - }, - "postcss-modules-extract-imports": { - "version": "1.0.0", - "from": "postcss-modules-extract-imports@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-1.0.0.tgz" - }, - "postcss-modules-local-by-default": { - "version": "1.0.1", - "from": "postcss-modules-local-by-default@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-1.0.1.tgz" - }, - "postcss-modules-scope": { - "version": "1.0.0", - "from": "postcss-modules-scope@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-1.0.0.tgz" - }, - "postcss-modules-values": { - "version": "1.1.2", - "from": "postcss-modules-values@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-1.1.2.tgz" - }, - "postcss-normalize-charset": { - "version": "1.1.0", - "from": "postcss-normalize-charset@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-1.1.0.tgz" - }, - "postcss-normalize-url": { - "version": "3.0.7", - "from": "postcss-normalize-url@>=3.0.7 <4.0.0", - "resolved": "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-3.0.7.tgz" - }, - "postcss-ordered-values": { - "version": "2.1.0", - "from": "postcss-ordered-values@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-2.1.0.tgz" - }, - "postcss-reduce-idents": { - "version": "2.3.0", - "from": "postcss-reduce-idents@>=2.2.2 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-reduce-idents/-/postcss-reduce-idents-2.3.0.tgz" - }, - "postcss-reduce-transforms": { - "version": "1.0.3", - "from": "postcss-reduce-transforms@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-1.0.3.tgz" - }, - "postcss-selector-parser": { - "version": "1.3.3", - "from": "postcss-selector-parser@>=1.3.1 <2.0.0", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-1.3.3.tgz" - }, - "postcss-svgo": { - "version": "2.1.2", - "from": "postcss-svgo@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-2.1.2.tgz" - }, - "postcss-unique-selectors": { - "version": "2.0.2", - "from": "postcss-unique-selectors@>=2.0.2 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-2.0.2.tgz" - }, - "postcss-value-parser": { - "version": "3.3.0", - "from": "postcss-value-parser@>=3.2.3 <4.0.0", - "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.0.tgz" - }, - "postcss-zindex": { - "version": "2.0.1", - "from": "postcss-zindex@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-2.0.1.tgz" - }, - "prelude-ls": { - "version": "1.1.2", - "from": "prelude-ls@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz" - }, - "prepend-http": { - "version": "1.0.3", - "from": "prepend-http@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.3.tgz" - }, - "preserve": { - "version": "0.2.0", - "from": "preserve@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz" - }, - "private": { - "version": "0.1.6", - "from": "private@>=0.1.6 <0.2.0", - "resolved": "https://registry.npmjs.org/private/-/private-0.1.6.tgz" - }, - "process": { - "version": "0.11.2", - "from": "process@>=0.11.0 <0.12.0", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.2.tgz" - }, - "process-nextick-args": { - "version": "1.0.6", - "from": "process-nextick-args@>=1.0.6 <1.1.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" - }, - "progress": { - "version": "1.1.8", - "from": "progress@>=1.1.8 <2.0.0", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz" - }, - "promise": { - "version": "7.1.1", - "from": "promise@>=7.0.3 <8.0.0", - "resolved": "https://registry.npmjs.org/promise/-/promise-7.1.1.tgz" - }, - "prr": { - "version": "0.0.0", - "from": "prr@>=0.0.0 <0.1.0", - "resolved": "https://registry.npmjs.org/prr/-/prr-0.0.0.tgz" - }, - "punycode": { - "version": "1.4.1", - "from": "punycode@>=1.2.4 <2.0.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz" - }, - "q": { - "version": "1.4.1", - "from": "q@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/q/-/q-1.4.1.tgz" - }, - "query-string": { - "version": "3.0.3", - "from": "query-string@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-3.0.3.tgz" - }, - "querystring": { - "version": "0.2.0", - "from": "querystring@0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz" - }, - "querystring-es3": { - "version": "0.2.1", - "from": "querystring-es3@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz" - }, - "randomatic": { - "version": "1.1.5", - "from": "randomatic@>=1.1.3 <2.0.0", - "resolved": "https://registry.npmjs.org/randomatic/-/randomatic-1.1.5.tgz" - }, - "react": { - "version": "0.14.8", - "from": "react@>=0.14.0 <0.15.0", - "resolved": "https://registry.npmjs.org/react/-/react-0.14.8.tgz" - }, - "react-ace": { - "version": "3.2.0", - "from": "react-ace@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/react-ace/-/react-ace-3.2.0.tgz" - }, - "react-bootstrap": { - "version": "0.28.4", - "from": "react-bootstrap@>=0.28.1 <0.29.0", - "resolved": "https://registry.npmjs.org/react-bootstrap/-/react-bootstrap-0.28.4.tgz" - }, - "react-dom": { - "version": "0.14.8", - "from": "react-dom@>=0.14.5 <0.15.0", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-0.14.8.tgz" - }, - "react-hot-api": { - "version": "0.4.7", - "from": "react-hot-api@>=0.4.5 <0.5.0", - "resolved": "https://registry.npmjs.org/react-hot-api/-/react-hot-api-0.4.7.tgz" - }, - "react-hot-loader": { - "version": "1.3.0", - "from": "react-hot-loader@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/react-hot-loader/-/react-hot-loader-1.3.0.tgz", - "dependencies": { - "source-map": { - "version": "0.4.4", - "from": "source-map@>=0.4.4 <0.5.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz" - } - } - }, - "react-overlays": { - "version": "0.6.1", - "from": "react-overlays@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/react-overlays/-/react-overlays-0.6.1.tgz", - "dependencies": { - "react-prop-types": { - "version": "0.2.2", - "from": "react-prop-types@>=0.2.1 <0.3.0", - "resolved": "https://registry.npmjs.org/react-prop-types/-/react-prop-types-0.2.2.tgz" - } - } - }, - "react-prop-types": { - "version": "0.3.0", - "from": "react-prop-types@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/react-prop-types/-/react-prop-types-0.3.0.tgz" - }, - "react-router": { - "version": "1.0.3", - "from": "react-router@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-1.0.3.tgz" - }, - "react-router-bootstrap": { - "version": "0.19.3", - "from": "react-router-bootstrap@>=0.19.0 <0.20.0", - "resolved": "https://registry.npmjs.org/react-router-bootstrap/-/react-router-bootstrap-0.19.3.tgz" - }, - "read-json-sync": { - "version": "1.1.1", - "from": "read-json-sync@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/read-json-sync/-/read-json-sync-1.1.1.tgz" - }, - "readable-stream": { - "version": "2.0.6", - "from": "readable-stream@>=2.0.0 <2.1.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz" - }, - "readdirp": { - "version": "2.0.0", - "from": "readdirp@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.0.0.tgz" - }, - "readline2": { - "version": "1.0.1", - "from": "readline2@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/readline2/-/readline2-1.0.1.tgz" - }, - "recast": { - "version": "0.10.33", - "from": "recast@0.10.33", - "resolved": "https://registry.npmjs.org/recast/-/recast-0.10.33.tgz", - "dependencies": { - "esprima-fb": { - "version": "15001.1001.0-dev-harmony-fb", - "from": "esprima-fb@>=15001.1001.0-dev-harmony-fb <15001.1002.0", - "resolved": "https://registry.npmjs.org/esprima-fb/-/esprima-fb-15001.1001.0-dev-harmony-fb.tgz" - }, - "source-map": { - "version": "0.5.3", - "from": "source-map@>=0.5.0 <0.6.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" - } - } - }, - "reduce-css-calc": { - "version": "1.2.1", - "from": "reduce-css-calc@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/reduce-css-calc/-/reduce-css-calc-1.2.1.tgz", - "dependencies": { - "balanced-match": { - "version": "0.1.0", - "from": "balanced-match@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.1.0.tgz" - } - } - }, - "reduce-function-call": { - "version": "1.0.1", - "from": "reduce-function-call@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/reduce-function-call/-/reduce-function-call-1.0.1.tgz", - "dependencies": { - "balanced-match": { - "version": "0.1.0", - "from": "balanced-match@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.1.0.tgz" - } - } - }, - "reflux": { - "version": "0.2.13", - "from": "reflux@>=0.2.12 <0.3.0", - "resolved": "https://registry.npmjs.org/reflux/-/reflux-0.2.13.tgz" - }, - "reflux-core": { - "version": "0.2.1", - "from": "reflux-core@>=0.2.1 <0.3.0", - "resolved": "https://registry.npmjs.org/reflux-core/-/reflux-core-0.2.1.tgz" - }, - "regenerate": { - "version": "1.2.1", - "from": "regenerate@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.2.1.tgz" - }, - "regenerator": { - "version": "0.8.40", - "from": "regenerator@0.8.40", - "resolved": "https://registry.npmjs.org/regenerator/-/regenerator-0.8.40.tgz", - "dependencies": { - "esprima-fb": { - "version": "15001.1001.0-dev-harmony-fb", - "from": "esprima-fb@>=15001.1001.0-dev-harmony-fb <15001.1002.0", - "resolved": "https://registry.npmjs.org/esprima-fb/-/esprima-fb-15001.1001.0-dev-harmony-fb.tgz" - } - } - }, - "regex-cache": { - "version": "0.4.2", - "from": "regex-cache@>=0.4.2 <0.5.0", - "resolved": "https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.2.tgz" - }, - "regexpu": { - "version": "1.3.0", - "from": "regexpu@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/regexpu/-/regexpu-1.3.0.tgz", - "dependencies": { - "esprima": { - "version": "2.7.2", - "from": "esprima@>=2.6.0 <3.0.0", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz" - } - } - }, - "regjsgen": { - "version": "0.2.0", - "from": "regjsgen@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz" - }, - "regjsparser": { - "version": "0.1.5", - "from": "regjsparser@>=0.1.4 <0.2.0", - "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz" - }, - "relateurl": { - "version": "0.2.6", - "from": "relateurl@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.6.tgz" - }, - "repeat-element": { - "version": "1.1.2", - "from": "repeat-element@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.2.tgz" - }, - "repeat-string": { - "version": "1.5.4", - "from": "repeat-string@>=1.5.2 <2.0.0", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.5.4.tgz" - }, - "repeating": { - "version": "1.1.3", - "from": "repeating@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/repeating/-/repeating-1.1.3.tgz" - }, - "require-uncached": { - "version": "1.0.2", - "from": "require-uncached@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.2.tgz" - }, - "resolve": { - "version": "1.1.7", - "from": "resolve@>=1.1.6 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz" - }, - "resolve-from": { - "version": "1.0.1", - "from": "resolve-from@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz" - }, - "restore-cursor": { - "version": "1.0.1", - "from": "restore-cursor@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz" - }, - "right-align": { - "version": "0.1.3", - "from": "right-align@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz" - }, - "rimraf": { - "version": "2.5.2", - "from": "rimraf@>=2.2.8 <3.0.0", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.2.tgz", - "dependencies": { - "glob": { - "version": "7.0.3", - "from": "glob@>=7.0.0 <8.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.0.3.tgz" - } - } - }, - "ripemd160": { - "version": "0.2.0", - "from": "ripemd160@0.2.0", - "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-0.2.0.tgz" - }, - "run-async": { - "version": "0.1.0", - "from": "run-async@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-0.1.0.tgz" - }, - "rx-lite": { - "version": "3.1.2", - "from": "rx-lite@>=3.1.2 <4.0.0", - "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-3.1.2.tgz" - }, - "sax": { - "version": "1.1.6", - "from": "sax@>=1.1.6 <1.2.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.6.tgz" - }, - "semver": { - "version": "5.1.0", - "from": "semver@>=5.0.1 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz" - }, - "sentence-case": { - "version": "1.1.3", - "from": "sentence-case@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/sentence-case/-/sentence-case-1.1.3.tgz" - }, - "sha.js": { - "version": "2.2.6", - "from": "sha.js@2.2.6", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.2.6.tgz" - }, - "shebang-regex": { - "version": "1.0.0", - "from": "shebang-regex@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz" - }, - "shelljs": { - "version": "0.6.0", - "from": "shelljs@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.6.0.tgz" - }, - "simple-fmt": { - "version": "0.1.0", - "from": "simple-fmt@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/simple-fmt/-/simple-fmt-0.1.0.tgz" - }, - "simple-is": { - "version": "0.2.0", - "from": "simple-is@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/simple-is/-/simple-is-0.2.0.tgz" - }, - "slash": { - "version": "1.0.0", - "from": "slash@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz" - }, - "slice-ansi": { - "version": "0.0.4", - "from": "slice-ansi@0.0.4", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-0.0.4.tgz" - }, - "snake-case": { - "version": "1.1.2", - "from": "snake-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-1.1.2.tgz" - }, - "sort-keys": { - "version": "1.1.1", - "from": "sort-keys@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.1.tgz" - }, - "source-list-map": { - "version": "0.1.6", - "from": "source-list-map@>=0.1.4 <0.2.0", - "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-0.1.6.tgz" - }, - "source-map": { - "version": "0.1.31", - "from": "source-map@0.1.31", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.31.tgz" - }, - "source-map-support": { - "version": "0.2.10", - "from": "source-map-support@>=0.2.10 <0.3.0", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.2.10.tgz", - "dependencies": { - "source-map": { - "version": "0.1.32", - "from": "source-map@0.1.32", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.32.tgz" - } - } - }, - "sprintf-js": { - "version": "1.0.3", - "from": "sprintf-js@>=1.0.2 <1.1.0", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" - }, - "stable": { - "version": "0.1.5", - "from": "stable@>=0.1.3 <0.2.0", - "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.5.tgz" - }, - "stream-browserify": { - "version": "1.0.0", - "from": "stream-browserify@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-1.0.0.tgz", - "dependencies": { - "isarray": { - "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" - }, - "readable-stream": { - "version": "1.1.13", - "from": "readable-stream@>=1.0.27-1 <2.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.13.tgz" - } - } - }, - "strict-uri-encode": { - "version": "1.1.0", - "from": "strict-uri-encode@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz" - }, - "string_decoder": { - "version": "0.10.31", - "from": "string_decoder@>=0.10.0 <0.11.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" - }, - "string-width": { - "version": "1.0.1", - "from": "string-width@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.1.tgz" - }, - "stringmap": { - "version": "0.2.2", - "from": "stringmap@>=0.2.2 <0.3.0", - "resolved": "https://registry.npmjs.org/stringmap/-/stringmap-0.2.2.tgz" - }, - "stringset": { - "version": "0.2.1", - "from": "stringset@>=0.2.1 <0.3.0", - "resolved": "https://registry.npmjs.org/stringset/-/stringset-0.2.1.tgz" - }, - "strip-ansi": { - "version": "3.0.1", - "from": "strip-ansi@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz" - }, - "strip-json-comments": { - "version": "1.0.4", - "from": "strip-json-comments@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz" - }, - "style-loader": { - "version": "0.13.1", - "from": "style-loader@>=0.13.0 <0.14.0", - "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-0.13.1.tgz" - }, - "supports-color": { - "version": "2.0.0", - "from": "supports-color@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" - }, - "svgo": { - "version": "0.6.3", - "from": "svgo@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-0.6.3.tgz" - }, - "swap-case": { - "version": "1.1.2", - "from": "swap-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/swap-case/-/swap-case-1.1.2.tgz" - }, - "table": { - "version": "3.7.8", - "from": "table@>=3.7.8 <4.0.0", - "resolved": "https://registry.npmjs.org/table/-/table-3.7.8.tgz", - "dependencies": { - "bluebird": { - "version": "3.3.4", - "from": "bluebird@>=3.1.1 <4.0.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.3.4.tgz" - }, - "lodash": { - "version": "4.7.0", - "from": "lodash@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.7.0.tgz" - } - } - }, - "tapable": { - "version": "0.1.10", - "from": "tapable@>=0.1.8 <0.2.0", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-0.1.10.tgz" - }, - "text-table": { - "version": "0.2.0", - "from": "text-table@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" - }, - "through": { - "version": "2.3.8", - "from": "through@>=2.3.4 <2.4.0", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz" - }, - "timers-browserify": { - "version": "1.4.2", - "from": "timers-browserify@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-1.4.2.tgz" - }, - "title-case": { - "version": "1.1.2", - "from": "title-case@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/title-case/-/title-case-1.1.2.tgz" - }, - "to-fast-properties": { - "version": "1.0.2", - "from": "to-fast-properties@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.2.tgz" - }, - "trim-right": { - "version": "1.0.1", - "from": "trim-right@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz" - }, - "try-resolve": { - "version": "1.0.1", - "from": "try-resolve@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/try-resolve/-/try-resolve-1.0.1.tgz" - }, - "tryit": { - "version": "1.0.2", - "from": "tryit@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/tryit/-/tryit-1.0.2.tgz" - }, - "tryor": { - "version": "0.1.2", - "from": "tryor@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/tryor/-/tryor-0.1.2.tgz" - }, - "ts-loader": { - "version": "0.8.1", - "from": "ts-loader@>=0.8.0 <0.9.0", - "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-0.8.1.tgz", - "dependencies": { - "object-assign": { - "version": "2.1.1", - "from": "object-assign@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-2.1.1.tgz" - } - } - }, - "tty-browserify": { - "version": "0.0.0", - "from": "tty-browserify@0.0.0", - "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz" - }, - "tv4": { - "version": "1.2.7", - "from": "tv4@>=1.2.7 <2.0.0", - "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.2.7.tgz" - }, - "type-check": { - "version": "0.3.2", - "from": "type-check@>=0.3.2 <0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz" - }, - "typedarray": { - "version": "0.0.6", - "from": "typedarray@>=0.0.5 <0.1.0", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz" - }, - "ua-parser-js": { - "version": "0.7.10", - "from": "ua-parser-js@>=0.7.9 <0.8.0", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.10.tgz" - }, - "uglify-js": { - "version": "2.6.2", - "from": "uglify-js@>=2.6.0 <2.7.0", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.6.2.tgz", - "dependencies": { - "source-map": { - "version": "0.5.3", - "from": "source-map@>=0.5.1 <0.6.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz" - }, - "window-size": { - "version": "0.1.0", - "from": "window-size@0.1.0", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz" - }, - "yargs": { - "version": "3.10.0", - "from": "yargs@>=3.10.0 <3.11.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz" - } - } - }, - "uglify-to-browserify": { - "version": "1.0.2", - "from": "uglify-to-browserify@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz" - }, - "uncontrollable": { - "version": "3.2.3", - "from": "uncontrollable@>=3.1.3 <4.0.0", - "resolved": "https://registry.npmjs.org/uncontrollable/-/uncontrollable-3.2.3.tgz" - }, - "uniq": { - "version": "1.0.1", - "from": "uniq@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz" - }, - "uniqid": { - "version": "1.0.0", - "from": "uniqid@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/uniqid/-/uniqid-1.0.0.tgz" - }, - "uniqs": { - "version": "2.0.0", - "from": "uniqs@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz" - }, - "upper-case": { - "version": "1.1.3", - "from": "upper-case@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/upper-case/-/upper-case-1.1.3.tgz" - }, - "upper-case-first": { - "version": "1.1.2", - "from": "upper-case-first@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/upper-case-first/-/upper-case-first-1.1.2.tgz" - }, - "url": { - "version": "0.10.3", - "from": "url@>=0.10.1 <0.11.0", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "dependencies": { - "punycode": { - "version": "1.3.2", - "from": "punycode@1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz" - } - } - }, - "user-home": { - "version": "1.1.1", - "from": "user-home@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/user-home/-/user-home-1.1.1.tgz" - }, - "util": { - "version": "0.10.3", - "from": "util@>=0.10.3 <0.11.0", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz" - }, - "util-deprecate": { - "version": "1.0.2", - "from": "util-deprecate@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" - }, - "vm-browserify": { - "version": "0.0.4", - "from": "vm-browserify@0.0.4", - "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz" - }, - "w3c-blob": { - "version": "0.0.1", - "from": "w3c-blob@0.0.1", - "resolved": "https://registry.npmjs.org/w3c-blob/-/w3c-blob-0.0.1.tgz" - }, - "warning": { - "version": "2.1.0", - "from": "warning@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/warning/-/warning-2.1.0.tgz" - }, - "watchpack": { - "version": "0.2.9", - "from": "watchpack@>=0.2.1 <0.3.0", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-0.2.9.tgz", - "dependencies": { - "async": { - "version": "0.9.2", - "from": "async@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz" - } - } - }, - "webpack": { - "version": "1.12.14", - "from": "webpack@>=1.12.2 <2.0.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-1.12.14.tgz", - "dependencies": { - "async": { - "version": "1.5.2", - "from": "async@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" - }, - "esprima": { - "version": "2.7.2", - "from": "esprima@>=2.5.0 <3.0.0" - }, - "supports-color": { - "version": "3.1.2", - "from": "supports-color@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz" - } - } - }, - "webpack-cleanup-plugin": { - "version": "0.1.1", - "from": "webpack-cleanup-plugin@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/webpack-cleanup-plugin/-/webpack-cleanup-plugin-0.1.1.tgz" - }, - "webpack-core": { - "version": "0.6.8", - "from": "webpack-core@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/webpack-core/-/webpack-core-0.6.8.tgz", - "dependencies": { - "source-map": { - "version": "0.4.4", - "from": "source-map@>=0.4.1 <0.5.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz" - } - } - }, - "webpack-merge": { - "version": "0.7.3", - "from": "webpack-merge@>=0.7.1 <0.8.0", - "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-0.7.3.tgz" - }, - "whatwg-fetch": { - "version": "0.9.0", - "from": "whatwg-fetch@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-0.9.0.tgz" - }, - "whet.extend": { - "version": "0.9.9", - "from": "whet.extend@>=0.9.9 <0.10.0", - "resolved": "https://registry.npmjs.org/whet.extend/-/whet.extend-0.9.9.tgz" - }, - "window-size": { - "version": "0.1.4", - "from": "window-size@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.4.tgz" - }, - "wordwrap": { - "version": "0.0.2", - "from": "wordwrap@0.0.2", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz" - }, - "wrappy": { - "version": "1.0.1", - "from": "wrappy@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" - }, - "write": { - "version": "0.2.1", - "from": "write@>=0.2.1 <0.3.0", - "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz" - }, - "xml-char-classes": { - "version": "1.0.0", - "from": "xml-char-classes@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/xml-char-classes/-/xml-char-classes-1.0.0.tgz" - }, - "xregexp": { - "version": "3.1.0", - "from": "xregexp@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-3.1.0.tgz" - }, - "xtend": { - "version": "4.0.1", - "from": "xtend@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" - }, - "y18n": { - "version": "3.2.1", - "from": "y18n@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz" - }, - "yargs": { - "version": "3.27.0", - "from": "yargs@>=3.27.0 <3.28.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.27.0.tgz" - } - } -} From f883be9cf426a7d2f7ca7976fcb03919d8f15381 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 1 Jul 2016 13:44:25 +0200 Subject: [PATCH 260/521] Use eslint-config-graylog for linter configuration --- .eslintrc | 16 +--------------- package.json | 7 ++++--- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/.eslintrc b/.eslintrc index c441df40ee46..5a9c5f2edef3 100644 --- a/.eslintrc +++ b/.eslintrc @@ -5,21 +5,7 @@ "jsx": true, }, "extends": [ - "eslint:recommended", - "airbnb", + "graylog", ], - "rules": { - "arrow-body-style": 0, - "indent": [2, 2, { "SwitchCase" : 1}], - "max-len" : 0, - "new-cap": 0, - "no-else-return": 1, - "no-nested-ternary": 1, - "object-shorthand": [2, "methods"], - "react/jsx-closing-bracket-location": 0, - "react/jsx-indent-props": 0, - "react/jsx-space-before-closing": 0, - "react/prefer-es6-class": 0, - }, } diff --git a/package.json b/package.json index cac610112a37..3674025445fb 100644 --- a/package.json +++ b/package.json @@ -29,12 +29,13 @@ }, "devDependencies": { "babel-core": "^5.8.25", + "babel-eslint": "^4.1.6", "babel-loader": "^5.3.2", "css-loader": "^0.23.1", - "eslint": "^2.2.0", - "eslint-config-airbnb": "6.0.2", + "eslint": "^2.10.2", + "eslint-config-graylog": "^1.0.0", "eslint-loader": "^1.0.0", - "eslint-plugin-react": "^4.1.0", + "estraverse-fb": "^1.3.1", "graylog-web-manifests": "^2.0.0-alpha.3", "graylog-web-plugin": "latest", "json-loader": "^0.5.4", From 8f6cc74d3b5930e8a1db9b47448927818d66d411 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 4 Jul 2016 15:37:50 +0200 Subject: [PATCH 261/521] Use raw messages on simulator (#42) * Use arrow function to keep 'this' binding * Adapt simulator to use raw message loader At this point the pipeline simulator only makes sense for messages that were not already processed by Graylog, as it can be really tricky to see changes done in already processed messages. Therefore, we replace the message loader, using the raw message loader. --- .../rest/SimulationRequest.java | 17 +++---- .../rest/SimulatorResource.java | 45 ++++++++----------- src/web/simulator/ProcessorSimulator.jsx | 29 +++++++----- src/web/simulator/SimulatorStore.js | 7 ++- 4 files changed, 45 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java index cbf1e8963d0c..97755331be2a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; +import java.util.Map; + @AutoValue @JsonAutoDetect public abstract class SimulationRequest { @@ -28,10 +30,7 @@ public abstract class SimulationRequest { public abstract String streamId(); @JsonProperty - public abstract String index(); - - @JsonProperty - public abstract String messageId(); + public abstract Map message(); public static Builder builder() { return new AutoValue_SimulationRequest.Builder(); @@ -39,12 +38,10 @@ public static Builder builder() { @JsonCreator public static SimulationRequest create (@JsonProperty("stream_id") String streamId, - @JsonProperty("index") String index, - @JsonProperty("message_id") String messageId) { + @JsonProperty("message") Map message) { return builder() .streamId(streamId) - .index(index) - .messageId(messageId) + .message(message) .build(); } @@ -54,8 +51,6 @@ public abstract static class Builder { public abstract Builder streamId(String streamId); - public abstract Builder index(String index); - - public abstract Builder messageId(String messageId); + public abstract Builder message(Map message); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java index 8e7be6ec966c..1d0664517019 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -24,15 +24,13 @@ import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTracer; import org.graylog2.database.NotFoundException; -import org.graylog2.indexer.messages.DocumentNotFoundException; -import org.graylog2.indexer.messages.Messages; -import org.graylog2.indexer.results.ResultMessage; import org.graylog2.messageprocessors.OrderedMessageProcessors; import org.graylog2.plugin.Message; import org.graylog2.plugin.messageprocessors.MessageProcessor; import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.plugin.streams.Stream; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; +import org.graylog2.rest.resources.messages.MessageResource; import org.graylog2.shared.rest.resources.RestResource; import org.graylog2.shared.security.RestPermissions; import org.graylog2.streams.StreamService; @@ -54,13 +52,13 @@ @RequiresAuthentication public class SimulatorResource extends RestResource implements PluginRestResource { private final OrderedMessageProcessors orderedMessageProcessors; - private final Messages messages; + private final MessageResource messageResource; private final StreamService streamService; @Inject - public SimulatorResource(OrderedMessageProcessors orderedMessageProcessors, Messages messages, StreamService streamService) { + public SimulatorResource(OrderedMessageProcessors orderedMessageProcessors, MessageResource messageResource, StreamService streamService) { this.orderedMessageProcessors = orderedMessageProcessors; - this.messages = messages; + this.messageResource = messageResource; this.streamService = streamService; } @@ -68,31 +66,26 @@ public SimulatorResource(OrderedMessageProcessors orderedMessageProcessors, Mess @POST @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_READ) public SimulationResponse simulate(@ApiParam(name = "simulation", required = true) @NotNull SimulationRequest request) throws NotFoundException { - checkPermission(RestPermissions.MESSAGES_READ, request.messageId()); checkPermission(RestPermissions.STREAMS_READ, request.streamId()); - try { - final ResultMessage resultMessage = messages.get(request.messageId(), request.index()); - final Message message = resultMessage.getMessage(); - if (!request.streamId().equals("default")) { - final Stream stream = streamService.load(request.streamId()); - message.addStream(stream); - } - final List simulationResults = new ArrayList<>(); - final PipelineInterpreterTracer pipelineInterpreterTracer = new PipelineInterpreterTracer(); + final Message message = new Message(request.message()); + if (!request.streamId().equals("default")) { + final Stream stream = streamService.load(request.streamId()); + message.addStream(stream); + } + + final List simulationResults = new ArrayList<>(); + final PipelineInterpreterTracer pipelineInterpreterTracer = new PipelineInterpreterTracer(); - for (MessageProcessor messageProcessor : orderedMessageProcessors) { - if (messageProcessor instanceof PipelineInterpreter) { - org.graylog2.plugin.Messages processedMessages = ((PipelineInterpreter)messageProcessor).process(message, pipelineInterpreterTracer.getSimulatorInterpreterListener()); - for (Message processedMessage : processedMessages) { - simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); - } + for (MessageProcessor messageProcessor : orderedMessageProcessors) { + if (messageProcessor instanceof PipelineInterpreter) { + org.graylog2.plugin.Messages processedMessages = ((PipelineInterpreter) messageProcessor).process(message, pipelineInterpreterTracer.getSimulatorInterpreterListener()); + for (Message processedMessage : processedMessages) { + simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); } } - - return SimulationResponse.create(simulationResults, pipelineInterpreterTracer.getExecutionTrace(), pipelineInterpreterTracer.took()); - } catch (DocumentNotFoundException e) { - throw new NotFoundException(e); } + + return SimulationResponse.create(simulationResults, pipelineInterpreterTracer.getExecutionTrace(), pipelineInterpreterTracer.took()); } } diff --git a/src/web/simulator/ProcessorSimulator.jsx b/src/web/simulator/ProcessorSimulator.jsx index a7aa59cefebf..f84ce8e996bf 100644 --- a/src/web/simulator/ProcessorSimulator.jsx +++ b/src/web/simulator/ProcessorSimulator.jsx @@ -1,10 +1,11 @@ import React from 'react'; import { Col, Row } from 'react-bootstrap'; -import LoaderTabs from 'components/messageloaders/LoaderTabs'; +import RawMessageLoader from 'components/messageloaders/RawMessageLoader'; import SimulationResults from './SimulationResults'; import SimulatorActions from './SimulatorActions'; +// eslint-disable-next-line no-unused-vars import SimulatorStore from './SimulatorStore'; const ProcessorSimulator = React.createClass({ @@ -24,14 +25,16 @@ const ProcessorSimulator = React.createClass({ _onMessageLoad(message) { this.setState({ message: message, simulation: undefined, loading: true, error: undefined }); - SimulatorActions.simulate.triggerPromise(this.props.stream, message.index, message.id).then( - response => { - this.setState({ simulation: response, loading: false }); - }, - error => { - this.setState({ loading: false, error: error }); - } - ); + SimulatorActions.simulate + .triggerPromise(this.props.stream, message.fields) + .then( + response => { + this.setState({ simulation: response, loading: false }); + }, + error => { + this.setState({ loading: false, error: error }); + } + ); }, render() { @@ -40,9 +43,11 @@ const ProcessorSimulator = React.createClass({

    Load a message

    -

    Load a message to be used in the simulation. No changes will be done in your stored - messages.

    - +

    + Load a message to be used in the simulation.{' '} + No changes will be done in your stored messages. +

    +     { const formattedResponse = ObjectUtils.clone(response); - formattedResponse.messages = response.messages.map(MessageFormatter.formatMessageSummary); + formattedResponse.messages = response.messages.map(msg => MessageFormatter.formatMessageSummary(msg)); return formattedResponse; }); From 10caade7ec1c271bb8c68515ea024a3bb6700a6e Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Tue, 5 Jul 2016 15:59:45 +0200 Subject: [PATCH 262/521] Bumping (dependency) versions to 2.1.0-alpha.1/1.1.0-alpha.1. --- pom.xml | 6 ++---- .../pipelineprocessor/PipelineProcessorMetaData.java | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 206a268d61d3..72b751f4202c 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,5 @@ - + 4.0.0 3.1.0 @@ -39,7 +37,7 @@ true true true - 2.1.0-SNAPSHOT + 2.1.0-alpha.1 /usr/share/graylog-server/plugin 4.5.1 1.7.13 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index 6814e8df0541..de230a4dc52f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 0, 0, "beta.2"); + return new Version(1, 1, 0, "alpha.1"); } @Override @@ -60,7 +60,7 @@ public String getDescription() { @Override public Version getRequiredVersion() { - return new Version(2, 0, 0); + return new Version(2, 1, 0); } @Override From 466014926f0be1bce0c5ef2bff1e9cfdee32e473 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Tue, 5 Jul 2016 14:00:42 +0000 Subject: [PATCH 263/521] [graylog-plugin-pipeline-processor-release] prepare release 1.1.0-alpha.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 72b751f4202c..46641893d6e5 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.3-SNAPSHOT + 1.1.0-alpha.1 jar ${project.artifactId} From 400dec09c1ef559c2facd21e2e17a541fd4ac3ff Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Tue, 5 Jul 2016 14:00:53 +0000 Subject: [PATCH 264/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 46641893d6e5..240b95f83acf 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-alpha.1 + 1.1.0-alpha.2-SNAPSHOT jar ${project.artifactId} From c7fbfa5750f22b4b6b45acfd604f5e297be91ce0 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Wed, 6 Jul 2016 17:25:42 +0200 Subject: [PATCH 265/521] Bumping graylog dependency version to next development iteration. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 240b95f83acf..79adc85c19c1 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-alpha.1 + 2.1.0-alpha.2-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From e2c0f69e08977f00d39543aa0130533936c97197 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 7 Jul 2016 13:31:08 +0200 Subject: [PATCH 266/521] Bumping versions to 2.1.0-alpha.2/1.1.0-alpha.2 --- package.json | 2 +- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 3674025445fb..e60f1c7a9a95 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PipelineProcessor", - "version": "1.0.0-alpha.7-SNAPSHOT", + "version": "1.1.0-alpha.2", "description": "", "repository": { "type": "git", diff --git a/pom.xml b/pom.xml index 79adc85c19c1..6b8e653f75db 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-alpha.2-SNAPSHOT + 2.1.0-alpha.2 /usr/share/graylog-server/plugin 4.5.1 1.7.13 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index de230a4dc52f..cee9ae91d60b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 1, 0, "alpha.1"); + return new Version(1, 1, 0, "alpha.2"); } @Override @@ -60,7 +60,7 @@ public String getDescription() { @Override public Version getRequiredVersion() { - return new Version(2, 1, 0); + return new Version(2, 1, 0, "alpha.2"); } @Override From c14959aea0d35301245a59ef1f0c23e88e9aa180 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 7 Jul 2016 11:33:20 +0000 Subject: [PATCH 267/521] [graylog-plugin-pipeline-processor-release] prepare release 1.1.0-alpha.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b8e653f75db..b81ba31017ba 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-alpha.2-SNAPSHOT + 1.1.0-alpha.2 jar ${project.artifactId} From 7a63c1a1847d9c93711e82d537ed30c295493cac Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 7 Jul 2016 11:33:30 +0000 Subject: [PATCH 268/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b81ba31017ba..e3922bc64c27 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-alpha.2 + 1.1.0-alpha.3-SNAPSHOT jar ${project.artifactId} From 158877efd080eb3da46d7ca9cffa9d0e6fa97a8a Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 7 Jul 2016 14:32:21 +0200 Subject: [PATCH 269/521] Bumping graylog dependency version to next development iteration. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e3922bc64c27..bb43e2c899fe 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-alpha.2 + 2.1.0-alpha.3-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From 72f7d4bc87beb883d1b41dd40f84839741c306a2 Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Tue, 12 Jul 2016 17:06:41 -0500 Subject: [PATCH 270/521] small copy/UX changes --- src/web/simulator/ProcessorSimulator.jsx | 4 ++-- src/web/simulator/SimulationPreview.jsx | 4 ++-- src/web/simulator/SimulationResults.jsx | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/web/simulator/ProcessorSimulator.jsx b/src/web/simulator/ProcessorSimulator.jsx index f84ce8e996bf..f063ed7caa4e 100644 --- a/src/web/simulator/ProcessorSimulator.jsx +++ b/src/web/simulator/ProcessorSimulator.jsx @@ -44,8 +44,8 @@ const ProcessorSimulator = React.createClass({

    Load a message

    - Load a message to be used in the simulation.{' '} - No changes will be done in your stored messages. + Build an example message that will be used in the simulation.{' '} + No real messages stored in Graylog will be changed. All actions are purely simulated on the temporary input you provide below.

    diff --git a/src/web/simulator/SimulationPreview.jsx b/src/web/simulator/SimulationPreview.jsx index 1658dcb54639..c0f12056790c 100644 --- a/src/web/simulator/SimulationPreview.jsx +++ b/src/web/simulator/SimulationPreview.jsx @@ -17,8 +17,8 @@ const SimulationPreview = React.createClass({

    Message would be dropped

    - Processing the loaded message would drop it from the system. That means that the message would - not be stored, and would not be available on searches, alerts, or dashboards. + The pipeline processor would drop such a message. That means that the message would + not be stored, and would not be available for searches, alerts, outputs, or dashboards.

    ); diff --git a/src/web/simulator/SimulationResults.jsx b/src/web/simulator/SimulationResults.jsx index 5f068d8fa5e0..e5cbc567379d 100644 --- a/src/web/simulator/SimulationResults.jsx +++ b/src/web/simulator/SimulationResults.jsx @@ -131,7 +131,7 @@ const SimulationResults = React.createClass({
    - {this._getViewOptionsMenuItems()} From da4f1ecc446695d00074513f9fd0a86a32eb0728 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Thu, 14 Jul 2016 16:53:50 +0200 Subject: [PATCH 271/521] Provide pipeline-based message decorator (#41) * Providing a message decorator that uses pipelines. * Making decorator configurable. * Allow adding new messages by pipeline decorator. * Adding changes related due to introduced listener. * Adapt to naming changes, using easier forEach idiom. --- .../PipelineProcessorMessageDecorator.java | 109 +++++++++ .../PipelineProcessorModule.java | 2 + .../processors/PipelineInterpreter.java | 212 +++++++++--------- 3 files changed, 221 insertions(+), 102 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java new file mode 100644 index 000000000000..e298e7e7c48c --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java @@ -0,0 +1,109 @@ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.inject.assistedinject.Assisted; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; +import org.graylog2.decorators.Decorator; +import org.graylog2.indexer.results.ResultMessage; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.configuration.ConfigurationRequest; +import org.graylog2.plugin.configuration.fields.ConfigurationField; +import org.graylog2.plugin.configuration.fields.DropdownField; +import org.graylog2.plugin.decorators.MessageDecorator; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class PipelineProcessorMessageDecorator implements MessageDecorator { + private static final String CONFIG_FIELD_PIPELINE = "pipeline"; + + private final PipelineInterpreter pipelineInterpreter; + private final ImmutableSet pipelines; + + public interface Factory extends MessageDecorator.Factory { + @Override + PipelineProcessorMessageDecorator create(Decorator decorator); + + @Override + Config getConfig(); + + @Override + Descriptor getDescriptor(); + } + + public static class Config implements MessageDecorator.Config { + private final PipelineService pipelineService; + + @Inject + public Config(PipelineService pipelineService) { + this.pipelineService = pipelineService; + } + + @Override + public ConfigurationRequest getRequestedConfiguration() { + final Map pipelineOptions = this.pipelineService.loadAll().stream() + .sorted((o1, o2) -> o1.title().compareTo(o2.title())) + .collect(Collectors.toMap(PipelineDao::id, PipelineDao::title)); + return new ConfigurationRequest() {{ + addField(new DropdownField(CONFIG_FIELD_PIPELINE, + "Pipeline", + "", + pipelineOptions, + "Which pipeline to use for message decoration", + ConfigurationField.Optional.NOT_OPTIONAL)); + }}; + }; + } + + public static class Descriptor extends MessageDecorator.Descriptor { + public Descriptor() { + super("Pipeline Processor Decorator", false, "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Pipeline Processor Decorator"); + } + } + + @Inject + public PipelineProcessorMessageDecorator(PipelineInterpreter pipelineInterpreter, + @Assisted Decorator decorator) { + this.pipelineInterpreter = pipelineInterpreter; + final String pipelineId = (String)decorator.config().get(CONFIG_FIELD_PIPELINE); + if (Strings.isNullOrEmpty(pipelineId)) { + this.pipelines = ImmutableSet.of(); + } else { + this.pipelines = ImmutableSet.of(pipelineId); + } + } + + @Override + public List apply(List resultMessages) { + final List results = new ArrayList<>(); + if (pipelines.isEmpty()) { + return resultMessages; + } + resultMessages.forEach((inMessage) -> { + final Message message = inMessage.getMessage(); + final List additionalCreatedMessages = pipelineInterpreter.processForPipelines(message, + message.getId(), + pipelines, + new NoopInterpreterListener()); + final ResultMessage outMessage = ResultMessage.createFromMessage(message, inMessage.getIndex(), inMessage.getHighlightRanges()); + + results.add(outMessage); + additionalCreatedMessages.forEach((additionalMessage) -> { + // TODO: pass proper highlight ranges. Need to rebuild them for new messages. + results.add(ResultMessage.createFromMessage(additionalMessage, "[created from decorator]", ImmutableMultimap.of())); + }); + }); + + return results; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 4ad14f0236fa..578c83ed05f2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -46,5 +46,7 @@ protected void configure() { addPermissions(PipelineRestPermissions.class); install(new ProcessorFunctionsModule()); + + installMessageDecorator(messageDecoratorBinder(), PipelineProcessorMessageDecorator.class, PipelineProcessorMessageDecorator.Factory.class); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index a78def740b96..0387d7ee4315 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -235,108 +235,8 @@ public Messages process(Messages messages, InterpreterListener interpreterListen log.debug("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); } - // record execution of pipeline in metrics - pipelinesToRun.stream().forEach(pipeline -> metricRegistry.counter(name(Pipeline.class, pipeline.id(), "executed")).inc()); - - final StageIterator stages = new StageIterator(pipelinesToRun); - final Set pipelinesToSkip = Sets.newHashSet(); - - // iterate through all stages for all matching pipelines, per "stage slice" instead of per pipeline. - // pipeline execution ordering is not guaranteed - while (stages.hasNext()) { - final Set> stageSet = stages.next(); - for (Tuple2 pair : stageSet) { - final Stage stage = pair.v1(); - final Pipeline pipeline = pair.v2(); - if (pipelinesToSkip.contains(pipeline)) { - log.debug("[{}] previous stage result prevents further processing of pipeline `{}`", - msgId, - pipeline.name()); - continue; - } - metricRegistry.counter(name(Pipeline.class, pipeline.id(), "stage", String.valueOf(stage.stage()), "executed")).inc(); - interpreterListener.enterStage(stage); - log.debug("[{}] evaluating rule conditions in stage {}: match {}", - msgId, - stage.stage(), - stage.matchAll() ? "all" : "either"); - - // TODO the message should be decorated to allow layering changes and isolate stages - final EvaluationContext context = new EvaluationContext(message); - - // 3. iterate over all the stages in these pipelines and execute them in order - final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); - boolean anyRulesMatched = false; - for (Rule rule : stage.getRules()) { - interpreterListener.evaluateRule(rule, pipeline); - if (rule.when().evaluateBool(context)) { - anyRulesMatched = true; - countRuleExecution(rule, pipeline, stage, "matched"); - - if (context.hasEvaluationErrors()) { - final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); - appendProcessingError(rule, message, lastError.toString()); - interpreterListener.failEvaluateRule(rule, pipeline); - log.debug("Encountered evaluation error during condition, skipping rule actions: {}", - lastError); - continue; - } - interpreterListener.satisfyRule(rule, pipeline); - log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); - rulesToRun.add(rule); - } else { - countRuleExecution(rule, pipeline, stage, "not-matched"); - interpreterListener.dissatisfyRule(rule, pipeline); - log.debug("[{}] rule `{}` does not match", msgId, rule.name()); - } - } - RULES: - for (Rule rule : rulesToRun) { - countRuleExecution(rule, pipeline, stage, "executed"); - interpreterListener.executeRule(rule, pipeline); - log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); - for (Statement statement : rule.then()) { - statement.evaluate(context); - if (context.hasEvaluationErrors()) { - // if the last statement resulted in an error, do not continue to execute this rules - final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); - appendProcessingError(rule, message, lastError.toString()); - interpreterListener.failExecuteRule(rule, pipeline); - log.debug("Encountered evaluation error, skipping rest of the rule: {}", - lastError); - countRuleExecution(rule, pipeline, stage, "failed"); - break RULES; - } - } - } - // stage needed to match all rule conditions to enable the next stage, - // record that it is ok to proceed with this pipeline - // OR - // any rule could match, but at least one had to, - // record that it is ok to proceed with the pipeline - if ((stage.matchAll() && (rulesToRun.size() == stage.getRules().size())) - || (rulesToRun.size() > 0 && anyRulesMatched)) { - interpreterListener.continuePipelineExecution(pipeline, stage); - log.debug("[{}] stage {} for pipeline `{}` required match: {}, ok to proceed with next stage", - msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); - } else { - // no longer execute stages from this pipeline, the guard prevents it - interpreterListener.stopPipelineExecution(pipeline, stage); - log.debug("[{}] stage {} for pipeline `{}` required match: {}, NOT ok to proceed with next stage", - msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); - pipelinesToSkip.add(pipeline); - } - - // 4. after each complete stage run, merge the processing changes, stages are isolated from each other - // TODO message changes become visible immediately for now + toProcess.addAll(processForPipelines(message, msgId, pipelinesToRun.stream().map(Pipeline::id).collect(Collectors.toSet()), interpreterListener)); - // 4a. also add all new messages from the context to the toProcess work list - Iterables.addAll(toProcess, context.createdMessages()); - context.clearCreatedMessages(); - interpreterListener.exitStage(stage); - } - - } boolean addedStreams = false; // 5. add each message-stream combination to the blacklist set for (Stream stream : message.getStreams()) { @@ -365,10 +265,118 @@ public Messages process(Messages messages, InterpreterListener interpreterListen } } } + // 7. return the processed messages + return new MessageCollection(fullyProcessed); + } + + public List processForPipelines(Message message, String msgId, Set pipelines, InterpreterListener interpreterListener) { + final ImmutableSet pipelinesToRun = ImmutableSet.copyOf(pipelines.stream().map(pipelineId -> this.currentPipelines.get().get(pipelineId)).collect(Collectors.toSet())); + final List result = new ArrayList<>(); + // record execution of pipeline in metrics + pipelinesToRun.stream().forEach(pipeline -> metricRegistry.counter(name(Pipeline.class, pipeline.id(), "executed")).inc()); + + final StageIterator stages = new StageIterator(pipelinesToRun); + final Set pipelinesToSkip = Sets.newHashSet(); + + // iterate through all stages for all matching pipelines, per "stage slice" instead of per pipeline. + // pipeline execution ordering is not guaranteed + while (stages.hasNext()) { + final Set> stageSet = stages.next(); + for (Tuple2 pair : stageSet) { + final Stage stage = pair.v1(); + final Pipeline pipeline = pair.v2(); + if (pipelinesToSkip.contains(pipeline)) { + log.debug("[{}] previous stage result prevents further processing of pipeline `{}`", + msgId, + pipeline.name()); + continue; + } + metricRegistry.counter(name(Pipeline.class, pipeline.id(), "stage", String.valueOf(stage.stage()), "executed")).inc(); + interpreterListener.enterStage(stage); + log.debug("[{}] evaluating rule conditions in stage {}: match {}", + msgId, + stage.stage(), + stage.matchAll() ? "all" : "either"); + + // TODO the message should be decorated to allow layering changes and isolate stages + final EvaluationContext context = new EvaluationContext(message); + + // 3. iterate over all the stages in these pipelines and execute them in order + final ArrayList rulesToRun = Lists.newArrayListWithCapacity(stage.getRules().size()); + boolean anyRulesMatched = false; + for (Rule rule : stage.getRules()) { + interpreterListener.evaluateRule(rule, pipeline); + if (rule.when().evaluateBool(context)) { + anyRulesMatched = true; + countRuleExecution(rule, pipeline, stage, "matched"); + + if (context.hasEvaluationErrors()) { + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(rule, message, lastError.toString()); + interpreterListener.failEvaluateRule(rule, pipeline); + log.debug("Encountered evaluation error during condition, skipping rule actions: {}", + lastError); + continue; + } + interpreterListener.satisfyRule(rule, pipeline); + log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); + rulesToRun.add(rule); + } else { + countRuleExecution(rule, pipeline, stage, "not-matched"); + interpreterListener.dissatisfyRule(rule, pipeline); + log.debug("[{}] rule `{}` does not match", msgId, rule.name()); + } + } + RULES: + for (Rule rule : rulesToRun) { + countRuleExecution(rule, pipeline, stage, "executed"); + interpreterListener.executeRule(rule, pipeline); + log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); + for (Statement statement : rule.then()) { + statement.evaluate(context); + if (context.hasEvaluationErrors()) { + // if the last statement resulted in an error, do not continue to execute this rules + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(rule, message, lastError.toString()); + interpreterListener.failExecuteRule(rule, pipeline); + log.debug("Encountered evaluation error, skipping rest of the rule: {}", + lastError); + countRuleExecution(rule, pipeline, stage, "failed"); + break RULES; + } + } + } + // stage needed to match all rule conditions to enable the next stage, + // record that it is ok to proceed with this pipeline + // OR + // any rule could match, but at least one had to, + // record that it is ok to proceed with the pipeline + if ((stage.matchAll() && (rulesToRun.size() == stage.getRules().size())) + || (rulesToRun.size() > 0 && anyRulesMatched)) { + interpreterListener.continuePipelineExecution(pipeline, stage); + log.debug("[{}] stage {} for pipeline `{}` required match: {}, ok to proceed with next stage", + msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); + } else { + // no longer execute stages from this pipeline, the guard prevents it + interpreterListener.stopPipelineExecution(pipeline, stage); + log.debug("[{}] stage {} for pipeline `{}` required match: {}, NOT ok to proceed with next stage", + msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); + pipelinesToSkip.add(pipeline); + } + + // 4. after each complete stage run, merge the processing changes, stages are isolated from each other + // TODO message changes become visible immediately for now + + // 4a. also add all new messages from the context to the toProcess work list + Iterables.addAll(result, context.createdMessages()); + context.clearCreatedMessages(); + interpreterListener.exitStage(stage); + } + } interpreterListener.finishProcessing(); // 7. return the processed messages - return new MessageCollection(fullyProcessed); + return result; } private void countRuleExecution(Rule rule, Pipeline pipeline, Stage stage, String type) { From 88a9ed329e7bcaea0a64dcc00fdf541ef86ca09a Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Thu, 14 Jul 2016 16:56:41 +0200 Subject: [PATCH 272/521] Adding license header. --- .../PipelineProcessorMessageDecorator.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java index e298e7e7c48c..b6ff46f07332 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor; import com.google.common.base.Strings; From 8140cac8c8bf123c21691a4e5272b08ab9942822 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Tue, 19 Jul 2016 12:32:43 +0200 Subject: [PATCH 273/521] inject the pipeline interpreter directly (#44) this still is a problem, because the interpreter leaks references (it's missing lifecycle handling) --- .../rest/SimulatorResource.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java index 1d0664517019..1f0c30d2af26 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -24,13 +24,10 @@ import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTracer; import org.graylog2.database.NotFoundException; -import org.graylog2.messageprocessors.OrderedMessageProcessors; import org.graylog2.plugin.Message; -import org.graylog2.plugin.messageprocessors.MessageProcessor; import org.graylog2.plugin.rest.PluginRestResource; import org.graylog2.plugin.streams.Stream; import org.graylog2.rest.models.messages.responses.ResultMessageSummary; -import org.graylog2.rest.resources.messages.MessageResource; import org.graylog2.shared.rest.resources.RestResource; import org.graylog2.shared.security.RestPermissions; import org.graylog2.streams.StreamService; @@ -51,14 +48,13 @@ @Produces(MediaType.APPLICATION_JSON) @RequiresAuthentication public class SimulatorResource extends RestResource implements PluginRestResource { - private final OrderedMessageProcessors orderedMessageProcessors; - private final MessageResource messageResource; private final StreamService streamService; + private final PipelineInterpreter pipelineInterpreter; @Inject - public SimulatorResource(OrderedMessageProcessors orderedMessageProcessors, MessageResource messageResource, StreamService streamService) { - this.orderedMessageProcessors = orderedMessageProcessors; - this.messageResource = messageResource; + public SimulatorResource(PipelineInterpreter pipelineInterpreter, + StreamService streamService) { + this.pipelineInterpreter = pipelineInterpreter; this.streamService = streamService; } @@ -77,15 +73,13 @@ public SimulationResponse simulate(@ApiParam(name = "simulation", required = tru final List simulationResults = new ArrayList<>(); final PipelineInterpreterTracer pipelineInterpreterTracer = new PipelineInterpreterTracer(); - for (MessageProcessor messageProcessor : orderedMessageProcessors) { - if (messageProcessor instanceof PipelineInterpreter) { - org.graylog2.plugin.Messages processedMessages = ((PipelineInterpreter) messageProcessor).process(message, pipelineInterpreterTracer.getSimulatorInterpreterListener()); - for (Message processedMessage : processedMessages) { - simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); - } - } + org.graylog2.plugin.Messages processedMessages = pipelineInterpreter.process(message, + pipelineInterpreterTracer.getSimulatorInterpreterListener()); + for (Message processedMessage : processedMessages) { + simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); } - - return SimulationResponse.create(simulationResults, pipelineInterpreterTracer.getExecutionTrace(), pipelineInterpreterTracer.took()); + return SimulationResponse.create(simulationResults, + pipelineInterpreterTracer.getExecutionTrace(), + pipelineInterpreterTracer.took()); } } From ec86125643a5affb83202aea7ac7a167c031d215 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Tue, 19 Jul 2016 18:37:31 +0200 Subject: [PATCH 274/521] Add missing key to --- src/web/pipeline-connections/Connection.jsx | 2 +- src/web/pipelines/Stage.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx index 8643c330a4e4..1e821628fbf2 100644 --- a/src/web/pipeline-connections/Connection.jsx +++ b/src/web/pipeline-connections/Connection.jsx @@ -18,7 +18,7 @@ const Connection = React.createClass({ _pipelineRowFormatter(pipeline) { return ( - + {pipeline.title} diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 1c64eedc050e..9f924d7f4e43 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -41,7 +41,7 @@ const Stage = React.createClass({ } return ( - + {ruleTitle} From ac3fdd133560cd76ab485a60b523d71e8674003f Mon Sep 17 00:00:00 2001 From: Lennart Koopmann Date: Wed, 20 Jul 2016 00:29:15 -0500 Subject: [PATCH 275/521] Allow null matcher group values in regex function (#49) * Allow null matcher group values in regex function --- .../pipelineprocessor/functions/strings/RegexMatch.java | 6 ++++++ .../plugins/pipelineprocessor/functions/regexMatch.txt | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index 4ffc83cd05b9..46d6dfaf3e14 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -97,6 +97,12 @@ public RegexMatchResult(boolean matches, MatchResult matchResult, List g final int groupCount = matchResult.groupCount(); for (int i = 1; i <= groupCount; i++) { final String groupValue = matchResult.group(i); + + if (groupValue == null) { + // You cannot add null values to an ImmutableMap but optional matcher groups may be null. + continue; + } + // try to get a group name, if that fails use a 0-based index as the name final String groupName = Iterables.get(groupNames, i - 1, null); builder.put(groupName != null ? groupName : String.valueOf(i - 1), groupValue); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt index 7a0d6c22745d..9aed823708da 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt @@ -1,6 +1,7 @@ rule "regexMatch" when - regex(".*(cde).*", "abcdefg").matches == true +// the second matcher group is always empty and will not be present in the result + regex(".*(cde)(:(\\d+))?.*", "abcdefg").matches == true then let result = regex(".*(cde).*", "abcdefg"); set_field("group_1", result["0"]); From 5cd12d67fff260485f25083ef39d03765ff1ff03 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 20 Jul 2016 07:35:25 +0200 Subject: [PATCH 276/521] Unescape string literals before using them (#47) Pipeline rules use the same lexical structure as Java for string literals. Before using the parsed string literals, we need to unescape them, removing double backslashes, otherwise the string will actually include two backslashes instead of one. Fixes #46 --- .../pipelineprocessor/parser/PipelineRuleParser.java | 9 +++++++-- .../functions/FunctionsSnippetsTest.java | 2 ++ .../plugins/pipelineprocessor/functions/regexMatch.txt | 5 ++--- .../plugins/pipelineprocessor/functions/strings.txt | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java index 53e98c310c5c..ad929986a083 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -31,6 +31,7 @@ import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTreeProperty; import org.antlr.v4.runtime.tree.ParseTreeWalker; +import org.apache.commons.lang3.StringEscapeUtils; import org.apache.mina.util.IdentityHashSet; import org.graylog.plugins.pipelineprocessor.ast.Pipeline; import org.graylog.plugins.pipelineprocessor.ast.Rule; @@ -194,6 +195,10 @@ public static String unquote(String string, char quoteChar) { return string; } + public static String unescape(String string) { + return StringEscapeUtils.unescapeJava(string); + } + private static class SyntaxErrorListener extends BaseErrorListener { private final ParseContext parseContext; @@ -474,7 +479,7 @@ public void exitChar(RuleLangParser.CharContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { - final String text = unquote(ctx.getText(), '\"'); + final String text = unescape(unquote(ctx.getText(), '\"')); final StringExpression expr = new StringExpression(ctx.getStart(), text); log.trace("STRING: ctx {} => {}", ctx, expr); exprs.put(ctx, expr); @@ -869,7 +874,7 @@ public void exitInteger(RuleLangParser.IntegerContext ctx) { @Override public void exitString(RuleLangParser.StringContext ctx) { - final String text = unquote(ctx.getText(), '\"'); + final String text = unescape(unquote(ctx.getText(), '\"')); final StringExpression expr = new StringExpression(ctx.getStart(), text); log.trace("STRING: ctx {} => {}", ctx, expr); parseContext.exprs.put(ctx, expr); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index aae745e294f6..32ac359a73cd 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -305,6 +305,8 @@ public void strings() { assertThat(message).isNotNull(); assertThat(message.getField("has_xyz")).isInstanceOf(Boolean.class); assertThat((boolean)message.getField("has_xyz")).isFalse(); + assertThat(message.getField("string_literal")).isInstanceOf(String.class); + assertThat((String)message.getField("string_literal")).isEqualTo("abcd\\.e\tfg\u03a9\363"); } @Test diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt index 9aed823708da..670d7ee103a6 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt @@ -1,9 +1,8 @@ rule "regexMatch" when -// the second matcher group is always empty and will not be present in the result - regex(".*(cde)(:(\\d+))?.*", "abcdefg").matches == true + regex(".*(cde\\.)(:(\\d+))?.*", "abcde.fg").matches == true then - let result = regex(".*(cde).*", "abcdefg"); + let result = regex(".*(cd\\.e).*", "abcd.efg"); set_field("group_1", result["0"]); set_field("matched_regex", result.matches); end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt index 5c4c974883c7..ac3a92fa7af9 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt @@ -17,5 +17,6 @@ when concat("foo", "bar") == "foobar" then set_field("has_xyz", contains("abcdef", "xyz")); + set_field("string_literal", "abcd\\.e\tfg\u03a9\363"); trigger_test(); end \ No newline at end of file From 2025d361a0cd4964961d3b07d10239fa95b0191c Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 17 May 2016 17:12:17 +0200 Subject: [PATCH 277/521] IpAddressConversion caught wrong exception (#32) The first try block caught a too specific version of the IllegalArgumentException, allowing the exception to unwind too much. Properly return null or the default value in this case. Fixes #28 (cherry picked from commit fc5b8a543f47ba85f9e9e65b98ddca3066994602) --- .../functions/ips/IpAddressConversion.java | 2 +- .../functions/FunctionsSnippetsTest.java | 9 +++++++++ .../pipelineprocessor/functions/ipMatchingIssue28.txt | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index 099b30ada849..7715dd0f6a02 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -47,7 +47,7 @@ public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { try { final InetAddress inetAddress = InetAddresses.forString(ipString); return new IpAddress(inetAddress); - } catch (IllegalFormatException e) { + } catch (IllegalArgumentException e) { final Optional defaultValue = defaultParam.optional(args, context); if (!defaultValue.isPresent()) { return null; diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 32ac359a73cd..a0470eb5e545 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -443,4 +443,13 @@ public void syslog() { assertThat(message.getField("prio4_facility")).isEqualTo("local4"); assertThat(message.getField("prio4_level")).isEqualTo("Notice"); } + + @Test + public void ipMatchingIssue28() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message in = new Message("some message", "somehost.graylog.org", Tools.nowUTC()); + evaluateRule(rule, in); + + assertThat(actionsTriggered.get()).isFalse(); + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt new file mode 100644 index 000000000000..7dcd035e63c5 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt @@ -0,0 +1,6 @@ +rule "IP subnet" +when + cidr_match("10.20.30.0/24", to_ip($message.source)) +then + trigger_test(); +end \ No newline at end of file From 912802ac82cd687e7e489e09e8f9e4c8c876fff2 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 20 Jul 2016 12:15:41 +0200 Subject: [PATCH 278/521] add rename_field function (#50) this is a convenience function which renames a field, if present. it takes care not to drop fields if they are renamed to themselves fixes #37 --- .../functions/ProcessorFunctionsModule.java | 2 + .../functions/messages/RenameField.java | 54 +++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 17 ++++++ .../functions/fieldRenaming.txt | 8 +++ 4 files changed, 81 insertions(+) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 1a9a3cea622f..9c426601e87b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -44,6 +44,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RenameField; import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; @@ -78,6 +79,7 @@ protected void configure() { addMessageProcessorFunction(HasField.NAME, HasField.class); addMessageProcessorFunction(SetField.NAME, SetField.class); addMessageProcessorFunction(SetFields.NAME, SetFields.class); + addMessageProcessorFunction(RenameField.NAME, RenameField.class); addMessageProcessorFunction(RemoveField.NAME, RemoveField.class); addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java new file mode 100644 index 000000000000..7005ec516e39 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java @@ -0,0 +1,54 @@ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class RenameField extends AbstractFunction { + + public static final String NAME = "rename_field"; + + private final ParameterDescriptor oldFieldParam; + private final ParameterDescriptor newFieldParam; + private final ParameterDescriptor messageParam; + + public RenameField() { + oldFieldParam = string("old_field").build(); + newFieldParam = string("new_field").build(); + messageParam = type("message", Message.class).optional().build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final String oldName = oldFieldParam.required(args, context); + final String newName = newFieldParam.required(args, context); + + // exit early if the field names are the same (so we don't drop the field) + if (oldName != null && oldName.equals(newName)) { + return null; + } + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + + if (message.hasField(oldName)) { + message.addField(newName, message.getField(oldName)); + message.removeField(oldName); + } + + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(oldFieldParam, newFieldParam, messageParam) + .build(); + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index a0470eb5e545..8979e6c34202 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -49,6 +49,7 @@ import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RenameField; import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; @@ -113,6 +114,7 @@ public static void registerFunctions() { functions.put(HasField.NAME, new HasField()); functions.put(SetField.NAME, new SetField()); functions.put(SetFields.NAME, new SetFields()); + functions.put(RenameField.NAME, new RenameField()); functions.put(RemoveField.NAME, new RemoveField()); functions.put(DropMessage.NAME, new DropMessage()); @@ -452,4 +454,19 @@ public void ipMatchingIssue28() { assertThat(actionsTriggered.get()).isFalse(); } + + @Test + public void fieldRenaming() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final Message in = new Message("some message", "somehost.graylog.org", Tools.nowUTC()); + in.addField("field_a", "fieldAContent"); + in.addField("field_b", "not deleted"); + + final Message message = evaluateRule(rule, in); + + assertThat(message.hasField("field_1")).isFalse(); + assertThat(message.hasField("field_2")).isTrue(); + assertThat(message.hasField("field_b")).isTrue(); + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt new file mode 100644 index 000000000000..1fc8c285c622 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt @@ -0,0 +1,8 @@ +rule "fieldRenaming" +when true +then + + rename_field("no_such_field", "field_1"); + rename_field("field_a", "field_2"); + rename_field("field_b", "field_b"); +end \ No newline at end of file From cf55dcb111bc0769442bef8c0fd1820648213498 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 20 Jul 2016 12:16:23 +0200 Subject: [PATCH 279/521] Call `finishProcessing` in the right place We should call `InterpreterListener.finishProcessing()` only once, but before that was not necessarily true, as it was being called inside a loop. Instead, now we call it in the same method as `startProcessing()`, after all processing has been finished. Fixes #51 --- .../pipelineprocessor/processors/PipelineInterpreter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 0387d7ee4315..ad61b9cb2701 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -265,6 +265,8 @@ public Messages process(Messages messages, InterpreterListener interpreterListen } } } + + interpreterListener.finishProcessing(); // 7. return the processed messages return new MessageCollection(fullyProcessed); } @@ -374,7 +376,6 @@ public List processForPipelines(Message message, String msgId, Set Date: Wed, 20 Jul 2016 15:43:03 +0200 Subject: [PATCH 280/521] Adapt to changed decorators interface (#43) * Providing a message decorator that uses pipelines. * Making decorator configurable. * Allow adding new messages by pipeline decorator. * Adding changes related due to introduced listener. * Adapt to naming changes, using easier forEach idiom. * Changing decorator to work on SearchResponse instead of message list. --- .../PipelineProcessorMessageDecorator.java | 32 +++++++++---------- .../PipelineProcessorModule.java | 4 ++- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java index b6ff46f07332..7ae33b15c3c9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java @@ -19,34 +19,33 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Multimap; import com.google.inject.assistedinject.Assisted; import org.graylog.plugins.pipelineprocessor.db.PipelineDao; import org.graylog.plugins.pipelineprocessor.db.PipelineService; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; import org.graylog2.decorators.Decorator; -import org.graylog2.indexer.results.ResultMessage; import org.graylog2.plugin.Message; import org.graylog2.plugin.configuration.ConfigurationRequest; import org.graylog2.plugin.configuration.fields.ConfigurationField; import org.graylog2.plugin.configuration.fields.DropdownField; -import org.graylog2.plugin.decorators.MessageDecorator; +import org.graylog2.plugin.decorators.SearchResponseDecorator; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; +import org.graylog2.rest.resources.search.responses.SearchResponse; import javax.inject.Inject; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -public class PipelineProcessorMessageDecorator implements MessageDecorator { +public class PipelineProcessorMessageDecorator implements SearchResponseDecorator { private static final String CONFIG_FIELD_PIPELINE = "pipeline"; private final PipelineInterpreter pipelineInterpreter; private final ImmutableSet pipelines; - public interface Factory extends MessageDecorator.Factory { + public interface Factory extends SearchResponseDecorator.Factory { @Override PipelineProcessorMessageDecorator create(Decorator decorator); @@ -57,7 +56,7 @@ public interface Factory extends MessageDecorator.Factory { Descriptor getDescriptor(); } - public static class Config implements MessageDecorator.Config { + public static class Config implements SearchResponseDecorator.Config { private final PipelineService pipelineService; @Inject @@ -81,7 +80,7 @@ public ConfigurationRequest getRequestedConfiguration() { }; } - public static class Descriptor extends MessageDecorator.Descriptor { + public static class Descriptor extends SearchResponseDecorator.Descriptor { public Descriptor() { super("Pipeline Processor Decorator", false, "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Pipeline Processor Decorator"); } @@ -100,26 +99,25 @@ public PipelineProcessorMessageDecorator(PipelineInterpreter pipelineInterpreter } @Override - public List apply(List resultMessages) { - final List results = new ArrayList<>(); + public SearchResponse apply(SearchResponse searchResponse) { + final List results = new ArrayList<>(); if (pipelines.isEmpty()) { - return resultMessages; + return searchResponse; } - resultMessages.forEach((inMessage) -> { - final Message message = inMessage.getMessage(); + searchResponse.messages().forEach((inMessage) -> { + final Message message = new Message(inMessage.message()); final List additionalCreatedMessages = pipelineInterpreter.processForPipelines(message, message.getId(), pipelines, new NoopInterpreterListener()); - final ResultMessage outMessage = ResultMessage.createFromMessage(message, inMessage.getIndex(), inMessage.getHighlightRanges()); - results.add(outMessage); + results.add(ResultMessageSummary.create(inMessage.highlightRanges(), message.getFields(), inMessage.index())); additionalCreatedMessages.forEach((additionalMessage) -> { // TODO: pass proper highlight ranges. Need to rebuild them for new messages. - results.add(ResultMessage.createFromMessage(additionalMessage, "[created from decorator]", ImmutableMultimap.of())); + results.add(ResultMessageSummary.create(ImmutableMultimap.of(), additionalMessage.getFields(), "[created from decorator]")); }); }); - return results; + return searchResponse.toBuilder().messages(results).build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 578c83ed05f2..e93bae59a2d4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -47,6 +47,8 @@ protected void configure() { install(new ProcessorFunctionsModule()); - installMessageDecorator(messageDecoratorBinder(), PipelineProcessorMessageDecorator.class, PipelineProcessorMessageDecorator.Factory.class); + installSearchResponseDecorator(searchResponseDecoratorBinder(), + PipelineProcessorMessageDecorator.class, + PipelineProcessorMessageDecorator.Factory.class); } } From ca8d894cbb59a8289fc8847b5a158b3ca6330ec3 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 20 Jul 2016 18:08:41 +0200 Subject: [PATCH 281/521] Improve error message on regex() --- .../pipelineprocessor/functions/strings/RegexMatch.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index 46d6dfaf3e14..90d4642859c0 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -52,7 +52,8 @@ public RegexMatchResult evaluate(FunctionArgs args, EvaluationContext context) { final Pattern regex = pattern.required(args, context); final String value = this.value.required(args, context); if (regex == null || value == null) { - throw new IllegalArgumentException(); + final String nullArgument = regex == null ? "pattern" : "value"; + throw new IllegalArgumentException("Argument '" + nullArgument + "' cannot be 'null'"); } //noinspection unchecked final List groupNames = From 3fa6e5bcba3fdd861422ed89321f2ea6ee7f689e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 20 Jul 2016 18:09:15 +0200 Subject: [PATCH 282/521] Update evalError test rule Since #32, the behaviour of `to_ip()` changed and now it doesn't raise an exception in those circumstances. Now we use `regex()` to throw an exception instead. --- .../pipelineprocessor/functions/FunctionsSnippetsTest.java | 2 +- .../graylog/plugins/pipelineprocessor/functions/evalError.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 8979e6c34202..2c0cc7fd5174 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -332,7 +332,7 @@ public void evalError() { assertThat(context).isNotNull(); assertThat(context.hasEvaluationErrors()).isTrue(); - assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'to_ip' at 5:28 an exception was thrown: 'null' is not an IP string literal."); + assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'regex' at 5:28 an exception was thrown: Argument 'value' cannot be 'null'"); } @Test diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt index 33d692fdc9b2..c2fd322ff66a 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt @@ -2,5 +2,5 @@ rule "trigger null" when true then - set_field("this_is_null", to_ip($message.does_not_exist)); + set_field("do_not_exist", regex(".*", to_string($message.do_not_exist)).matches); end From 2038504d2f83e45a8cabde7e4b22e769946bab63 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Fri, 22 Jul 2016 17:38:20 +0200 Subject: [PATCH 283/521] Add sample Decorator preset. (#52) * Providing a message decorator that uses pipelines. * Making decorator configurable. * Allow adding new messages by pipeline decorator. * Adding changes related due to introduced listener. * Adapt to naming changes, using easier forEach idiom. * Changing decorator to work on SearchResponse instead of message list. * Adding decoration stats for pipeline processor decorator. * Add uppercase decorator using pipelines interpreter with preset. * Decorators don't need to generate decoration stats on their own anymore. --- .../PipelineProcessorMessageDecorator.java | 9 +- .../PipelineProcessorModule.java | 3 + .../pipelineprocessor/UpperCaseDecorator.java | 112 ++++++++++++++++++ .../processors/PipelineInterpreter.java | 14 ++- 4 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java index 7ae33b15c3c9..04a84cf5bc0f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java @@ -17,6 +17,7 @@ package org.graylog.plugins.pipelineprocessor; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.inject.assistedinject.Assisted; @@ -82,7 +83,7 @@ public ConfigurationRequest getRequestedConfiguration() { public static class Descriptor extends SearchResponseDecorator.Descriptor { public Descriptor() { - super("Pipeline Processor Decorator", false, "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Pipeline Processor Decorator"); + super("Pipeline Processor Decorator", "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Pipeline Processor Decorator"); } } @@ -114,7 +115,11 @@ public SearchResponse apply(SearchResponse searchResponse) { results.add(ResultMessageSummary.create(inMessage.highlightRanges(), message.getFields(), inMessage.index())); additionalCreatedMessages.forEach((additionalMessage) -> { // TODO: pass proper highlight ranges. Need to rebuild them for new messages. - results.add(ResultMessageSummary.create(ImmutableMultimap.of(), additionalMessage.getFields(), "[created from decorator]")); + results.add(ResultMessageSummary.create( + ImmutableMultimap.of(), + additionalMessage.getFields(), + "[created from decorator]" + )); }); }); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index e93bae59a2d4..2c915d3d34a6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -50,5 +50,8 @@ protected void configure() { installSearchResponseDecorator(searchResponseDecoratorBinder(), PipelineProcessorMessageDecorator.class, PipelineProcessorMessageDecorator.Factory.class); + installSearchResponseDecorator(searchResponseDecoratorBinder(), + UpperCaseDecorator.class, + UpperCaseDecorator.Factory.class); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java new file mode 100644 index 000000000000..84f9a5343912 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java @@ -0,0 +1,112 @@ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.inject.assistedinject.Assisted; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; +import org.graylog2.decorators.Decorator; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.configuration.ConfigurationRequest; +import org.graylog2.plugin.configuration.fields.TextField; +import org.graylog2.plugin.decorators.SearchResponseDecorator; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; +import org.graylog2.rest.resources.search.responses.SearchResponse; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +public class UpperCaseDecorator implements SearchResponseDecorator { + private static final String CK_FIELD_NAME = "fieldName"; + private static final String CK_PIPELINE_DEFINITION = "pipeline \"Uppercase decorator\"\nstage 0 match either\nrule \"Uppercase field\"\nend"; + + private final List pipelines; + private final PipelineInterpreter pipelineInterpreter; + private final Decorator decorator; + + public interface Factory extends SearchResponseDecorator.Factory { + @Override + UpperCaseDecorator create(Decorator decorator); + + @Override + Config getConfig(); + + @Override + Descriptor getDescriptor(); + } + + public static class Config implements SearchResponseDecorator.Config { + @Inject + public Config() { + } + + @Override + public ConfigurationRequest getRequestedConfiguration() { + return new ConfigurationRequest() {{ + addField(new TextField(CK_FIELD_NAME, "Field Name", "", "The Name of the field which should be uppercased")); + }}; + }; + } + + public static class Descriptor extends SearchResponseDecorator.Descriptor { + public Descriptor() { + super("Uppercase Decorator", "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Uppercase Decorator"); + } + } + + @Inject + public UpperCaseDecorator(PipelineInterpreter pipelineInterpreter, + PipelineRuleParser pipelineRuleParser, + @Assisted Decorator decorator) { + this.pipelineInterpreter = pipelineInterpreter; + this.decorator = decorator; + final String fieldName = (String)decorator.config().get(CK_FIELD_NAME); + + this.pipelines = pipelineRuleParser.parsePipelines(CK_PIPELINE_DEFINITION); + final List rules = ImmutableList.of(pipelineRuleParser.parseRule(getRuleForField(fieldName), true)); + this.pipelines.forEach(pipeline -> { + pipeline.stages().forEach(stage -> stage.setRules(rules)); + }); + } + + @Override + public SearchResponse apply(SearchResponse searchResponse) { + final List results = new ArrayList<>(); + searchResponse.messages().forEach((inMessage) -> { + final Map originalMessage = ImmutableMap.copyOf(inMessage.message()); + final Message message = new Message(inMessage.message()); + final List additionalCreatedMessages = pipelineInterpreter.processForResolvedPipelines(message, + message.getId(), + new HashSet<>(this.pipelines), + new NoopInterpreterListener()); + + results.add(ResultMessageSummary.create(inMessage.highlightRanges(), message.getFields(), inMessage.index())); + additionalCreatedMessages.forEach((additionalMessage) -> { + // TODO: pass proper highlight ranges. Need to rebuild them for new messages. + results.add(ResultMessageSummary.create( + ImmutableMultimap.of(), + additionalMessage.getFields(), + "[created from decorator]" + )); + }); + }); + + return searchResponse.toBuilder().messages(results).build(); + } + + private String getRuleForField(String fieldName) { + return "rule \"Uppercase field\"\n" + + "when\n" + + "has_field(\"" + fieldName + "\")\n" + + "then\n" + + "set_field(\"" + fieldName + "\", uppercase(to_string($message." + fieldName + ")));\n" + + "end"; + } +} diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index ad61b9cb2701..888735ee4fa4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -272,12 +272,20 @@ public Messages process(Messages messages, InterpreterListener interpreterListen } public List processForPipelines(Message message, String msgId, Set pipelines, InterpreterListener interpreterListener) { - final ImmutableSet pipelinesToRun = ImmutableSet.copyOf(pipelines.stream().map(pipelineId -> this.currentPipelines.get().get(pipelineId)).collect(Collectors.toSet())); + final ImmutableSet pipelinesToRun = ImmutableSet.copyOf(pipelines + .stream() + .map(pipelineId -> this.currentPipelines.get().get(pipelineId)) + .collect(Collectors.toSet())); + + return processForResolvedPipelines(message, msgId, pipelinesToRun, interpreterListener); + } + + public List processForResolvedPipelines(Message message, String msgId, Set pipelines, InterpreterListener interpreterListener) { final List result = new ArrayList<>(); // record execution of pipeline in metrics - pipelinesToRun.stream().forEach(pipeline -> metricRegistry.counter(name(Pipeline.class, pipeline.id(), "executed")).inc()); + pipelines.forEach(pipeline -> metricRegistry.counter(name(Pipeline.class, pipeline.id(), "executed")).inc()); - final StageIterator stages = new StageIterator(pipelinesToRun); + final StageIterator stages = new StageIterator(pipelines); final Set pipelinesToSkip = Sets.newHashSet(); // iterate through all stages for all matching pipelines, per "stage slice" instead of per pipeline. From 7e9cf3fef26b1c413d350e11001fe6321490874a Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 25 Jul 2016 17:22:28 +0200 Subject: [PATCH 284/521] Bumping versions to 2.1.0-beta.1 / 1.1.0-beta.1 --- package.json | 2 +- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index e60f1c7a9a95..ab44e11a72b6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PipelineProcessor", - "version": "1.1.0-alpha.2", + "version": "1.1.0-beta.1", "description": "", "repository": { "type": "git", diff --git a/pom.xml b/pom.xml index bb43e2c899fe..16603634852f 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-alpha.3-SNAPSHOT + 2.1.0-beta.1 /usr/share/graylog-server/plugin 4.5.1 1.7.13 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index cee9ae91d60b..cb3d550d911d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 1, 0, "alpha.2"); + return new Version(1, 1, 0, "beta.1"); } @Override @@ -60,7 +60,7 @@ public String getDescription() { @Override public Version getRequiredVersion() { - return new Version(2, 1, 0, "alpha.2"); + return new Version(2, 1, 0, "beta.1"); } @Override From 8d844b48148dbfd8df216c2bfe02e203b4b1ee17 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Mon, 25 Jul 2016 15:30:43 +0000 Subject: [PATCH 285/521] [graylog-plugin-pipeline-processor-release] prepare release 1.1.0-beta.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 16603634852f..67f827264282 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-alpha.3-SNAPSHOT + 1.1.0-beta.1 jar ${project.artifactId} From f9e8ef83e1498e1d0ead2be73ad011dde6e5c773 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Mon, 25 Jul 2016 15:30:53 +0000 Subject: [PATCH 286/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67f827264282..9b762ca097bf 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.1 + 1.1.0-beta.2-SNAPSHOT jar ${project.artifactId} From 39a232682e5c83b0ff04dc7b72f24add9cbbb866 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 25 Jul 2016 18:32:41 +0200 Subject: [PATCH 287/521] Bumping graylog dependency version to next development iteration. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9b762ca097bf..5071d6429fc4 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.1 + 2.1.0-beta.2-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.13 From ff550e35ff72db39e56b3cb05a302b0d70119188 Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Tue, 26 Jul 2016 12:43:19 +0200 Subject: [PATCH 288/521] Making pipeline decorator more robust if removed pipelines are ref'd. --- .../pipelineprocessor/processors/PipelineInterpreter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 888735ee4fa4..38f1fa1c4c03 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -275,6 +275,7 @@ public List processForPipelines(Message message, String msgId, Set pipelinesToRun = ImmutableSet.copyOf(pipelines .stream() .map(pipelineId -> this.currentPipelines.get().get(pipelineId)) + .filter(pipeline -> pipeline != null) .collect(Collectors.toSet())); return processForResolvedPipelines(message, msgId, pipelinesToRun, interpreterListener); From 1c1ea655a71f25241e34525d2ca001e2d3effcec Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 26 Jul 2016 14:41:45 +0200 Subject: [PATCH 289/521] Fix permissions on system navigation route --- src/web/index.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/index.jsx b/src/web/index.jsx index ef349c062969..97b765ebae06 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -18,6 +18,6 @@ PluginStore.register(new PluginManifest(packageJson, { ], systemnavigation: [ - { path: '/system/pipelines', description: 'Pipelines', permissions: 'INPUTS_CREATE' }, + { path: '/system/pipelines', description: 'Pipelines', permissions: 'inputs:create' }, ], })); From 1761cd96cf988150d684f516f31348876d9b4bba Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 27 Jul 2016 15:10:31 +0200 Subject: [PATCH 290/521] Update npm dependencies Refs Graylog2/graylog2-server#2544 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ab44e11a72b6..d7fd5610639d 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "json-loader": "^0.5.4", "react-hot-loader": "^1.3.0", "style-loader": "^0.13.0", - "ts-loader": "^0.8.0", + "ts-loader": "^0.8.2", "webpack": "^1.12.2" } } From 2641c97f848968ad5e6ef79c710633507c7b4b01 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Fri, 29 Jul 2016 15:10:36 +0200 Subject: [PATCH 291/521] Fix cancel button on new pipeline form Fixes #57 --- src/web/pipelines/PipelineDetails.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/web/pipelines/PipelineDetails.jsx b/src/web/pipelines/PipelineDetails.jsx index d54737958275..0fef392fed26 100644 --- a/src/web/pipelines/PipelineDetails.jsx +++ b/src/web/pipelines/PipelineDetails.jsx @@ -11,11 +11,12 @@ const PipelineDetails = React.createClass({ pipeline: React.PropTypes.object, create: React.PropTypes.bool, onChange: React.PropTypes.func.isRequired, + onCancel: React.PropTypes.func, }, render() { if (this.props.create) { - return ; + return ; } const pipeline = this.props.pipeline; From 930e433db10e604abd3163ea02bf317282cd6d11 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 12:45:12 +0200 Subject: [PATCH 292/521] Update Travis CI configuration Disable building the web-part of this project because it would require a full checkout of the Graylog web interface. --- .travis.yml | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6dd9f083839..c6ade1bca41d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,25 +2,10 @@ sudo: false language: java jdk: - oraclejdk8 -addons: - apt: - packages: - - rpm -before_deploy: - - mvn jdeb:jdeb && export RELEASE_DEB_FILE=$(ls target/*.deb) - - mvn rpm:rpm && export RELEASE_RPM_FILE=$(find target/ -name '*.rpm' | tail -1) - - rm -f target/original-*.jar - - export RELEASE_PKG_FILE=$(ls target/*.jar) - - echo "Deploying release to GitHub releases" -deploy: - provider: releases - api_key: - secure: - file: - - "${RELEASE_PKG_FILE}" - - "${RELEASE_DEB_FILE}" - - "${RELEASE_RPM_FILE}" - skip_cleanup: true - on: - tags: true - jdk: oraclejdk8 +env: + global: + - MAVEN_OPTS="-Dskip.web.build=true" +install: + - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V +script: + - mvn test -B From d2a5b2d9228fb1b55cf1956d01bfa5a2ef64c92f Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 12:46:01 +0200 Subject: [PATCH 293/521] Add missing license headers --- .../pipelineprocessor/UpperCaseDecorator.java | 16 ++++++++++++++++ .../functions/messages/RenameField.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java index 84f9a5343912..e11a1246d28c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor; import com.google.common.collect.ImmutableList; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java index 7005ec516e39..2bc096ec1861 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java @@ -1,3 +1,19 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ package org.graylog.plugins.pipelineprocessor.functions.messages; import org.graylog.plugins.pipelineprocessor.EvaluationContext; From 210fda5d43920d7e73126cc1989a8f4f62877f6f Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 13:02:28 +0200 Subject: [PATCH 294/521] Move -Dskip.web.build=true into install/script commands (Travis CI) --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6ade1bca41d..fbb6846a5466 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,7 @@ sudo: false language: java jdk: - oraclejdk8 -env: - global: - - MAVEN_OPTS="-Dskip.web.build=true" install: - - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V + - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.web.build=true -B -V script: - - mvn test -B + - mvn test -Dskip.web.build=true -B From 1caa2b11fe1a580677f72676a170c65960e141d3 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 13:04:23 +0200 Subject: [PATCH 295/521] Fix Travis CI badge in README.md [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5dc6a4bfe5df..a36945f66929 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PipelineProcessor Plugin for Graylog -[![Build Status](https://travis-ci.org/graylog-plugin-pipeline-processor.svg?branch=master)](https://travis-ci.org/graylog-plugin-pipeline-processor) +[![Build Status](https://travis-ci.org/Graylog2/graylog-plugin-pipeline-processor.svg?branch=master)](https://travis-ci.org/Graylog2/graylog-plugin-pipeline-processor) __Use this paragraph to enter a description of your plugin.__ From 90e20f1e80cb85780aa8514fb1c55d512b682d5a Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 13:05:10 +0200 Subject: [PATCH 296/521] Run on Trusty build environment (Travis CI) --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fbb6846a5466..f8dd3acae5bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ -sudo: false +sudo: required +dist: trusty language: java jdk: - oraclejdk8 From 56b801b13751f3b1c87f6497abc507db0f662e22 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Mon, 1 Aug 2016 17:27:32 +0200 Subject: [PATCH 297/521] Fix issues with app prefix (#66) * Use core Routes instead of literals This allows us to prefix routes if needed. * Make plugin aware of __webpack_public_path__ setting Refs Graylog2/graylog2-server#2564 --- src/web/index.jsx | 3 +++ src/web/pipeline-connections/Connection.jsx | 6 ++++-- src/web/pipeline-connections/ConnectionForm.jsx | 6 ++++-- src/web/pipeline-connections/PipelineConnectionsPage.jsx | 5 +++-- src/web/pipelines/NewPipeline.jsx | 4 +++- src/web/pipelines/PipelineDetailsPage.jsx | 6 ++++-- src/web/pipelines/PipelinesOverviewPage.jsx | 6 ++++-- src/web/pipelines/ProcessingTimelineComponent.jsx | 8 +++++--- src/web/pipelines/Stage.jsx | 4 +++- src/web/pipelines/StageForm.jsx | 4 +++- src/web/rules/Rule.jsx | 6 ++++-- src/web/rules/RuleForm.jsx | 4 +++- src/web/rules/RuleList.jsx | 8 +++++--- src/web/rules/RulesPage.jsx | 6 ++++-- src/web/simulator/SimulatorPage.jsx | 5 +++-- 15 files changed, 55 insertions(+), 26 deletions(-) diff --git a/src/web/index.jsx b/src/web/index.jsx index 97b765ebae06..6d8c41cacc65 100644 --- a/src/web/index.jsx +++ b/src/web/index.jsx @@ -1,3 +1,6 @@ +// eslint-disable-next-line no-unused-vars +import webpackEntry from 'webpack-entry'; + import packageJson from '../../package.json'; import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage'; diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx index 1e821628fbf2..c31b008afa54 100644 --- a/src/web/pipeline-connections/Connection.jsx +++ b/src/web/pipeline-connections/Connection.jsx @@ -5,6 +5,8 @@ import { LinkContainer } from 'react-router-bootstrap'; import { DataTable, EntityListItem, Timestamp } from 'components/common'; import ConnectionForm from './ConnectionForm'; +import Routes from 'routing/Routes'; + const Connection = React.createClass({ propTypes: { stream: React.PropTypes.object.isRequired, @@ -20,7 +22,7 @@ const Connection = React.createClass({ return ( - {pipeline.title} + {pipeline.title} {pipeline.description} @@ -47,7 +49,7 @@ const Connection = React.createClass({ render() { const actions = [ - + , Select the stream you want to connect pipelines to, or create one in the{' '} - Streams page. + Streams page. ); streamSelector = ( @@ -131,7 +133,7 @@ const ConnectionForm = React.createClass({ const pipelineHelp = ( Select the pipelines to connect to this stream, or create one in the{' '} - Pipelines Overview page. + Pipelines Overview page. ); diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx index 71cbe32a6217..237c80782bef 100644 --- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx +++ b/src/web/pipeline-connections/PipelineConnectionsPage.jsx @@ -16,6 +16,7 @@ import StoreProvider from 'injection/StoreProvider'; const StreamsStore = StoreProvider.getStore('Streams'); import DocsHelper from 'util/DocsHelper'; +import Routes from 'routing/Routes'; const PipelineConnectionsPage = React.createClass({ mixins: [Reflux.connect(PipelinesStore), Reflux.connect(PipelineConnectionsStore)], @@ -71,11 +72,11 @@ const PipelineConnectionsPage = React.createClass({ - +   - + diff --git a/src/web/pipelines/NewPipeline.jsx b/src/web/pipelines/NewPipeline.jsx index 7bb4aacae0d6..b13bdbeede08 100644 --- a/src/web/pipelines/NewPipeline.jsx +++ b/src/web/pipelines/NewPipeline.jsx @@ -3,6 +3,8 @@ import { Row, Col } from 'react-bootstrap'; import PipelineDetails from './PipelineDetails'; +import Routes from 'routing/Routes'; + const NewPipeline = React.createClass({ propTypes: { onChange: React.PropTypes.func.isRequired, @@ -14,7 +16,7 @@ const NewPipeline = React.createClass({ }, _goToPipeline(pipeline) { - this.props.history.pushState(null, `/system/pipelines/${pipeline.id}`); + this.props.history.pushState(null, Routes.pluginRoute('SYSTEM_PIPELINES_PIPELINEID')(pipeline.id)); }, _goBack() { diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index 5aa7d65b5dd7..5d9533a1066c 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -13,6 +13,8 @@ import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; import RulesStore from 'rules/RulesStore'; +import Routes from 'routing/Routes'; + function filterPipeline(state) { return state.pipelines ? state.pipelines.filter(p => p.id === this.props.params.pipelineId)[0] : undefined; } @@ -101,11 +103,11 @@ const PipelineDetailsPage = React.createClass({ - + {' '} - + diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx index 1a16547e3261..8fb03744f00c 100644 --- a/src/web/pipelines/PipelinesOverviewPage.jsx +++ b/src/web/pipelines/PipelinesOverviewPage.jsx @@ -5,6 +5,8 @@ import { LinkContainer } from 'react-router-bootstrap'; import { PageHeader } from 'components/common'; import ProcessingTimelineComponent from './ProcessingTimelineComponent'; +import Routes from 'routing/Routes'; + const PipelinesOverviewPage = React.createClass({ render() { return ( @@ -19,11 +21,11 @@ const PipelinesOverviewPage = React.createClass({ - + {' '} - + diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 15daef3762ad..52162b3d03a5 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -10,6 +10,8 @@ import { MetricContainer, CounterRate } from 'components/metrics'; import PipelinesActions from 'pipelines/PipelinesActions'; import PipelinesStore from 'pipelines/PipelinesStore'; +import Routes from 'routing/Routes'; + const ProcessingTimelineComponent = React.createClass({ mixins: [Reflux.connect(PipelinesStore)], @@ -66,7 +68,7 @@ const ProcessingTimelineComponent = React.createClass({ return ( - {pipeline.title}
    + {pipeline.title}
    {pipeline.description}
    @@ -77,7 +79,7 @@ const ProcessingTimelineComponent = React.createClass({   - + @@ -100,7 +102,7 @@ const ProcessingTimelineComponent = React.createClass({ const addNewPipelineButton = (
    - +
    diff --git a/src/web/pipelines/Stage.jsx b/src/web/pipelines/Stage.jsx index 9f924d7f4e43..b3098950d0ad 100644 --- a/src/web/pipelines/Stage.jsx +++ b/src/web/pipelines/Stage.jsx @@ -8,6 +8,8 @@ import RulesStore from 'rules/RulesStore'; import StageForm from './StageForm'; import { MetricContainer, CounterRate } from 'components/metrics'; +import Routes from 'routing/Routes'; + const Stage = React.createClass({ propTypes: { stage: PropTypes.object.isRequired, @@ -34,7 +36,7 @@ const Stage = React.createClass({ }; ruleTitle = {stage.rules[ruleIdx]}; } else { - ruleTitle = ( + ruleTitle = ( {rule.title} ); diff --git a/src/web/pipelines/StageForm.jsx b/src/web/pipelines/StageForm.jsx index bf872f45c121..a2cb91e7054f 100644 --- a/src/web/pipelines/StageForm.jsx +++ b/src/web/pipelines/StageForm.jsx @@ -10,6 +10,8 @@ import FormsUtils from 'util/FormsUtils'; import RulesStore from 'rules/RulesStore'; +import Routes from 'routing/Routes'; + const StageForm = React.createClass({ propTypes: { stage: PropTypes.object, @@ -89,7 +91,7 @@ const StageForm = React.createClass({ const rulesHelp = ( Select the rules evaluated on this stage, or create one in the{' '} - Pipeline Rules page. + Pipeline Rules page. ); diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx index 355d9da52e30..f8e61ede5564 100644 --- a/src/web/rules/Rule.jsx +++ b/src/web/rules/Rule.jsx @@ -10,6 +10,8 @@ import DocsHelper from 'util/DocsHelper'; import RuleForm from './RuleForm'; import RuleHelper from './RuleHelper'; +import Routes from 'routing/Routes'; + const Rule = React.createClass({ propTypes: { rule: React.PropTypes.object, @@ -42,11 +44,11 @@ const Rule = React.createClass({ - +   - + diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index 88ea4122a35f..ac756a4c0a17 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -7,6 +7,8 @@ import brace from 'brace'; import 'brace/mode/text'; import 'brace/theme/chrome'; +import Routes from 'routing/Routes'; + const RuleForm = React.createClass({ propTypes: { rule: PropTypes.object, @@ -103,7 +105,7 @@ const RuleForm = React.createClass({ }, _saved() { - this.props.history.pushState(null, '/system/pipelines/rules'); + this.props.history.pushState(null, Routes.pluginRoute('SYSTEM_PIPELINES_RULES')); }, _save() { diff --git a/src/web/rules/RuleList.jsx b/src/web/rules/RuleList.jsx index 3a60838a3f1d..e3e7490f2bbd 100644 --- a/src/web/rules/RuleList.jsx +++ b/src/web/rules/RuleList.jsx @@ -8,6 +8,8 @@ import RulesActions from './RulesActions'; import { MetricContainer, CounterRate } from 'components/metrics'; +import Routes from 'routing/Routes'; + const RuleList = React.createClass({ propTypes: { rules: PropTypes.array.isRequired, @@ -31,7 +33,7 @@ const RuleList = React.createClass({ Delete ,  , - + , ]; @@ -39,7 +41,7 @@ const RuleList = React.createClass({ return ( - + {rule.title} @@ -77,7 +79,7 @@ const RuleList = React.createClass({ filterLabel="Filter Rules" filterKeys={filterKeys}>
    - +
    diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx index 58642af23c97..913539009b19 100644 --- a/src/web/rules/RulesPage.jsx +++ b/src/web/rules/RulesPage.jsx @@ -13,6 +13,8 @@ import RulesComponent from './RulesComponent'; import RulesStore from './RulesStore'; import RulesActions from './RulesActions'; +import Routes from 'routing/Routes'; + const RulesPage = React.createClass({ mixins: [ Reflux.connect(RulesStore), @@ -36,11 +38,11 @@ const RulesPage = React.createClass({ - +   - + diff --git a/src/web/simulator/SimulatorPage.jsx b/src/web/simulator/SimulatorPage.jsx index 175d0fa6d451..114baa21e69e 100644 --- a/src/web/simulator/SimulatorPage.jsx +++ b/src/web/simulator/SimulatorPage.jsx @@ -16,6 +16,7 @@ import StoreProvider from 'injection/StoreProvider'; const StreamsStore = StoreProvider.getStore('Streams'); import DocsHelper from 'util/DocsHelper'; +import Routes from 'routing/Routes'; const SimulatorPage = React.createClass({ propTypes: { @@ -78,11 +79,11 @@ const SimulatorPage = React.createClass({ - +   - + From 84a9d6848cba741c1246d998f42b8926cf332d05 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 1 Aug 2016 21:48:52 +0200 Subject: [PATCH 298/521] Support "only named captures" for pipeline grok function (#65) The server cache is necessary because the named captures support needs a separately compiled regex. So far the cache is only used by the grok function in the pipeline processor Closes #59 --- .../pipelineprocessor/functions/strings/GrokMatch.java | 9 +++++++-- .../functions/FunctionsSnippetsTest.java | 8 ++++++-- .../graylog/plugins/pipelineprocessor/functions/grok.txt | 4 ++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java index de4c5c10ea31..6fa54a53211c 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java @@ -37,6 +37,8 @@ public class GrokMatch extends AbstractFunction { private final ParameterDescriptor valueParam; private final ParameterDescriptor patternParam; + private final ParameterDescriptor namedOnly; + private final GrokPatternRegistry grokPatternRegistry; @Inject @@ -45,17 +47,20 @@ public GrokMatch(GrokPatternRegistry grokPatternRegistry) { valueParam = ParameterDescriptor.string("value").build(); patternParam = ParameterDescriptor.string("pattern").build(); + namedOnly = ParameterDescriptor.bool("only_named_captures").optional().build(); } @Override public GrokResult evaluate(FunctionArgs args, EvaluationContext context) { final String value = valueParam.required(args, context); final String pattern = patternParam.required(args, context); + final boolean onlyNamedCaptures = namedOnly.optional(args, context).orElse(false); + if (value == null || pattern == null) { return null; } - final Grok grok = grokPatternRegistry.cachedGrokForPattern(pattern); + final Grok grok = grokPatternRegistry.cachedGrokForPattern(pattern, onlyNamedCaptures); final Match match = grok.match(value); match.captures(); @@ -67,7 +72,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(GrokResult.class) - .params(of(patternParam, valueParam)) + .params(of(patternParam, valueParam, namedOnly)) .build(); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 2c0cc7fd5174..7223e25e1229 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -183,7 +183,8 @@ public static void registerFunctions() { Set patterns = Sets.newHashSet( GrokPattern.create("GREEDY", ".*"), GrokPattern.create("BASE10NUM", "(?[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"), - GrokPattern.create("NUMBER", "(?:%{BASE10NUM:UNWANTED})") + GrokPattern.create("NUMBER", "(?:%{BASE10NUM:UNWANTED})"), + GrokPattern.create("NUM", "%{BASE10NUM}") ); when(grokPatternService.loadAll()).thenReturn(patterns); final EventBus clusterBus = new EventBus(); @@ -370,8 +371,11 @@ public void grok() { final Message message = evaluateRule(rule); assertThat(message).isNotNull(); - assertThat(message.getFieldCount()).isEqualTo(4); + assertThat(message.getFieldCount()).isEqualTo(5); assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2015-07-31T10:05:36.773Z")); + // named captures only + assertThat(message.hasField("num")).isTrue(); + assertThat(message.hasField("BASE10NUM")).isFalse(); } @Test diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt index 6578459ebf1e..49e57025f301 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt @@ -3,4 +3,8 @@ when true then let matches = grok(pattern: "%{GREEDY:timestamp;date;yyyy-MM-dd'T'HH:mm:ss.SSSX}", value: "2015-07-31T10:05:36.773Z"); set_fields(matches); + + // only named captures + let matches1 = grok("%{NUM:num}", "10", true); + set_fields(matches1); end From cdac60d3dd5d15a41ad6b231fdda409b506dd306 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 1 Aug 2016 21:50:31 +0200 Subject: [PATCH 299/521] Make conversion functions more consistent (#64) * there was a bug with to_string returning null instead of its default value (refs #63) * all core conversion functions now return their "default empty" value if the value is `null` - String: "" - bool: false - double: 0d - long 0L - IP: V4 ANY (0.0.0.0) * adds test cases for all cases, including the edge cases --- .../conversion/BooleanConversion.java | 16 ++++++-- .../conversion/StringConversion.java | 2 +- .../functions/ips/IpAddress.java | 14 +++++++ .../functions/ips/IpAddressConversion.java | 4 +- .../functions/FunctionsSnippetsTest.java | 41 +++++++++++++++++++ .../functions/conversions.txt | 30 ++++++++++++++ 6 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java index 500c3b055cb4..8f5fb5b9e1e2 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java @@ -23,20 +23,28 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.bool; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; public class BooleanConversion extends AbstractFunction { public static final String NAME = "to_bool"; - private final ParameterDescriptor valueParam; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + public BooleanConversion() { - valueParam = object("value", Boolean.class).transform(o -> Boolean.parseBoolean(String.valueOf(o))).build(); + valueParam = object("value").build(); + defaultParam = bool("default").optional().build(); } @Override public Boolean evaluate(FunctionArgs args, EvaluationContext context) { - return valueParam.required(args, context); + final Object value = valueParam.required(args, context); + if (value == null) { + return defaultParam.optional(args, context).orElse(false); + } + return Boolean.parseBoolean(String.valueOf(value)); } @Override @@ -44,7 +52,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Boolean.class) - .params(of(valueParam)) + .params(of(valueParam, defaultParam)) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 7590c9b9f354..5dc9a3a0797f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -60,7 +60,7 @@ protected boolean removeEldestEntry(Map.Entry, Class> eldest) { public String evaluate(FunctionArgs args, EvaluationContext context) { final Object evaluated = valueParam.required(args, context); if (evaluated == null) { - return null; + return defaultParam.optional(args, context).orElse(""); } // fast path for the most common targets if (evaluated instanceof String diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java index ddd7943b17b8..df6b3eb7feed 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java @@ -20,6 +20,7 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Objects; /** * Graylog's rule language wrapper for InetAddress. @@ -57,4 +58,17 @@ public IpAddress getAnonymized() { throw new IllegalStateException(e); } } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof IpAddress)) return false; + IpAddress ipAddress = (IpAddress) o; + return Objects.equals(address, ipAddress.address); + } + + @Override + public int hashCode() { + return Objects.hash(address); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index 7715dd0f6a02..78f3d3044605 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -32,6 +32,8 @@ public class IpAddressConversion extends AbstractFunction { public static final String NAME = "to_ip"; + private static final InetAddress ANYV4 = InetAddresses.forString("0.0.0.0"); + private final ParameterDescriptor ipParam; private final ParameterDescriptor defaultParam; @@ -50,7 +52,7 @@ public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { } catch (IllegalArgumentException e) { final Optional defaultValue = defaultParam.optional(args, context); if (!defaultValue.isPresent()) { - return null; + return new IpAddress(ANYV4); } try { return new IpAddress(InetAddresses.forString(defaultValue.get())); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 7223e25e1229..31ad2d75e0ec 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; +import com.google.common.net.InetAddresses; import org.graylog.plugins.pipelineprocessor.BaseParserTest; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.Rule; @@ -42,6 +43,7 @@ import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; import org.graylog.plugins.pipelineprocessor.functions.ips.CidrMatch; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddress; import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddressConversion; import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; @@ -473,4 +475,43 @@ public void fieldRenaming() { assertThat(message.hasField("field_2")).isTrue(); assertThat(message.hasField("field_b")).isTrue(); } + + @Test + public void conversions() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final EvaluationContext context = contextForRuleEval(rule, new Message("test", "test", Tools.nowUTC())); + + assertThat(context.evaluationErrors()).isEmpty(); + final Message message = context.currentMessage(); + + assertNotNull(message); + assertThat(message.getField("string_1")).isEqualTo("1"); + assertThat(message.getField("string_2")).isEqualTo("2"); + // special case, Message doesn't allow adding fields with empty string values + assertThat(message.hasField("string_3")).isFalse(); + assertThat(message.getField("string_4")).isEqualTo("default"); + + assertThat(message.getField("long_1")).isEqualTo(1L); + assertThat(message.getField("long_2")).isEqualTo(2L); + assertThat(message.getField("long_3")).isEqualTo(0L); + assertThat(message.getField("long_4")).isEqualTo(1L); + + assertThat(message.getField("double_1")).isEqualTo(1d); + assertThat(message.getField("double_2")).isEqualTo(2d); + assertThat(message.getField("double_3")).isEqualTo(0d); + assertThat(message.getField("double_4")).isEqualTo(1d); + + assertThat(message.getField("bool_1")).isEqualTo(true); + assertThat(message.getField("bool_2")).isEqualTo(false); + assertThat(message.getField("bool_3")).isEqualTo(false); + assertThat(message.getField("bool_4")).isEqualTo(true); + + // the is wrapped in our own class for safey in rules + assertThat(message.getField("ip_1")).isEqualTo(new IpAddress(InetAddresses.forString("127.0.0.1"))); + assertThat(message.getField("ip_2")).isEqualTo(new IpAddress(InetAddresses.forString("127.0.0.1"))); + assertThat(message.getField("ip_3")).isEqualTo(new IpAddress(InetAddresses.forString("0.0.0.0"))); + assertThat(message.getField("ip_4")).isEqualTo(new IpAddress(InetAddresses.forString("::1"))); + + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt new file mode 100644 index 000000000000..b0fa7f775f8b --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt @@ -0,0 +1,30 @@ +rule "conversions" +when true +then + set_fields({ + string_1: to_string("1"), // "1" + string_2: to_string("2", "default"), // "2" + string_3: to_string($message.not_there), // "" -> not being set in message! + string_4: to_string($message.not_there, "default"), // "default" + + long_1: to_long(1), // 1L + long_2: to_long(2, 1), // 2L + long_3: to_long($message.not_there), // 0L + long_4: to_long($message.not_there, 1), // 1L + + double_1: to_double(1d), // 1d + double_2: to_double(2d, 1d), // 2d + double_3: to_double($message.not_there), // 0d + double_4: to_double($message.not_there, 1d), // 1d + + bool_1: to_bool("true"), // true + bool_2: to_bool("false", true), // false + bool_3: to_bool($message.not_there), // false + bool_4: to_bool($message.not_there, true), // true + + ip_1: to_ip("127.0.0.1"), // 127.0.0.1 + ip_2: to_ip("127.0.0.1", "2001:db8::1"), // 127.0.0.1 + ip_3: to_ip($message.not_there), // 0.0.0.0 + ip_4: to_ip($message.not_there, "::1") // ::1 (v6) + }); +end \ No newline at end of file From 194f0dd31c4475f99657c7aaf567ce5a2b28fa1f Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 21:58:36 +0200 Subject: [PATCH 300/521] Add type hint for Travis CI's Java compiler to be happy --- .../plugins/pipelineprocessor/ast/functions/FunctionArgs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java index b157d7cca9f3..88a569f5569f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -40,7 +40,7 @@ public class FunctionArgs { public FunctionArgs(Function func, Map args) { function = func; descriptor = function.descriptor(); - this.args = firstNonNull(args, Collections.emptyMap()); + this.args = firstNonNull(args, Collections.emptyMap()); } @Nonnull From a2d6b8ea0a1ff70748f2494a765782320da2fb0d Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 22:14:13 +0200 Subject: [PATCH 301/521] Update Maven plugins --- pom.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 5071d6429fc4..309788524d71 100644 --- a/pom.xml +++ b/pom.xml @@ -205,7 +205,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.3 + 3.5.1 com.google.auto.value.processor.AutoValueProcessor @@ -243,12 +243,12 @@ org.codehaus.mojo findbugs-maven-plugin - 3.0.2 + 3.0.4 org.apache.maven.plugins maven-pmd-plugin - 3.5 + 3.6 org.codehaus.mojo @@ -271,7 +271,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.4.1 + 2.4.3 false false @@ -293,7 +293,7 @@ org.apache.maven.plugins maven-release-plugin - 2.5.2 + 2.5.3 true forked-path @@ -326,7 +326,7 @@ org.codehaus.mojo rpm-maven-plugin - 2.1.4 + 2.1.5 Application/Internet @@ -444,14 +444,14 @@ org.codehaus.plexus plexus-compiler-javac-errorprone - 2.5 + 2.8 com.google.errorprone error_prone_core - 2.0.4 + 2.0.11 From 7a27aaac440b1cab4902c731a2ee508ec9a74b04 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 22:16:28 +0200 Subject: [PATCH 302/521] Sync dependencies with Graylog 2.1.0-beta.2-SNAPSHOT --- pom.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 309788524d71..6e5b4b76bdc0 100644 --- a/pom.xml +++ b/pom.xml @@ -40,8 +40,8 @@ 2.1.0-beta.2-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 - 1.7.13 - 2.5 + 1.7.21 + 2.6.2 @@ -54,7 +54,7 @@ com.google.auto.value auto-value - 1.1 + 1.2 junit @@ -64,13 +64,13 @@ org.mockito mockito-core - 2.0.40-beta + 2.0.95-beta test org.assertj assertj-core - 3.3.0 + 3.5.2 test @@ -183,7 +183,6 @@ org.assertj assertj-core - 3.3.0 test From 83c2316c48822da18cb4e130090d22826aa21548 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 1 Aug 2016 22:36:00 +0200 Subject: [PATCH 303/521] Disable failing tests in FunctionsSnippetsTest Refs #64 --- .../pipelineprocessor/functions/FunctionsSnippetsTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 31ad2d75e0ec..fe662a5d7812 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -86,6 +86,7 @@ import org.joda.time.DateTimeUtils; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import java.util.Map; @@ -328,6 +329,7 @@ public void ipMatching() { } @Test + @Ignore("Failing after merging https://github.com/Graylog2/graylog-plugin-pipeline-processor/pull/64") public void evalError() { final Rule rule = parser.parseRule(ruleForTest(), false); @@ -339,6 +341,7 @@ public void evalError() { } @Test + @Ignore("Failing after merging https://github.com/Graylog2/graylog-plugin-pipeline-processor/pull/64") public void evalErrorSuppressed() { final Rule rule = parser.parseRule(ruleForTest(), false); From 1e0f65f3fde804919e0fe247345d060983a901ed Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 2 Aug 2016 10:50:43 +0200 Subject: [PATCH 304/521] Fix FunctionsSnippetsTest after conversion function unification (#70) `evalError()` can no longer trigger the error tested for and was removed. `evalErrorSuppressed()` now tests an illegal default value in `to_ip()`. Fixes #64 --- .../functions/FunctionsSnippetsTest.java | 14 -------------- .../pipelineprocessor/functions/evalError.txt | 6 ------ .../functions/evalErrorSuppressed.txt | 2 +- 3 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index fe662a5d7812..96a83e5e1d7a 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -86,7 +86,6 @@ import org.joda.time.DateTimeUtils; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import java.util.Map; @@ -329,19 +328,6 @@ public void ipMatching() { } @Test - @Ignore("Failing after merging https://github.com/Graylog2/graylog-plugin-pipeline-processor/pull/64") - public void evalError() { - final Rule rule = parser.parseRule(ruleForTest(), false); - - final EvaluationContext context = contextForRuleEval(rule, new Message("test", "test", Tools.nowUTC())); - - assertThat(context).isNotNull(); - assertThat(context.hasEvaluationErrors()).isTrue(); - assertThat(Iterables.getLast(context.evaluationErrors()).toString()).isEqualTo("In call to function 'regex' at 5:28 an exception was thrown: Argument 'value' cannot be 'null'"); - } - - @Test - @Ignore("Failing after merging https://github.com/Graylog2/graylog-plugin-pipeline-processor/pull/64") public void evalErrorSuppressed() { final Rule rule = parser.parseRule(ruleForTest(), false); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt deleted file mode 100644 index c2fd322ff66a..000000000000 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalError.txt +++ /dev/null @@ -1,6 +0,0 @@ -rule "trigger null" -when - true -then - set_field("do_not_exist", regex(".*", to_string($message.do_not_exist)).matches); -end diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt index 36e1873f912e..ea0989acdecd 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt @@ -1,6 +1,6 @@ rule "suppressing exceptions/nulls" when - is_null(to_ip($message.does_not_exist)) && is_not_null($message.this_field_was_set) + is_null(to_ip($message.does_not_exist, "d.f.f.f")) && is_not_null($message.this_field_was_set) then trigger_test(); end From 8956d6d04a8c8ca9b4362bd0512d81bebb199f42 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Tue, 2 Aug 2016 17:06:55 +0200 Subject: [PATCH 305/521] Remove wildcard type cast to make IDEA 2016.2 happy (#71) This change should not affect `javac` at all, but intellij flags the collect call with having two errors. --- .../pipelineprocessor/functions/json/SelectJsonPath.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java index 28dcc1b72d54..4aca6961ab36 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -54,8 +54,8 @@ public SelectJsonPath(ObjectMapper objectMapper) { // sigh generics and type erasure //noinspection unchecked pathsParam = ParameterDescriptor.type("paths", - (Class>) new TypeLiteral>() {}.getRawType(), - (Class>) new TypeLiteral>() {}.getRawType()) + (Class>) new TypeLiteral>() {}.getRawType(), + (Class>) new TypeLiteral>() {}.getRawType()) .transform(inputMap -> inputMap .entrySet().stream() .collect(toMap(Map.Entry::getKey, e -> JsonPath.compile(e.getValue())))) From d7e11b7c2a1025ad8fc8449d81f3a46cc711a4c2 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 2 Aug 2016 17:28:19 +0200 Subject: [PATCH 306/521] Bumping versions to 2.1.0-beta.2 / 1.1.0-beta.2 --- package.json | 2 +- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index d7fd5610639d..6db7712f2716 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PipelineProcessor", - "version": "1.1.0-beta.1", + "version": "1.1.0-beta.2", "description": "", "repository": { "type": "git", diff --git a/pom.xml b/pom.xml index 6e5b4b76bdc0..4bef6f86d0ad 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.2-SNAPSHOT + 2.1.0-beta.2 /usr/share/graylog-server/plugin 4.5.1 1.7.21 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index cb3d550d911d..bf4e28d76186 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 1, 0, "beta.1"); + return new Version(1, 1, 0, "beta.2"); } @Override @@ -60,7 +60,7 @@ public String getDescription() { @Override public Version getRequiredVersion() { - return new Version(2, 1, 0, "beta.1"); + return new Version(2, 1, 0); } @Override From 6a921eed29e1f172eb0684a3509a0c6f2d961b88 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Tue, 2 Aug 2016 15:41:07 +0000 Subject: [PATCH 307/521] [graylog-plugin-pipeline-processor-release] prepare release 1.1.0-beta.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4bef6f86d0ad..d73697d29c64 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.2-SNAPSHOT + 1.1.0-beta.2 jar ${project.artifactId} From abfa355a646a74a35fc1cdc1704a21bb2d661d1d Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Tue, 2 Aug 2016 15:41:20 +0000 Subject: [PATCH 308/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d73697d29c64..ee2a94838834 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.2 + 1.1.0-beta.3-SNAPSHOT jar ${project.artifactId} From f77b9e3d4c2fc2e77be3024c4d4c3fd82193e514 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Wed, 3 Aug 2016 10:22:47 +0200 Subject: [PATCH 309/521] Bumping graylog dependency version to next development iteration. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ee2a94838834..d99f8fa5abf6 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.2 + 2.1.0-beta.3-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.21 From c7ca2f7a1ab63b6b5f91922c00688da44ab35cf0 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Wed, 3 Aug 2016 10:50:49 +0200 Subject: [PATCH 310/521] Remove upper case decorator (#73) There is no real use case for this besides being a good test case for the decorator system development. Fixes Graylog2/graylog2-server#2588 --- .../PipelineProcessorModule.java | 3 - .../pipelineprocessor/UpperCaseDecorator.java | 128 ------------------ 2 files changed, 131 deletions(-) delete mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java index 2c915d3d34a6..e93bae59a2d4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -50,8 +50,5 @@ protected void configure() { installSearchResponseDecorator(searchResponseDecoratorBinder(), PipelineProcessorMessageDecorator.class, PipelineProcessorMessageDecorator.Factory.class); - installSearchResponseDecorator(searchResponseDecoratorBinder(), - UpperCaseDecorator.class, - UpperCaseDecorator.Factory.class); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java deleted file mode 100644 index e11a1246d28c..000000000000 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/UpperCaseDecorator.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * This file is part of Graylog Pipeline Processor. - * - * Graylog Pipeline Processor is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Graylog Pipeline Processor is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Graylog Pipeline Processor. If not, see . - */ -package org.graylog.plugins.pipelineprocessor; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMultimap; -import com.google.inject.assistedinject.Assisted; -import org.graylog.plugins.pipelineprocessor.ast.Pipeline; -import org.graylog.plugins.pipelineprocessor.ast.Rule; -import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; -import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; -import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; -import org.graylog2.decorators.Decorator; -import org.graylog2.plugin.Message; -import org.graylog2.plugin.configuration.ConfigurationRequest; -import org.graylog2.plugin.configuration.fields.TextField; -import org.graylog2.plugin.decorators.SearchResponseDecorator; -import org.graylog2.rest.models.messages.responses.ResultMessageSummary; -import org.graylog2.rest.resources.search.responses.SearchResponse; - -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -public class UpperCaseDecorator implements SearchResponseDecorator { - private static final String CK_FIELD_NAME = "fieldName"; - private static final String CK_PIPELINE_DEFINITION = "pipeline \"Uppercase decorator\"\nstage 0 match either\nrule \"Uppercase field\"\nend"; - - private final List pipelines; - private final PipelineInterpreter pipelineInterpreter; - private final Decorator decorator; - - public interface Factory extends SearchResponseDecorator.Factory { - @Override - UpperCaseDecorator create(Decorator decorator); - - @Override - Config getConfig(); - - @Override - Descriptor getDescriptor(); - } - - public static class Config implements SearchResponseDecorator.Config { - @Inject - public Config() { - } - - @Override - public ConfigurationRequest getRequestedConfiguration() { - return new ConfigurationRequest() {{ - addField(new TextField(CK_FIELD_NAME, "Field Name", "", "The Name of the field which should be uppercased")); - }}; - }; - } - - public static class Descriptor extends SearchResponseDecorator.Descriptor { - public Descriptor() { - super("Uppercase Decorator", "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Uppercase Decorator"); - } - } - - @Inject - public UpperCaseDecorator(PipelineInterpreter pipelineInterpreter, - PipelineRuleParser pipelineRuleParser, - @Assisted Decorator decorator) { - this.pipelineInterpreter = pipelineInterpreter; - this.decorator = decorator; - final String fieldName = (String)decorator.config().get(CK_FIELD_NAME); - - this.pipelines = pipelineRuleParser.parsePipelines(CK_PIPELINE_DEFINITION); - final List rules = ImmutableList.of(pipelineRuleParser.parseRule(getRuleForField(fieldName), true)); - this.pipelines.forEach(pipeline -> { - pipeline.stages().forEach(stage -> stage.setRules(rules)); - }); - } - - @Override - public SearchResponse apply(SearchResponse searchResponse) { - final List results = new ArrayList<>(); - searchResponse.messages().forEach((inMessage) -> { - final Map originalMessage = ImmutableMap.copyOf(inMessage.message()); - final Message message = new Message(inMessage.message()); - final List additionalCreatedMessages = pipelineInterpreter.processForResolvedPipelines(message, - message.getId(), - new HashSet<>(this.pipelines), - new NoopInterpreterListener()); - - results.add(ResultMessageSummary.create(inMessage.highlightRanges(), message.getFields(), inMessage.index())); - additionalCreatedMessages.forEach((additionalMessage) -> { - // TODO: pass proper highlight ranges. Need to rebuild them for new messages. - results.add(ResultMessageSummary.create( - ImmutableMultimap.of(), - additionalMessage.getFields(), - "[created from decorator]" - )); - }); - }); - - return searchResponse.toBuilder().messages(results).build(); - } - - private String getRuleForField(String fieldName) { - return "rule \"Uppercase field\"\n" + - "when\n" + - "has_field(\"" + fieldName + "\")\n" + - "then\n" + - "set_field(\"" + fieldName + "\", uppercase(to_string($message." + fieldName + ")));\n" + - "end"; - } -} From 06e64b7223c15302ac5aa35a959ff3df747e923a Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 3 Aug 2016 14:44:36 +0200 Subject: [PATCH 311/521] do not preprocess arguments of the error function fixes #25 --- .../plugins/pipelineprocessor/ast/functions/Function.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index cdd46b30a52a..bd51d06ceb55 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -34,6 +34,11 @@ public Void evaluate(FunctionArgs args, EvaluationContext context) { return null; } + @Override + public void preprocessArgs(FunctionArgs args) { + // intentionally left blank + } + @Override public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() From 6dc3b143eef852b33b5d04f5b81bb09ebbc50989 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 8 Aug 2016 15:11:32 +0200 Subject: [PATCH 312/521] unwrap JsonNode values (#72) e.g. strings would be double quoted without this fixes #68 --- .../functions/json/SelectJsonPath.java | 35 ++++++++++++++++++- .../functions/FunctionsSnippetsTest.java | 1 + 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java index 4aca6961ab36..bf8149def2c8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import com.google.inject.TypeLiteral; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; @@ -30,6 +31,7 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import javax.inject.Inject; +import java.io.IOException; import java.util.Map; import static com.google.common.collect.ImmutableList.of; @@ -73,10 +75,41 @@ public Map evaluate(FunctionArgs args, EvaluationContext context .entrySet().stream() .collect(toMap( Map.Entry::getKey, - e -> e.getValue().read(json, configuration) + e -> unwrapJsonNode(e.getValue().read(json, configuration)) )); } + private Object unwrapJsonNode(Object value) { + if (!(value instanceof JsonNode)) { + return value; + } + JsonNode read = ((JsonNode) value); + switch (read.getNodeType()) { + case ARRAY: + return ImmutableList.copyOf(read.elements()); + case BINARY: + try { + return read.binaryValue(); + } catch (IOException e) { + return null; + } + case BOOLEAN: + return read.booleanValue(); + case MISSING: + case NULL: + return null; + case NUMBER: + return read.numberValue(); + case OBJECT: + return read; + case POJO: + return read; + case STRING: + return read.textValue(); + } + return read; + } + @Override public FunctionDescriptor> descriptor() { //noinspection unchecked diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 96a83e5e1d7a..c85f7575bb52 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -242,6 +242,7 @@ public void jsonpath() { final Message message = evaluateRule(rule, new Message(json, "test", Tools.nowUTC())); assertThat(message.hasField("author_first")).isTrue(); + assertThat(message.getField("author_first")).isEqualTo("Nigel Rees"); assertThat(message.hasField("author_last")).isTrue(); } From c05119134fa29f9963c171975a22c32fe8556032 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 8 Aug 2016 15:27:26 +0200 Subject: [PATCH 313/521] add optional prefix/suffix to set_fields functions (#75) fixes #74 --- .../functions/messages/SetField.java | 25 +++++++++--- .../functions/messages/SetFields.java | 27 +++++++++++-- .../functions/FunctionsSnippetsTest.java | 21 ++++++++++ .../functions/fieldPrefixSuffix.txt | 40 +++++++++++++++++++ .../functions/newlyCreatedMessage.txt | 8 ++-- 5 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index 17063f9c617e..48df2b8d8b96 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -24,6 +24,8 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; import org.graylog2.plugin.Message; +import java.util.Optional; + import static com.google.common.collect.ImmutableList.of; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; @@ -32,26 +34,37 @@ public class SetField extends AbstractFunction { public static final String NAME = "set_field"; - public static final String FIELD = "field"; - public static final String VALUE = "value"; private final ParameterDescriptor fieldParam; private final ParameterDescriptor valueParam; + private final ParameterDescriptor prefixParam; + private final ParameterDescriptor suffixParam; private final ParameterDescriptor messageParam; public SetField() { - fieldParam = string(FIELD).build(); - valueParam = object(VALUE).build(); + fieldParam = string("field").build(); + valueParam = object("value").build(); + prefixParam = string("prefix").optional().build(); + suffixParam = string("suffix").optional().build(); messageParam = type("message", Message.class).optional().build(); } @Override public Void evaluate(FunctionArgs args, EvaluationContext context) { - final String field = fieldParam.required(args, context); + String field = fieldParam.required(args, context); final Object value = valueParam.required(args, context); if (!Strings.isNullOrEmpty(field)) { final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + final Optional prefix = prefixParam.optional(args, context); + final Optional suffix = suffixParam.optional(args, context); + + if (prefix.isPresent()) { + field = prefix.get() + field; + } + if (suffix.isPresent()) { + field = field + suffix.get(); + } message.addField(field, value); } return null; @@ -64,6 +77,8 @@ public FunctionDescriptor descriptor() { .returnType(Void.class) .params(of(fieldParam, valueParam, + prefixParam, + suffixParam, messageParam)) .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java index ecee09e9926c..009da4331c54 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -24,19 +24,25 @@ import org.graylog2.plugin.Message; import java.util.Map; +import java.util.Optional; import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; public class SetFields extends AbstractFunction { public static final String NAME = "set_fields"; - public static final String FIELDS = "fields"; + private final ParameterDescriptor fieldsParam; + private final ParameterDescriptor prefixParam; + private final ParameterDescriptor suffixParam; private final ParameterDescriptor messageParam; public SetFields() { - fieldsParam = ParameterDescriptor.type(FIELDS, Map.class).build(); + fieldsParam = type("fields", Map.class).build(); + prefixParam = string("prefix").optional().build(); + suffixParam = string("suffix").optional().build(); messageParam = type("message", Message.class).optional().build(); } @@ -45,7 +51,20 @@ public Void evaluate(FunctionArgs args, EvaluationContext context) { //noinspection unchecked final Map fields = fieldsParam.required(args, context); final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); - message.addFields(fields); + final Optional prefix = prefixParam.optional(args, context); + final Optional suffix = suffixParam.optional(args, context); + + if (fields != null) { + fields.forEach((field, value) -> { + if (prefix.isPresent()) { + field = prefix.get() + field; + } + if (suffix.isPresent()) { + field = field + suffix.get(); + } + message.addField(field, value); + }); + } return null; } @@ -54,7 +73,7 @@ public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() .name(NAME) .returnType(Void.class) - .params(of(fieldsParam, messageParam)) + .params(of(fieldsParam, prefixParam, suffixParam, messageParam)) .build(); } diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index c85f7575bb52..51cf0ad0387d 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -504,4 +504,25 @@ public void conversions() { assertThat(message.getField("ip_4")).isEqualTo(new IpAddress(InetAddresses.forString("::1"))); } + + @Test + public void fieldPrefixSuffix() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final Message message = evaluateRule(rule); + + assertThat(message).isNotNull(); + + assertThat(message.getField("field")).isEqualTo("1"); + assertThat(message.getField("prae_field_sueff")).isEqualTo("2"); + assertThat(message.getField("field_sueff")).isEqualTo("3"); + assertThat(message.getField("prae_field")).isEqualTo("4"); + assertThat(message.getField("pre_field1_suff")).isEqualTo("5"); + assertThat(message.getField("pre_field2_suff")).isEqualTo("6"); + assertThat(message.getField("pre_field1")).isEqualTo("7"); + assertThat(message.getField("pre_field2")).isEqualTo("8"); + assertThat(message.getField("field1_suff")).isEqualTo("9"); + assertThat(message.getField("field2_suff")).isEqualTo("10"); + + } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt new file mode 100644 index 000000000000..8e94a920e621 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt @@ -0,0 +1,40 @@ +rule "prefixsuffix" +when true +then + // plain set field + set_field("field", "1"); + // both prefix and suffix, doesn't touch the above + set_field("field", "2", "prae_", "_sueff"); + + // combinations of optional prefix, suffix + set_field(field: "field", value: "3", suffix: "_sueff"); + set_field(field: "field", value: "4", prefix: "prae_"); + + // set multiple fields with the same prefix + set_fields( + fields: { + field1: "5", + field2: "6" + }, + prefix: "pre_", + suffix: "_suff" + ); + + // set multiple fields with the same prefix, suffix optional + set_fields( + fields: { + field1: "7", + field2: "8" + }, + prefix: "pre_" + ); + // set multiple fields with the same suffix, prefix optional + set_fields( + fields: { + field1: "9", + field2: "10" + }, + suffix: "_suff" + ); +end + diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt index d4c63c6fb095..f4f32ae20c25 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt @@ -3,10 +3,10 @@ when true then let x = create_message("new", "synthetic", now()); - set_field("removed_again", "foo", x); - set_field("only_in", "new message", x); - set_fields({ multi: "new message" }, x); - set_field("has_source", has_field("source", x), x); + set_field(field: "removed_again", value: "foo", message: x); + set_field(field: "only_in", value: "new message", message: x); + set_fields(fields: { multi: "new message" }, message: x); + set_field(field: "has_source", value: has_field("source", x), message: x); route_to_stream(name: "some stream", message: x); remove_field("removed_again", x); end \ No newline at end of file From 812b245a4c0514a423587cdebbdb34aa70c2acb2 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Mon, 8 Aug 2016 16:53:48 +0200 Subject: [PATCH 314/521] Add key-value parsing function (#77) Fixes #38 --- .../ast/functions/Function.java | 3 + .../functions/ProcessorFunctionsModule.java | 2 + .../functions/strings/KeyValue.java | 184 ++++++++++++++++++ .../functions/FunctionsSnippetsTest.java | 43 +++- .../pipelineprocessor/functions/keyValue.txt | 23 +++ .../functions/keyValueFailure.txt | 12 ++ 6 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java index bd51d06ceb55..bd455c5901e8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -57,6 +57,9 @@ default void preprocessArgs(FunctionArgs args) { if (value != null) { //noinspection unchecked final ParameterDescriptor param = (ParameterDescriptor) args.param(name); + if (param == null) { + throw new IllegalStateException("Unknown parameter " + name + "! Cannot continue."); + } args.setPreComputedValue(name, param.transform().apply(value)); } } catch (Exception exception) { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java index 9c426601e87b..0fd5b2df0572 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -53,6 +53,7 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Concat; import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.KeyValue; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; @@ -103,6 +104,7 @@ protected void configure() { addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); addMessageProcessorFunction(Concat.NAME, Concat.class); + addMessageProcessorFunction(KeyValue.NAME, KeyValue.class); // json addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java new file mode 100644 index 000000000000..679d811d4fd2 --- /dev/null +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java @@ -0,0 +1,184 @@ +/** + * This file is part of Graylog Pipeline Processor. + * + * Graylog Pipeline Processor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog Pipeline Processor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog Pipeline Processor. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.inject.TypeLiteral; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.bool; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class KeyValue extends AbstractFunction> { + + public static final String NAME = "key_value"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor splitParam; + private final ParameterDescriptor valueSplitParam; + private final ParameterDescriptor ignoreEmptyValuesParam; + private final ParameterDescriptor allowDupeKeysParam; + private final ParameterDescriptor duplicateHandlingParam; + private final ParameterDescriptor trimCharactersParam; + private final ParameterDescriptor trimValueCharactersParam; + + public KeyValue() { + valueParam = string("value").build(); + splitParam = string("delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().build(); + valueSplitParam = string("kv_delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().build(); + + ignoreEmptyValuesParam = bool("ignore_empty_values").optional().build(); + allowDupeKeysParam = bool("allow_dup_keys").optional().build(); + duplicateHandlingParam = string("handle_dup_keys").optional().build(); + trimCharactersParam = string("trim_key_chars", CharMatcher.class) + .transform(CharMatcher::anyOf) + .optional() + .build(); + trimValueCharactersParam = string("trim_value_chars", CharMatcher.class) + .transform(CharMatcher::anyOf) + .optional() + .build(); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + if (Strings.isNullOrEmpty(value)) { + return null; + } + final CharMatcher kvPairsMatcher = splitParam.optional(args, context).orElse(CharMatcher.whitespace()); + final CharMatcher kvDelimMatcher = valueSplitParam.optional(args, context).orElse(CharMatcher.anyOf("=")); + + Splitter outerSplitter = Splitter.on(kvPairsMatcher) + .omitEmptyStrings() + .trimResults(); + + final Splitter entrySplitter = Splitter.on(kvDelimMatcher) + .omitEmptyStrings() + .trimResults(); + return new MapSplitter(outerSplitter, + entrySplitter, + ignoreEmptyValuesParam.optional(args, context).orElse(true), + trimCharactersParam.optional(args, context).orElse(CharMatcher.none()), + trimValueCharactersParam.optional(args, context).orElse(CharMatcher.none()), + allowDupeKeysParam.optional(args, context).orElse(true), + duplicateHandlingParam.optional(args, context).orElse("take_first")) + .split(value); + } + + @Override + public FunctionDescriptor> descriptor() { + //noinspection unchecked + return FunctionDescriptor.>builder() + .name(NAME) + .returnType((Class>) new TypeLiteral>() {}.getRawType()) + .params(valueParam, + splitParam, + valueSplitParam, + ignoreEmptyValuesParam, + allowDupeKeysParam, + duplicateHandlingParam, + trimCharactersParam, + trimValueCharactersParam + ) + .build(); + } + + + private static class MapSplitter { + + private final Splitter outerSplitter; + private final Splitter entrySplitter; + private final boolean ignoreEmptyValues; + private final CharMatcher keyTrimMatcher; + private final CharMatcher valueTrimMatcher; + private final Boolean allowDupeKeys; + private final String duplicateHandling; + + MapSplitter(Splitter outerSplitter, + Splitter entrySplitter, + boolean ignoreEmptyValues, + CharMatcher keyTrimMatcher, + CharMatcher valueTrimMatcher, + Boolean allowDupeKeys, + String duplicateHandling) { + this.outerSplitter = outerSplitter; + this.entrySplitter = entrySplitter; + this.ignoreEmptyValues = ignoreEmptyValues; + this.keyTrimMatcher = keyTrimMatcher; + this.valueTrimMatcher = valueTrimMatcher; + this.allowDupeKeys = allowDupeKeys; + this.duplicateHandling = duplicateHandling; + } + + + public Map split(CharSequence sequence) { + final Map map = new LinkedHashMap<>(); + + for (String entry : outerSplitter.split(sequence)) { + boolean concat = false; + Iterator entryFields = entrySplitter.split(entry).iterator(); + + if (!entryFields.hasNext()) { + continue; + } + String key = entryFields.next(); + key = keyTrimMatcher.trimFrom(key); + if (map.containsKey(key)) { + if (!allowDupeKeys) { + throw new IllegalArgumentException("Duplicate key " + key + " is not allowed in key_value function."); + } + switch (Strings.nullToEmpty(duplicateHandling).toLowerCase(Locale.ENGLISH)) { + case "take_first": + // ignore this value + continue; + case "take_last": + // simply reset the entry + break; + default: + concat = true; + } + } + + if (entryFields.hasNext()) { + String value = entryFields.next(); + value = valueTrimMatcher.trimFrom(value); + // already have a value, concating old+delim+new + if (concat) { + value = map.get(key) + duplicateHandling + value; + } + map.put(key, value); + } else if (!ignoreEmptyValues) { + throw new IllegalArgumentException("Missing value for key " + key); + } + + } + return Collections.unmodifiableMap(map); + } + } +} diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index 51cf0ad0387d..ab29816ea2d4 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -60,6 +60,7 @@ import org.graylog.plugins.pipelineprocessor.functions.strings.Concat; import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.KeyValue; import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; @@ -149,6 +150,7 @@ public static void registerFunctions() { functions.put(Swapcase.NAME, new Swapcase()); functions.put(Uncapitalize.NAME, new Uncapitalize()); functions.put(Uppercase.NAME, new Uppercase()); + functions.put(KeyValue.NAME, new KeyValue()); final ObjectMapper objectMapper = new ObjectMapperProvider().get(); functions.put(JsonParse.NAME, new JsonParse(objectMapper)); @@ -310,9 +312,9 @@ public void strings() { assertThat(actionsTriggered.get()).isTrue(); assertThat(message).isNotNull(); assertThat(message.getField("has_xyz")).isInstanceOf(Boolean.class); - assertThat((boolean)message.getField("has_xyz")).isFalse(); + assertThat((boolean) message.getField("has_xyz")).isFalse(); assertThat(message.getField("string_literal")).isInstanceOf(String.class); - assertThat((String)message.getField("string_literal")).isEqualTo("abcd\\.e\tfg\u03a9\363"); + assertThat((String) message.getField("string_literal")).isEqualTo("abcd\\.e\tfg\u03a9\363"); } @Test @@ -381,9 +383,11 @@ public void urls() { assertThat(message.getField("user_info")).isEqualTo("admin:s3cr31"); assertThat(message.getField("host")).isEqualTo("some.host.with.lots.of.subdomains.com"); assertThat(message.getField("port")).isEqualTo(9999); - assertThat(message.getField("file")).isEqualTo("/path1/path2/three?q1=something&with_spaces=hello%20graylog&equal=can=containanotherone"); + assertThat(message.getField("file")).isEqualTo( + "/path1/path2/three?q1=something&with_spaces=hello%20graylog&equal=can=containanotherone"); assertThat(message.getField("fragment")).isEqualTo("anchorstuff"); - assertThat(message.getField("query")).isEqualTo("q1=something&with_spaces=hello%20graylog&equal=can=containanotherone"); + assertThat(message.getField("query")).isEqualTo( + "q1=something&with_spaces=hello%20graylog&equal=can=containanotherone"); assertThat(message.getField("q1")).isEqualTo("something"); assertThat(message.getField("with_spaces")).isEqualTo("hello graylog"); assertThat(message.getField("equal")).isEqualTo("can=containanotherone"); @@ -523,6 +527,37 @@ public void fieldPrefixSuffix() { assertThat(message.getField("pre_field2")).isEqualTo("8"); assertThat(message.getField("field1_suff")).isEqualTo("9"); assertThat(message.getField("field2_suff")).isEqualTo("10"); + } + + public void keyValue() { + final Rule rule = parser.parseRule(ruleForTest(), true); + + final EvaluationContext context = contextForRuleEval(rule, new Message("", "", Tools.nowUTC())); + + assertThat(context).isNotNull(); + assertThat(context.evaluationErrors()).isEmpty(); + final Message message = context.currentMessage(); + assertThat(message).isNotNull(); + + + assertThat(message.getField("a")).isEqualTo("1,4"); + assertThat(message.getField("b")).isEqualTo("2"); + assertThat(message.getField("c")).isEqualTo("3"); + assertThat(message.getField("d")).isEqualTo("44"); + assertThat(message.getField("e")).isEqualTo("4"); + assertThat(message.getField("f")).isEqualTo("1"); + assertThat(message.getField("g")).isEqualTo("3"); + assertThat(message.hasField("h")).isFalse(); + + assertThat(message.getField("dup_first")).isEqualTo("1"); + assertThat(message.getField("dup_last")).isEqualTo("2"); + } + + @Test + public void keyValueFailure() { + final Rule rule = parser.parseRule(ruleForTest(), true); + final EvaluationContext context = contextForRuleEval(rule, new Message("", "", Tools.nowUTC())); + assertThat(context.hasEvaluationErrors()).isTrue(); } } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt new file mode 100644 index 000000000000..95f7f983ce1c --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt @@ -0,0 +1,23 @@ +rule "kv" +when true +then + set_fields(key_value( + value: "a='1' =2 \n 'c'=3 [d]=44 a=4 \"e\"=4 [f=1][[g]:3] h=", + delimiters: " \t\n\r[", + kv_delimiters: "=:", + ignore_empty_values: true, + trim_key_chars: "\"[]<>'", + trim_value_chars: "']", + allow_dup_keys: true, // the default + handle_dup_keys: "," // meaning concat, default "take_first" + )); + + set_fields(key_value( + value: "dup_first=1 dup_first=2", + handle_dup_keys: "take_first" + )); + set_fields(key_value( + value: "dup_last=1 dup_last=2", + handle_dup_keys: "take_last" + )); +end \ No newline at end of file diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt new file mode 100644 index 000000000000..0e1f18107de3 --- /dev/null +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt @@ -0,0 +1,12 @@ +rule "kv" +when true +then + set_fields(key_value( + value: "dup_first=1 dup_first=2", + allow_dup_keys: false + )); + set_fields(key_value( + value: "dup_last=", + ignore_empty_values: false + )); +end \ No newline at end of file From 567256a7a6ecd207c55baa01b4982498d7af4ccf Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Wed, 10 Aug 2016 16:10:27 +0200 Subject: [PATCH 315/521] Allow selection of an input ID for the simulation message (#78) * Invert equals() call to avoid possible null pointer exception Fixes Graylog2/graylog2-server#2610 * Allow selection of an input ID for the simulation message Pipeline rules may use the `from_input` function in a condition so this is needed to make the simulation work. Refs Graylog2/graylog2-server#2610 --- .../pipelineprocessor/functions/FromInput.java | 2 +- .../pipelineprocessor/rest/SimulationRequest.java | 11 ++++++++++- .../pipelineprocessor/rest/SimulatorResource.java | 4 ++++ src/web/simulator/ProcessorSimulator.jsx | 6 +++--- src/web/simulator/SimulatorStore.js | 3 ++- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java index 85ab0a274a16..2d8558de8c3d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -72,7 +72,7 @@ public Boolean evaluate(FunctionArgs args, EvaluationContext context) { } return input != null - && context.currentMessage().getSourceInputId().equals(input.getId()); + && input.getId().equals(context.currentMessage().getSourceInputId()); } @Override diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java index 97755331be2a..457276f23d5a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; import java.util.Map; @AutoValue @@ -32,16 +33,22 @@ public abstract class SimulationRequest { @JsonProperty public abstract Map message(); + @JsonProperty + @Nullable + public abstract String inputId(); + public static Builder builder() { return new AutoValue_SimulationRequest.Builder(); } @JsonCreator public static SimulationRequest create (@JsonProperty("stream_id") String streamId, - @JsonProperty("message") Map message) { + @JsonProperty("message") Map message, + @JsonProperty("input_id") @Nullable String inputId) { return builder() .streamId(streamId) .message(message) + .inputId(inputId) .build(); } @@ -52,5 +59,7 @@ public abstract static class Builder { public abstract Builder streamId(String streamId); public abstract Builder message(Map message); + + public abstract Builder inputId(String inputId); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java index 1f0c30d2af26..a0424ef7b18b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiParam; import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.elasticsearch.common.Strings; import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTracer; import org.graylog2.database.NotFoundException; @@ -69,6 +70,9 @@ public SimulationResponse simulate(@ApiParam(name = "simulation", required = tru final Stream stream = streamService.load(request.streamId()); message.addStream(stream); } + if (!Strings.isNullOrEmpty(request.inputId())) { + message.setSourceInputId(request.inputId()); + } final List simulationResults = new ArrayList<>(); final PipelineInterpreterTracer pipelineInterpreterTracer = new PipelineInterpreterTracer(); diff --git a/src/web/simulator/ProcessorSimulator.jsx b/src/web/simulator/ProcessorSimulator.jsx index f063ed7caa4e..ba9e1e7dd0d0 100644 --- a/src/web/simulator/ProcessorSimulator.jsx +++ b/src/web/simulator/ProcessorSimulator.jsx @@ -22,11 +22,11 @@ const ProcessorSimulator = React.createClass({ }; }, - _onMessageLoad(message) { + _onMessageLoad(message, options) { this.setState({ message: message, simulation: undefined, loading: true, error: undefined }); SimulatorActions.simulate - .triggerPromise(this.props.stream, message.fields) + .triggerPromise(this.props.stream, message.fields, options.inputId) .then( response => { this.setState({ simulation: response, loading: false }); @@ -47,7 +47,7 @@ const ProcessorSimulator = React.createClass({ Build an example message that will be used in the simulation.{' '} No real messages stored in Graylog will be changed. All actions are purely simulated on the temporary input you provide below.

    - + Date: Thu, 11 Aug 2016 16:20:37 +0200 Subject: [PATCH 316/521] Bumping versions to 2.1.0-beta.3 / 1.1.0-beta.3 --- package.json | 2 +- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 6db7712f2716..924afce1b3a4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PipelineProcessor", - "version": "1.1.0-beta.2", + "version": "1.1.0-beta.3", "description": "", "repository": { "type": "git", diff --git a/pom.xml b/pom.xml index d99f8fa5abf6..d1b51c8713d1 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.3-SNAPSHOT + 2.1.0-beta.3 /usr/share/graylog-server/plugin 4.5.1 1.7.21 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index bf4e28d76186..a3e0fb9dc966 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 1, 0, "beta.2"); + return new Version(1, 1, 0, "beta.3"); } @Override From 8507b2ccb3ac1c3ae62e3093987b428492a0e994 Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 11 Aug 2016 14:33:40 +0000 Subject: [PATCH 317/521] [graylog-plugin-pipeline-processor-release] prepare release 1.1.0-beta.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d1b51c8713d1..4ed0481bb75b 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.3-SNAPSHOT + 1.1.0-beta.3 jar ${project.artifactId} From cf0b2766d298f0e50b2d6a0cdd5af3a7a1a045cd Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 11 Aug 2016 14:33:51 +0000 Subject: [PATCH 318/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4ed0481bb75b..09438c1b5028 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.3 + 1.1.0-beta.4-SNAPSHOT jar ${project.artifactId} From d5cdf0a6e4eb12f5f738d9c0eea85e47f7cb728a Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Thu, 11 Aug 2016 18:01:53 +0200 Subject: [PATCH 319/521] Bump server dependency to 2.1.0-beta.4-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 09438c1b5028..8666b04fa498 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.3 + 2.1.0-beta.4-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.21 From 944c7b93e558d6ec19e358b5830a3a441c4e6e60 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 16 Aug 2016 12:35:14 +0200 Subject: [PATCH 320/521] Unregister PipelineInterpreter from event bus (#79) Message decorators and the pipeline simulator create new instances of PipelineInterpreters that never get garbage collected, as they are still registered in the event bus. These changes add a simple workaround for that. We should probably refactor the lifecycle of the PipelineInterpreter, but this is probably not the best time to do it. --- .../PipelineProcessorMessageDecorator.java | 3 ++- .../processors/PipelineInterpreter.java | 10 ++++++++++ .../pipelineprocessor/rest/SimulatorResource.java | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java index 04a84cf5bc0f..3e7642b5d699 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java @@ -17,7 +17,6 @@ package org.graylog.plugins.pipelineprocessor; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.inject.assistedinject.Assisted; @@ -123,6 +122,8 @@ public SearchResponse apply(SearchResponse searchResponse) { }); }); + pipelineInterpreter.stop(); + return searchResponse.toBuilder().messages(results).build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java index 38f1fa1c4c03..45d6260acbc5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -85,6 +85,7 @@ public class PipelineInterpreter implements MessageProcessor { private final MetricRegistry metricRegistry; private final ScheduledExecutorService scheduler; private final Meter filteredOutMessages; + private EventBus serverEventBus; private final AtomicReference> currentPipelines = new AtomicReference<>(ImmutableMap.of()); private final AtomicReference> streamPipelineConnections = new AtomicReference<>(ImmutableSetMultimap.of()); @@ -107,6 +108,7 @@ public PipelineInterpreter(RuleService ruleService, this.metricRegistry = metricRegistry; this.scheduler = scheduler; this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); + this.serverEventBus = serverEventBus; // listens to cluster wide Rule, Pipeline and pipeline stream connection changes serverEventBus.register(this); @@ -114,6 +116,14 @@ public PipelineInterpreter(RuleService ruleService, reload(); } + /* + * Allow to unregister PipelineInterpreter from the event bus, allowing the object to be garbage collected. + * This is needed in some classes, when new PipelineInterpreter instances are created per request. + */ + public void stop() { + serverEventBus.unregister(this); + } + // this should not run in parallel private synchronized void reload() { // read all rules and compile them diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java index a0424ef7b18b..36cae1aab36a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -82,6 +82,8 @@ public SimulationResponse simulate(@ApiParam(name = "simulation", required = tru for (Message processedMessage : processedMessages) { simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); } + + pipelineInterpreter.stop(); return SimulationResponse.create(simulationResults, pipelineInterpreterTracer.getExecutionTrace(), pipelineInterpreterTracer.took()); From 0eba33c41dd45608b9c220812b678f5f562d0a36 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Tue, 16 Aug 2016 12:53:25 +0200 Subject: [PATCH 321/521] Pipeline UI improvements (#83) * Make changes summary the default view on simulator * Improve message when there are no pipeline connections Include links to rules and pipelines if there are no pipelines available. * Display pipelines using a certain rule On the edit rule page, show pipelines that are using that rule, including a link to them. * Fix add new pipeline button position * Improve navigation options Change some of the options in the top right navigation to adapt better to the workflow. * Disable message actions in simulator This is a left over of when we used real messages to test the pipelines. Fixes Graylog2/graylog2-server#2683 --- .../PipelineConnections.jsx | 26 +++++++++- src/web/pipelines/PipelineDetailsPage.jsx | 4 +- .../pipelines/ProcessingTimelineComponent.jsx | 2 +- src/web/rules/Rule.jsx | 9 ++-- src/web/rules/RuleDetailsPage.jsx | 18 +++++-- src/web/rules/RuleForm.css | 17 +++++++ src/web/rules/RuleForm.jsx | 49 ++++++++++++++++--- src/web/simulator/SimulationResults.jsx | 5 +- src/web/simulator/SimulatorPage.jsx | 8 +-- 9 files changed, 112 insertions(+), 26 deletions(-) create mode 100644 src/web/rules/RuleForm.css diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx index 88d9b6910a05..99290cbc4ac8 100644 --- a/src/web/pipeline-connections/PipelineConnections.jsx +++ b/src/web/pipeline-connections/PipelineConnections.jsx @@ -1,10 +1,12 @@ import React from 'react'; import { Row, Col } from 'react-bootstrap'; import naturalSort from 'javascript-natural-sort'; +import { LinkContainer } from 'react-router-bootstrap'; import { EntityList, TypeAheadDataFilter } from 'components/common'; import Connection from './Connection'; import ConnectionForm from './ConnectionForm'; +import Routes from 'routing/Routes'; const PipelineConnections = React.createClass({ propTypes: { @@ -50,6 +52,28 @@ const PipelineConnections = React.createClass({ }); const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id && this._isConnectionWithPipelines(c))); + const pipelinesNo = this.props.pipelines.length; + let noItemsText; + if (pipelinesNo === 0) { + noItemsText = ( + + There are no pipeline connections. You can start by creating your{' '} + + pipeline rules + {' '} + and then putting them together in a{' '} + + pipeline + ! + + ); + } else { + noItemsText = ( + + There are no pipeline connections. Click on "Add new connection" to connect your pipelines to a stream. + + ); + } return (
    @@ -68,7 +92,7 @@ const PipelineConnections = React.createClass({
    -
    ); diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx index 5d9533a1066c..562bccf56322 100644 --- a/src/web/pipelines/PipelineDetailsPage.jsx +++ b/src/web/pipelines/PipelineDetailsPage.jsx @@ -103,8 +103,8 @@ const PipelineDetailsPage = React.createClass({ - - + + {' '} diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx index 52162b3d03a5..6f41e4b41bcc 100644 --- a/src/web/pipelines/ProcessingTimelineComponent.jsx +++ b/src/web/pipelines/ProcessingTimelineComponent.jsx @@ -101,7 +101,7 @@ const ProcessingTimelineComponent = React.createClass({ } const addNewPipelineButton = ( -
    +
    diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx index f8e61ede5564..16936c746000 100644 --- a/src/web/rules/Rule.jsx +++ b/src/web/rules/Rule.jsx @@ -15,6 +15,7 @@ import Routes from 'routing/Routes'; const Rule = React.createClass({ propTypes: { rule: React.PropTypes.object, + usedInPipelines: React.PropTypes.array, create: React.PropTypes.bool, onSave: React.PropTypes.func.isRequired, validateRule: React.PropTypes.func.isRequired, @@ -44,8 +45,8 @@ const Rule = React.createClass({ - - + +   @@ -56,8 +57,8 @@ const Rule = React.createClass({ - + diff --git a/src/web/rules/RuleDetailsPage.jsx b/src/web/rules/RuleDetailsPage.jsx index 9fe47f1b7b32..cc5a923d232a 100644 --- a/src/web/rules/RuleDetailsPage.jsx +++ b/src/web/rules/RuleDetailsPage.jsx @@ -7,6 +7,9 @@ import Rule from './Rule'; import RulesStore from './RulesStore'; import RulesActions from './RulesActions'; +import PipelinesActions from '../pipelines/PipelinesActions'; +import PipelinesStore from '../pipelines/PipelinesStore'; + function filterRules(state) { return state.rules ? state.rules.filter(r => r.id === this.props.params.ruleId)[0] : undefined; } @@ -17,10 +20,11 @@ const RuleDetailsPage = React.createClass({ history: React.PropTypes.object.isRequired, }, - mixins: [Reflux.connectFilter(RulesStore, 'rule', filterRules)], + mixins: [Reflux.connectFilter(RulesStore, 'rule', filterRules), Reflux.connect(PipelinesStore)], componentDidMount() { if (this.props.params.ruleId !== 'new') { + PipelinesActions.list(); RulesActions.get(this.props.params.ruleId); } }, @@ -40,7 +44,7 @@ const RuleDetailsPage = React.createClass({ }, _isLoading() { - return this.props.params.ruleId !== 'new' && !this.state.rule; + return this.props.params.ruleId !== 'new' && !(this.state.rule && this.state.pipelines); }, render() { @@ -48,11 +52,15 @@ const RuleDetailsPage = React.createClass({ return ; } + const pipelinesUsingRule = this.props.params.ruleId === 'new' ? [] : this.state.pipelines.filter(pipeline => { + return pipeline.stages.some(stage => stage.rules.indexOf(this.state.rule.title) !== -1); + }); + return ( - + ); }, }); -export default RuleDetailsPage; \ No newline at end of file +export default RuleDetailsPage; diff --git a/src/web/rules/RuleForm.css b/src/web/rules/RuleForm.css new file mode 100644 index 000000000000..bbb8a8782e37 --- /dev/null +++ b/src/web/rules/RuleForm.css @@ -0,0 +1,17 @@ +:local(.usedInPipelines) { + margin: 0; + padding: 0; +} + +:local(.usedInPipelines li:not(:last-child)) { + float: left; +} + +:local(.usedInPipelines li:not(:last-child):after) { + content: ','; + margin-right: 5px; +} + +:local(.usedInPipelines li:last-child:after) { + content: '.'; +} \ No newline at end of file diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index ac756a4c0a17..c8699235441e 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -1,5 +1,6 @@ -import React, { PropTypes } from 'react'; -import { Row, Col, Button, Input } from 'react-bootstrap'; +import React from 'react'; +import { FormControls, Row, Col, Button, Input } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; import AceEditor from 'react-ace'; import brace from 'brace'; @@ -9,9 +10,12 @@ import 'brace/theme/chrome'; import Routes from 'routing/Routes'; +import RuleFormStyle from './RuleForm.css'; + const RuleForm = React.createClass({ propTypes: { - rule: PropTypes.object, + rule: React.PropTypes.object, + usedInPipelines: React.PropTypes.array, create: React.PropTypes.bool, onSave: React.PropTypes.func.isRequired, validateRule: React.PropTypes.func.isRequired, @@ -119,13 +123,42 @@ const RuleForm = React.createClass({ this._save(); }, + _formatPipelinesUsingRule() { + if (this.props.usedInPipelines.length === 0) { + return 'This rule is not being used in any pipelines.'; + } + + const formattedPipelines = this.props.usedInPipelines.map(pipeline => { + return ( +
  • + + {pipeline.title} + +
    • + ); + }); + + return
      {formattedPipelines}
    ; + }, + render() { + let pipelinesUsingRule; + if (!this.props.create) { + pipelinesUsingRule = ( + +
    + {this._formatPipelinesUsingRule()} +
    + + ); + } + return (
    - + + {pipelinesUsingRule} +
    + disableFieldActions + disableMessageActions /> ); } diff --git a/src/web/simulator/SimulatorPage.jsx b/src/web/simulator/SimulatorPage.jsx index 114baa21e69e..f924aaaf70df 100644 --- a/src/web/simulator/SimulatorPage.jsx +++ b/src/web/simulator/SimulatorPage.jsx @@ -79,13 +79,13 @@ const SimulatorPage = React.createClass({ - - - -   +   + + + From a2ec8292c9154b43e1350419e93e7d8f17416a33 Mon Sep 17 00:00:00 2001 From: Kay Roepke Date: Wed, 17 Aug 2016 19:04:58 +0200 Subject: [PATCH 322/521] Dynamic function list (#89) * add resource to access entire function registry * add descriptions to Functions and Parameters add basic UI for displaying function descriptors not searchable yet * descriptions for conversion functions * descriptions for date functions * descriptions for hash functions * description for cidrmatch * description for json functions * descriptions for message functions * descriptions for null/notnull functions * descriptions for from_input * descriptions for string functions * tweak table widths #lolcss --- .../ast/functions/FunctionDescriptor.java | 17 ++ .../ast/functions/ParameterDescriptor.java | 15 ++ .../functions/FromInput.java | 5 +- .../functions/IsNotNull.java | 3 +- .../pipelineprocessor/functions/IsNull.java | 3 +- .../conversion/BooleanConversion.java | 5 +- .../conversion/DoubleConversion.java | 5 +- .../functions/conversion/LongConversion.java | 5 +- .../conversion/StringConversion.java | 5 +- .../functions/dates/FlexParseDate.java | 9 +- .../functions/dates/FormatDate.java | 5 +- .../functions/dates/Now.java | 5 + .../functions/dates/ParseDate.java | 9 +- .../dates/TimezoneAwareFunction.java | 4 + .../hashing/SingleArgStringFunction.java | 13 +- .../functions/ips/CidrMatch.java | 5 +- .../functions/ips/IpAddressConversion.java | 5 +- .../functions/json/JsonParse.java | 3 +- .../functions/json/SelectJsonPath.java | 4 +- .../functions/messages/CreateMessage.java | 7 +- .../functions/messages/DropMessage.java | 3 +- .../functions/messages/HasField.java | 5 +- .../functions/messages/RemoveField.java | 5 +- .../functions/messages/RenameField.java | 7 +- .../functions/messages/RouteToStream.java | 8 +- .../functions/messages/SetField.java | 11 +- .../functions/messages/SetFields.java | 9 +- .../functions/strings/Abbreviate.java | 5 +- .../functions/strings/Capitalize.java | 5 + .../functions/strings/Concat.java | 5 +- .../functions/strings/Contains.java | 7 +- .../functions/strings/GrokMatch.java | 7 +- .../functions/strings/KeyValue.java | 15 +- .../functions/strings/Lowercase.java | 5 + .../functions/strings/RegexMatch.java | 7 +- .../strings/StringUtilsFunction.java | 12 +- .../functions/strings/Substring.java | 7 +- .../functions/strings/Swapcase.java | 6 +- .../functions/strings/Uncapitalize.java | 5 + .../functions/strings/Uppercase.java | 5 + .../syslog/SyslogFacilityConversion.java | 3 +- .../syslog/SyslogLevelConversion.java | 3 +- .../syslog/SyslogPriorityConversion.java | 3 +- .../SyslogPriorityToStringConversion.java | 3 +- .../functions/urls/UrlConversion.java | 5 +- .../parser/FunctionRegistry.java | 6 + .../pipelineprocessor/rest/RuleResource.java | 17 +- src/web/rules/Rule.jsx | 2 +- src/web/rules/RuleForm.jsx | 2 +- src/web/rules/RuleHelper.jsx | 225 +++++++++--------- src/web/rules/RulesActions.jsx | 1 + src/web/rules/RulesStore.js | 24 +- 52 files changed, 364 insertions(+), 201 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java index 66d37bc87898..c23021db53d1 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java @@ -16,28 +16,44 @@ */ package org.graylog.plugins.pipelineprocessor.ast.functions; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import javax.annotation.Nullable; + @AutoValue +@JsonAutoDetect public abstract class FunctionDescriptor { + @JsonProperty public abstract String name(); + @JsonProperty public abstract boolean pure(); + @JsonProperty public abstract Class returnType(); + @JsonProperty public abstract ImmutableList params(); + @JsonIgnore public abstract ImmutableMap paramMap(); + @JsonIgnore public ParameterDescriptor param(String name) { return paramMap().get(name); } + @JsonProperty + @Nullable + public abstract String description(); + public static Builder builder() { //noinspection unchecked return new AutoValue_FunctionDescriptor.Builder().pure(false); @@ -61,5 +77,6 @@ public Builder params(ParameterDescriptor... params) { public abstract Builder params(ImmutableList params); public abstract Builder paramMap(ImmutableMap map); public abstract ImmutableList params(); + public abstract Builder description(@Nullable String description); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java index f2b411aa7cbd..cba9db448762 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -16,6 +16,9 @@ */ package org.graylog.plugins.pipelineprocessor.ast.functions; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.auto.value.AutoValue; import org.graylog.plugins.pipelineprocessor.EvaluationContext; import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; @@ -24,18 +27,28 @@ import java.util.Optional; @AutoValue +@JsonAutoDetect public abstract class ParameterDescriptor { + @JsonProperty public abstract Class type(); + @JsonProperty public abstract Class transformedType(); + @JsonProperty public abstract String name(); + @JsonProperty public abstract boolean optional(); + @JsonIgnore public abstract java.util.function.Function transform(); + @JsonProperty + @Nullable + public abstract String description(); + public static Builder param() { return new AutoValue_ParameterDescriptor.Builder().optional(false); } @@ -116,6 +129,8 @@ public Builder optional() { return optional(true); } + public abstract Builder description(String description); + abstract ParameterDescriptor autoBuild(); public ParameterDescriptor build() { try { diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java index 2d8558de8c3d..053c8798272d 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -43,8 +43,8 @@ public class FromInput extends AbstractFunction { @Inject public FromInput(InputRegistry inputRegistry) { this.inputRegistry = inputRegistry; - idParam = string(ID_ARG).optional().build(); - nameParam = string(NAME_ARG).optional().build(); + idParam = string(ID_ARG).optional().description("The input's ID, this is much faster than 'name'").build(); + nameParam = string(NAME_ARG).optional().description("The input's name").build(); } @Override @@ -83,6 +83,7 @@ public FunctionDescriptor descriptor() { .params(of( idParam, nameParam)) + .description("Checks if a message arrived on a given input") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java index 7c5fd33114ce..fc96be859b2a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java @@ -30,7 +30,7 @@ public class IsNotNull extends AbstractFunction { private final ParameterDescriptor valueParam; public IsNotNull() { - valueParam = ParameterDescriptor.type("value", Object.class).build(); + valueParam = ParameterDescriptor.type("value", Object.class).description("The value to check").build(); } @Override @@ -49,6 +49,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of(valueParam)) + .description("Checks whether a value is not 'null'") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java index adecd99b6e63..acc7b9683fa3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java @@ -30,7 +30,7 @@ public class IsNull extends AbstractFunction { private final ParameterDescriptor valueParam; public IsNull() { - valueParam = ParameterDescriptor.type("value", Object.class).build(); + valueParam = ParameterDescriptor.type("value", Object.class).description("The value to check").build(); } @Override @@ -49,6 +49,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of(valueParam)) + .description("Checks whether a value is 'null'") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java index 8f5fb5b9e1e2..dfe039f2a875 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java @@ -34,8 +34,8 @@ public class BooleanConversion extends AbstractFunction { public BooleanConversion() { - valueParam = object("value").build(); - defaultParam = bool("default").optional().build(); + valueParam = object("value").description("Value to convert").build(); + defaultParam = bool("default").optional().description("Used when 'value' is null, defaults to false").build(); } @Override @@ -53,6 +53,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(of(valueParam, defaultParam)) + .description("Converts a value to a boolean value using its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java index 78c4eb9cc64c..73345dc65ce4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java @@ -38,8 +38,8 @@ public class DoubleConversion extends AbstractFunction { private final ParameterDescriptor defaultParam; public DoubleConversion() { - valueParam = object(VALUE).build(); - defaultParam = floating(DEFAULT).optional().build(); + valueParam = object(VALUE).description("Value to convert").build(); + defaultParam = floating(DEFAULT).optional().description("Used when 'value' is null, defaults to 0").build(); } @Override @@ -62,6 +62,7 @@ public FunctionDescriptor descriptor() { valueParam, defaultParam )) + .description("Converts a value to a double value using its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java index b65687467898..8a961157cb8b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java @@ -39,8 +39,8 @@ public class LongConversion extends AbstractFunction { private final ParameterDescriptor defaultParam; public LongConversion() { - valueParam = object(VALUE).build(); - defaultParam = integer(DEFAULT).optional().build(); + valueParam = object(VALUE).description("Value to convert").build(); + defaultParam = integer(DEFAULT).optional().description("Used when 'value' is null, defaults to 0").build(); } @Override @@ -60,6 +60,7 @@ public FunctionDescriptor descriptor() { valueParam, defaultParam )) + .description("Converts a value to a long value using its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java index 5dc9a3a0797f..36a8082aa54f 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -52,8 +52,8 @@ protected boolean removeEldestEntry(Map.Entry, Class> eldest) { }; } }; - valueParam = object("value").build(); - defaultParam = string("default").optional().build(); + valueParam = object("value").description("Value to convert").build(); + defaultParam = string("default").optional().description("Used when 'value' is null, defaults to \"\"").build(); } @Override @@ -102,6 +102,7 @@ public FunctionDescriptor descriptor() { valueParam, defaultParam )) + .description("Converts a value to its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java index d37315a094d1..8469be76aa53 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java @@ -37,8 +37,8 @@ public class FlexParseDate extends TimezoneAwareFunction { private final ParameterDescriptor defaultParam; public FlexParseDate() { - valueParam = ParameterDescriptor.string(VALUE).build(); - defaultParam = ParameterDescriptor.type(DEFAULT, DateTime.class).optional().build(); + valueParam = ParameterDescriptor.string(VALUE).description("Date string to parse").build(); + defaultParam = ParameterDescriptor.type(DEFAULT, DateTime.class).optional().description("Used when 'value' could not be parsed, 'null' otherwise").build(); } @Override @@ -57,6 +57,11 @@ protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTi return new DateTime(dates.get(0).getDates().get(0), timezone); } + @Override + protected String description() { + return "Parses a date string using natural language (see http://natty.joestelmach.com/)"; + } + @Override protected String getName() { return NAME; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java index 9a11e28faf9b..541fb540bdb8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java @@ -37,13 +37,15 @@ public class FormatDate extends AbstractFunction { private final ParameterDescriptor timeZoneParam; public FormatDate() { - value = ParameterDescriptor.type("value", DateTime.class).build(); + value = ParameterDescriptor.type("value", DateTime.class).description("The date to format").build(); format = ParameterDescriptor.string("format", DateTimeFormatter.class) .transform(DateTimeFormat::forPattern) + .description("The format string to use, see http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html") .build(); timeZoneParam = ParameterDescriptor.string("timezone", DateTimeZone.class) .transform(DateTimeZone::forID) .optional() + .description("The timezone to apply to the date, defaults to UTC") .build(); } @@ -65,6 +67,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of(value, format, timeZoneParam)) + .description("Formats a date using the given format string") .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java index c1d2856ef33b..8eedfab17819 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java @@ -32,6 +32,11 @@ protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTi return DateTime.now(timezone); } + @Override + protected String description() { + return "Returns the current time"; + } + @Override protected String getName() { return NAME; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java index 04d72b9ef21b..874a3b1c09be 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java @@ -34,8 +34,8 @@ public class ParseDate extends TimezoneAwareFunction { private final ParameterDescriptor patternParam; public ParseDate() { - valueParam = ParameterDescriptor.string(VALUE).build(); - patternParam = ParameterDescriptor.string(PATTERN).build(); + valueParam = ParameterDescriptor.string(VALUE).description("Date string to parse").build(); + patternParam = ParameterDescriptor.string(PATTERN).description("The pattern to parse the date with, see http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html").build(); } @Override @@ -62,4 +62,9 @@ public DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZ return formatter.parseDateTime(dateString); } + + @Override + protected String description() { + return "Parses a date string using the given date format"; + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java index 9f4870613f54..4a1996cca8e9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java @@ -35,6 +35,7 @@ public TimezoneAwareFunction() { .string(TIMEZONE, DateTimeZone.class) .transform(DateTimeZone::forID) .optional() + .description("The timezone to apply to the date, defaults to UTC") .build(); } @@ -56,9 +57,12 @@ public FunctionDescriptor descriptor() { .addAll(params()) .add(timeZoneParam) .build()) + .description(description()) .build(); } + protected abstract String description(); + protected abstract String getName(); protected abstract ImmutableList params(); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java index c7d20326d5b0..1b53cf1ff1ba 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java @@ -22,14 +22,16 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import java.util.Locale; + import static com.google.common.collect.ImmutableList.of; -public abstract class SingleArgStringFunction extends AbstractFunction { +abstract class SingleArgStringFunction extends AbstractFunction { private final ParameterDescriptor valueParam; - public SingleArgStringFunction() { - valueParam = ParameterDescriptor.string("value").build(); + SingleArgStringFunction() { + valueParam = ParameterDescriptor.string("value").description("The value to hash").build(); } @Override @@ -42,6 +44,10 @@ public String evaluate(FunctionArgs args, EvaluationContext context) { protected abstract String getName(); + protected String description() { + return getName().toUpperCase(Locale.ENGLISH) + " hash of the string"; + } + @Override public FunctionDescriptor descriptor() { return FunctionDescriptor.builder() @@ -50,6 +56,7 @@ public FunctionDescriptor descriptor() { .params(of( valueParam) ) + .description(description()) .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java index b5d16ae89058..4d06b765b5a4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java @@ -43,8 +43,8 @@ public CidrMatch() { } catch (UnknownHostException e) { throw new IllegalArgumentException(e); } - }).build(); - ipParam = ParameterDescriptor.type(IP, IpAddress.class).build(); + }).description("The CIDR subnet mask").build(); + ipParam = ParameterDescriptor.type(IP, IpAddress.class).description("The parsed IP address to match against the CIDR mask").build(); } @Override @@ -65,6 +65,7 @@ public FunctionDescriptor descriptor() { .params(of( cidrParam, ipParam)) + .description("Checks if an IP address matches a CIDR subnet mask") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java index 78f3d3044605..42d00f219cf4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -38,8 +38,8 @@ public class IpAddressConversion extends AbstractFunction { private final ParameterDescriptor defaultParam; public IpAddressConversion() { - ipParam = ParameterDescriptor.object("ip").build(); - defaultParam = ParameterDescriptor.string("default").optional().build(); + ipParam = ParameterDescriptor.object("ip").description("Value to convert").build(); + defaultParam = ParameterDescriptor.string("default").optional().description("Used when 'ip' is null or malformed, defaults to '0.0.0.0'").build(); } @Override @@ -72,6 +72,7 @@ public FunctionDescriptor descriptor() { ipParam, defaultParam )) + .description("Converts a value to an IPAddress using its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java index 17f1056f48c9..130842b8293b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java @@ -42,7 +42,7 @@ public class JsonParse extends AbstractFunction { @Inject public JsonParse(ObjectMapper objectMapper) { this.objectMapper = objectMapper; - valueParam = ParameterDescriptor.string("value").build(); + valueParam = ParameterDescriptor.string("value").description("The string to parse as a JSON tree").build(); } @Override @@ -64,6 +64,7 @@ public FunctionDescriptor descriptor() { .params(of( valueParam )) + .description("Parses a string as a JSON tree") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java index bf8149def2c8..16a5e9566ea4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -52,7 +52,7 @@ public SelectJsonPath(ObjectMapper objectMapper) { .jsonProvider(new JacksonJsonNodeJsonProvider(objectMapper)) .build(); - jsonParam = ParameterDescriptor.type("json", JsonNode.class).build(); + jsonParam = ParameterDescriptor.type("json", JsonNode.class).description("A parsed JSON tree").build(); // sigh generics and type erasure //noinspection unchecked pathsParam = ParameterDescriptor.type("paths", @@ -61,6 +61,7 @@ public SelectJsonPath(ObjectMapper objectMapper) { .transform(inputMap -> inputMap .entrySet().stream() .collect(toMap(Map.Entry::getKey, e -> JsonPath.compile(e.getValue())))) + .description("A map of names to a JsonPath expression, see http://jsonpath.com") .build(); } @@ -120,6 +121,7 @@ public FunctionDescriptor> descriptor() { jsonParam, pathsParam )) + .description("Selects a map of fields containing the result of their JsonPath expressions") .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java index 9ccfd4e4ab41..7df5c6752f06 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java @@ -43,9 +43,9 @@ public class CreateMessage extends AbstractFunction { private final ParameterDescriptor timestampParam; public CreateMessage() { - messageParam = string(MESSAGE_ARG).optional().build(); - sourceParam = string(SOURCE_ARG).optional().build(); - timestampParam = type(TIMESTAMP_ARG, DateTime.class).optional().build(); + messageParam = string(MESSAGE_ARG).optional().description("The 'message' field of the new message, defaults to '$message.message'").build(); + sourceParam = string(SOURCE_ARG).optional().description("The 'source' field of the new message, defaults to '$message.source'").build(); + timestampParam = type(TIMESTAMP_ARG, DateTime.class).optional().description("The 'timestamp' field of the message, defaults to 'now'").build(); } @Override @@ -76,6 +76,7 @@ public FunctionDescriptor descriptor() { sourceParam, timestampParam )) + .description("Creates a new message") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java index 8373bc030450..eda54dd7930a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java @@ -33,7 +33,7 @@ public class DropMessage extends AbstractFunction { private final ParameterDescriptor messageParam; public DropMessage() { - messageParam = type(MESSAGE_ARG, Message.class).optional().build(); + messageParam = type(MESSAGE_ARG, Message.class).optional().description("The message to drop, defaults to '$message'").build(); } @Override @@ -52,6 +52,7 @@ public FunctionDescriptor descriptor() { .params(ImmutableList.of( messageParam )) + .description("Discards a message from further processing") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java index 49691d176ad4..b0b3e0950c73 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -34,8 +34,8 @@ public class HasField extends AbstractFunction { private final ParameterDescriptor messageParam; public HasField() { - fieldParam = ParameterDescriptor.string(FIELD).build(); - messageParam = type("message", Message.class).optional().build(); + fieldParam = ParameterDescriptor.string(FIELD).description("The field to check").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); } @Override @@ -52,6 +52,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Boolean.class) .params(ImmutableList.of(fieldParam, messageParam)) + .description("Checks whether a message contains a value for a field") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java index bb2325269968..dc30bf928117 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -34,8 +34,8 @@ public class RemoveField extends AbstractFunction { private final ParameterDescriptor messageParam; public RemoveField() { - fieldParam = ParameterDescriptor.string(FIELD).build(); - messageParam = type("message", Message.class).optional().build(); + fieldParam = ParameterDescriptor.string(FIELD).description("The field to remove").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); } @Override @@ -53,6 +53,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(ImmutableList.of(fieldParam, messageParam)) + .description("Removes a field from a message") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java index 2bc096ec1861..5d51f8d73026 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java @@ -35,9 +35,9 @@ public class RenameField extends AbstractFunction { private final ParameterDescriptor messageParam; public RenameField() { - oldFieldParam = string("old_field").build(); - newFieldParam = string("new_field").build(); - messageParam = type("message", Message.class).optional().build(); + oldFieldParam = string("old_field").description("The old name of the field").build(); + newFieldParam = string("new_field").description("The new name of the field").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); } @Override @@ -65,6 +65,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(oldFieldParam, newFieldParam, messageParam) + .description("Rename a message field") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java index 38c4018c8adc..590257418021 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -48,9 +48,9 @@ public RouteToStream(StreamService streamService) { this.streamService = streamService; streamService.loadAllEnabled(); - messageParam = type("message", Message.class).optional().build(); - nameParam = string(NAME_ARG).optional().build(); - idParam = string(ID_ARG).optional().build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + nameParam = string(NAME_ARG).optional().description("The name of the stream to route the message to, must match exactly").build(); + idParam = string(ID_ARG).optional().description("The ID of the stream, this is much faster than using 'name'").build(); } @Override @@ -78,7 +78,6 @@ public Void evaluate(FunctionArgs args, EvaluationContext context) { return null; } } - // TODO needs message stack in context to pick message if (!stream.isPaused()) { final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); message.addStream(stream); @@ -95,6 +94,7 @@ public FunctionDescriptor descriptor() { nameParam, idParam, messageParam)) + .description("Routes a message to a stream") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java index 48df2b8d8b96..c83edbac76e8 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -42,11 +42,11 @@ public class SetField extends AbstractFunction { private final ParameterDescriptor messageParam; public SetField() { - fieldParam = string("field").build(); - valueParam = object("value").build(); - prefixParam = string("prefix").optional().build(); - suffixParam = string("suffix").optional().build(); - messageParam = type("message", Message.class).optional().build(); + fieldParam = string("field").description("The new field name").build(); + valueParam = object("value").description("The new field value").build(); + prefixParam = string("prefix").optional().description("The prefix for the field name").build(); + suffixParam = string("suffix").optional().description("The suffix for the field name").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); } @Override @@ -80,6 +80,7 @@ public FunctionDescriptor descriptor() { prefixParam, suffixParam, messageParam)) + .description("Sets a new field in a message") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java index 009da4331c54..82969b2821da 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -40,10 +40,10 @@ public class SetFields extends AbstractFunction { private final ParameterDescriptor messageParam; public SetFields() { - fieldsParam = type("fields", Map.class).build(); - prefixParam = string("prefix").optional().build(); - suffixParam = string("suffix").optional().build(); - messageParam = type("message", Message.class).optional().build(); + fieldsParam = type("fields", Map.class).description("The map of new fields to set").build(); + prefixParam = string("prefix").optional().description("The prefix for the field names").build(); + suffixParam = string("suffix").optional().description("The suffix for the field names").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); } @Override @@ -74,6 +74,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(Void.class) .params(of(fieldsParam, prefixParam, suffixParam, messageParam)) + .description("Sets new fields in a message") .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java index 2aabd57a0baa..dcc9bc33a648 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -35,8 +35,8 @@ public class Abbreviate extends AbstractFunction { private final ParameterDescriptor widthParam; public Abbreviate() { - valueParam = ParameterDescriptor.string(VALUE).build(); - widthParam = ParameterDescriptor.integer(WIDTH).build(); + valueParam = ParameterDescriptor.string(VALUE).description("The string to abbreviate").build(); + widthParam = ParameterDescriptor.integer(WIDTH).description("The maximum number of characters including the '...' (at least 4)").build(); } @Override @@ -63,6 +63,7 @@ public FunctionDescriptor descriptor() { valueParam, widthParam )) + .description("Abbreviates a string by appending '...' to fit into a maximum amount of characters") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java index cc3e85ac13ba..3c1638002f04 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java @@ -29,6 +29,11 @@ protected String getName() { return NAME; } + @Override + protected String description() { + return "Capitalizes a String changing the first letter to title case from lower case"; + } + @Override protected boolean isLocaleAware() { return false; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java index 2086fee45b61..401019535d30 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java @@ -31,8 +31,8 @@ public class Concat extends AbstractFunction { private final ParameterDescriptor secondParam; public Concat() { - firstParam = ParameterDescriptor.string("first").build(); - secondParam = ParameterDescriptor.string("second").build(); + firstParam = ParameterDescriptor.string("first").description("First string").build(); + secondParam = ParameterDescriptor.string("second").description("Second string").build(); } @Override @@ -49,6 +49,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(of(firstParam, secondParam)) + .description("Concatenates two strings") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java index b2e68062f76f..eaaac82ee0d3 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java @@ -33,9 +33,9 @@ public class Contains extends AbstractFunction { private final ParameterDescriptor ignoreCaseParam; public Contains() { - valueParam = ParameterDescriptor.string("value").build(); - searchParam = ParameterDescriptor.string("search").build(); - ignoreCaseParam = ParameterDescriptor.bool("ignore_case").optional().build(); + valueParam = ParameterDescriptor.string("value").description("The string to check").build(); + searchParam = ParameterDescriptor.string("search").description("The substring to find").build(); + ignoreCaseParam = ParameterDescriptor.bool("ignore_case").optional().description("Whether to search case insensitive, defaults to false").build(); } @Override @@ -60,6 +60,7 @@ public FunctionDescriptor descriptor() { searchParam, ignoreCaseParam )) + .description("Checks if a string contains a substring") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java index 6fa54a53211c..7ddd716f33a5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java @@ -45,9 +45,9 @@ public class GrokMatch extends AbstractFunction { public GrokMatch(GrokPatternRegistry grokPatternRegistry) { this.grokPatternRegistry = grokPatternRegistry; - valueParam = ParameterDescriptor.string("value").build(); - patternParam = ParameterDescriptor.string("pattern").build(); - namedOnly = ParameterDescriptor.bool("only_named_captures").optional().build(); + valueParam = ParameterDescriptor.string("value").description("The string to apply the Grok pattern against").build(); + patternParam = ParameterDescriptor.string("pattern").description("The Grok pattern").build(); + namedOnly = ParameterDescriptor.bool("only_named_captures").optional().description("Whether to only use explicitly named groups in the patterns").build(); } @Override @@ -73,6 +73,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(GrokResult.class) .params(of(patternParam, valueParam, namedOnly)) + .description("Applies a Grok pattern to a string") .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java index 679d811d4fd2..4acb543db410 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java @@ -48,20 +48,22 @@ public class KeyValue extends AbstractFunction> { private final ParameterDescriptor trimValueCharactersParam; public KeyValue() { - valueParam = string("value").build(); - splitParam = string("delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().build(); - valueSplitParam = string("kv_delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().build(); + valueParam = string("value").description("The string to extract key/value pairs from").build(); + splitParam = string("delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().description("The characters used to separate pairs, defaults to whitespace").build(); + valueSplitParam = string("kv_delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().description("The characters used to separate keys from values, defaults to '='").build(); - ignoreEmptyValuesParam = bool("ignore_empty_values").optional().build(); - allowDupeKeysParam = bool("allow_dup_keys").optional().build(); - duplicateHandlingParam = string("handle_dup_keys").optional().build(); + ignoreEmptyValuesParam = bool("ignore_empty_values").optional().description("Whether to ignore keys with empty values, defaults to true").build(); + allowDupeKeysParam = bool("allow_dup_keys").optional().description("Whether to allow duplicate keys, defaults to true").build(); + duplicateHandlingParam = string("handle_dup_keys").optional().description("How to handle duplicate keys: 'take_first': only use first value, 'take_last': only take last value, default is to concatenate the values").build(); trimCharactersParam = string("trim_key_chars", CharMatcher.class) .transform(CharMatcher::anyOf) .optional() + .description("The characters to trim from keys, default is not to trim") .build(); trimValueCharactersParam = string("trim_value_chars", CharMatcher.class) .transform(CharMatcher::anyOf) .optional() + .description("The characters to trim from keys, default is not to trim") .build(); } @@ -106,6 +108,7 @@ public FunctionDescriptor> descriptor() { trimCharactersParam, trimValueCharactersParam ) + .description("Extracts key/value pairs from a string") .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java index f03609918737..f9e0af31a75b 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java @@ -29,6 +29,11 @@ protected String getName() { return NAME; } + @Override + protected String description() { + return "Lowercases a string"; + } + @Override protected boolean isLocaleAware() { return true; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index 90d4642859c0..a9ac103076b9 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -42,9 +42,9 @@ public class RegexMatch extends AbstractFunction { private final ParameterDescriptor optionalGroupNames; public RegexMatch() { - pattern = ParameterDescriptor.string("pattern", Pattern.class).transform(Pattern::compile).build(); - value = ParameterDescriptor.string("value").build(); - optionalGroupNames = ParameterDescriptor.type("group_names", List.class).optional().build(); + pattern = ParameterDescriptor.string("pattern", Pattern.class).transform(Pattern::compile).description("The regular expression to match against 'value', uses Java regex syntax").build(); + value = ParameterDescriptor.string("value").description("The string to match the pattern against").build(); + optionalGroupNames = ParameterDescriptor.type("group_names", List.class).optional().description("List of names to use for matcher groups").build(); } @Override @@ -77,6 +77,7 @@ public FunctionDescriptor descriptor() { value, optionalGroupNames )) + .description("Match a string with a regular expression (Java syntax)") .build(); } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java index 582e30e4b418..474119914d97 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java @@ -33,9 +33,12 @@ public abstract class StringUtilsFunction extends AbstractFunction { private final ParameterDescriptor localeParam; public StringUtilsFunction() { - valueParam = ParameterDescriptor.string(VALUE).build(); - localeParam = ParameterDescriptor.string(LOCALE, - Locale.class).optional().transform(Locale::forLanguageTag).build(); + valueParam = ParameterDescriptor.string(VALUE).description("The input string").build(); + localeParam = ParameterDescriptor.string(LOCALE, Locale.class) + .optional() + .transform(Locale::forLanguageTag) + .description("The locale to use, defaults to English") + .build(); } @Override @@ -59,11 +62,14 @@ public FunctionDescriptor descriptor() { .name(getName()) .returnType(String.class) .params(params.build()) + .description(description()) .build(); } protected abstract String getName(); + protected abstract String description(); + protected abstract boolean isLocaleAware(); protected abstract String apply(String value, Locale locale); diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java index aa29242351f0..cb6d20b1b721 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java @@ -34,9 +34,9 @@ public class Substring extends AbstractFunction { private final ParameterDescriptor endParam; public Substring() { - valueParam = ParameterDescriptor.string("value").build(); - startParam = ParameterDescriptor.integer("start").build(); - endParam = ParameterDescriptor.integer("end").optional().build(); + valueParam = ParameterDescriptor.string("value").description("The string to extract from").build(); + startParam = ParameterDescriptor.integer("start").description("The position to start from, negative means count back from the end of the String by this many characters").build(); + endParam = ParameterDescriptor.integer("end").optional().description("The position to end at (exclusive), negative means count back from the end of the String by this many characters, defaults to length of the input string").build(); } @Override @@ -62,6 +62,7 @@ public FunctionDescriptor descriptor() { startParam, endParam )) + .description("Extract a substring from a string") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java index f5ead84cae44..2f4ea63393e7 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java @@ -18,7 +18,6 @@ import org.apache.commons.lang3.StringUtils; -import javax.annotation.Nullable; import java.util.Locale; public class Swapcase extends StringUtilsFunction { @@ -30,6 +29,11 @@ protected String getName() { return NAME; } + @Override + protected String description() { + return "Swaps the case of a String changing upper and title case to lower case, and lower case to upper case."; + } + @Override protected boolean isLocaleAware() { return false; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java index e1dc73994992..535a5ef1457e 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java @@ -29,6 +29,11 @@ protected String getName() { return NAME; } + @Override + protected String description() { + return "Uncapitalizes a String changing the first letter to lower case from title case"; + } + @Override protected boolean isLocaleAware() { return false; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java index 32396cb9bbf7..5a06d138d2a6 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java @@ -29,6 +29,11 @@ protected String getName() { return NAME; } + @Override + protected String description() { + return "Uppercases a string"; + } + @Override protected boolean isLocaleAware() { return true; diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java index 54fae2afadbd..b5de5033b352 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java @@ -29,7 +29,7 @@ public class SyslogFacilityConversion extends AbstractFunction { public static final String NAME = "syslog_facility"; - private final ParameterDescriptor valueParam = object("value").build(); + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); @Override public String evaluate(FunctionArgs args, EvaluationContext context) { @@ -45,6 +45,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(valueParam) + .description("Converts a syslog facility number to its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java index c60044bbe365..b640b9ee7ed4 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java @@ -29,7 +29,7 @@ public class SyslogLevelConversion extends AbstractFunction { public static final String NAME = "syslog_level"; - private final ParameterDescriptor valueParam = object("value").build(); + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); @Override public String evaluate(FunctionArgs args, EvaluationContext context) { @@ -45,6 +45,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(String.class) .params(valueParam) + .description("Converts a syslog level number to its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java index 3337bb70594d..684f1b0f6fb5 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java @@ -27,7 +27,7 @@ public class SyslogPriorityConversion extends AbstractFunction { public static final String NAME = "expand_syslog_priority"; - private final ParameterDescriptor valueParam = object("value").build(); + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); @Override public SyslogPriority evaluate(FunctionArgs args, EvaluationContext context) { @@ -45,6 +45,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(SyslogPriority.class) .params(valueParam) + .description("Converts a syslog priority number to its level and facility") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java index 8da7dcfc93b9..8ffbcbe66a5a 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java @@ -27,7 +27,7 @@ public class SyslogPriorityToStringConversion extends AbstractFunction { public static final String NAME = "expand_syslog_priority_as_string"; - private final ParameterDescriptor valueParam = object("value").build(); + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); @Override public SyslogPriorityAsString evaluate(FunctionArgs args, EvaluationContext context) { @@ -47,6 +47,7 @@ public FunctionDescriptor descriptor() { .name(NAME) .returnType(SyslogPriorityAsString.class) .params(valueParam) + .description("Converts a syslog priority number to its level and facility string representations") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java index 12a4d2c445c2..a2ee4b7c9b54 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java @@ -30,8 +30,8 @@ public class UrlConversion extends AbstractFunction { public static final String NAME = "to_url"; - private final ParameterDescriptor urlParam = ParameterDescriptor.object("url").build(); - private final ParameterDescriptor defaultParam = ParameterDescriptor.string("default").optional().build(); + private final ParameterDescriptor urlParam = ParameterDescriptor.object("url").description("Value to convert").build(); + private final ParameterDescriptor defaultParam = ParameterDescriptor.string("default").optional().description("Used when 'url' is null or malformed").build(); @Override public URL evaluate(FunctionArgs args, EvaluationContext context) { @@ -61,6 +61,7 @@ public FunctionDescriptor descriptor() { .returnType(URL.class) .params(urlParam, defaultParam) + .description("Converts a value to a valid URL using its string representation") .build(); } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java index 8f66aa95981f..848866fb8994 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java @@ -19,7 +19,9 @@ import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import javax.inject.Inject; +import java.util.Collection; import java.util.Map; +import java.util.stream.Collectors; public class FunctionRegistry { @@ -42,4 +44,8 @@ public Function resolveOrError(String name) { } return function; } + + public Collection> all() { + return functions.values().stream().collect(Collectors.toList()); + } } diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java index 0cae111382e1..d0e4b94bf202 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -23,9 +23,11 @@ import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; import org.graylog.plugins.pipelineprocessor.db.RuleDao; import org.graylog.plugins.pipelineprocessor.db.RuleService; import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; import org.graylog.plugins.pipelineprocessor.parser.ParseException; import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; import org.graylog2.database.NotFoundException; @@ -64,14 +66,17 @@ public class RuleResource extends RestResource implements PluginRestResource { private final RuleService ruleService; private final PipelineRuleParser pipelineRuleParser; private final EventBus clusterBus; + private final FunctionRegistry functionRegistry; @Inject public RuleResource(RuleService ruleService, PipelineRuleParser pipelineRuleParser, - ClusterEventBus clusterBus) { + ClusterEventBus clusterBus, + FunctionRegistry functionRegistry) { this.ruleService = ruleService; this.pipelineRuleParser = pipelineRuleParser; this.clusterBus = clusterBus; + this.functionRegistry = functionRegistry; } @@ -189,4 +194,14 @@ public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws Not clusterBus.post(RulesChangedEvent.deletedRuleId(id)); } + + @ApiOperation("Get function descriptors") + @Path("/functions") + @GET + public Collection functionDescriptors() { + return functionRegistry.all().stream() + .map(Function::descriptor) + .collect(Collectors.toList()); + } + } diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx index 16936c746000..b5be36527544 100644 --- a/src/web/rules/Rule.jsx +++ b/src/web/rules/Rule.jsx @@ -60,7 +60,7 @@ const Rule = React.createClass({ - + diff --git a/src/web/rules/RuleForm.jsx b/src/web/rules/RuleForm.jsx index c8699235441e..d8f6ba69c2c7 100644 --- a/src/web/rules/RuleForm.jsx +++ b/src/web/rules/RuleForm.jsx @@ -130,7 +130,7 @@ const RuleForm = React.createClass({ const formattedPipelines = this.props.usedInPipelines.map(pipeline => { return ( -
  • +
  • {pipeline.title} diff --git a/src/web/rules/RuleHelper.jsx b/src/web/rules/RuleHelper.jsx index 900c6db63b11..07a61a427c63 100644 --- a/src/web/rules/RuleHelper.jsx +++ b/src/web/rules/RuleHelper.jsx @@ -1,11 +1,36 @@ import React from 'react'; -import { Row, Col, Panel, Tabs, Tab } from 'react-bootstrap'; +import { Row, Col, Panel, Table, Tabs, Tab } from 'react-bootstrap'; + +import Reflux from 'reflux'; + +import RulesStore from './RulesStore'; +import RulesActions from './RulesActions'; +import ObjectUtils from 'util/ObjectUtils'; import DocumentationLink from 'components/support/DocumentationLink'; +import { PaginatedList, Spinner } from 'components/common'; import DocsHelper from 'util/DocsHelper'; const RuleHelper = React.createClass({ + mixins: [ + Reflux.connect(RulesStore), + ], + + getInitialState() { + return { + expanded: {}, + paginatedEntries: undefined, + filteredEntries: undefined, + currentPage: 1, + pageSize: 10, + }; + }, + + componentDidMount() { + RulesActions.loadFunctions(); + }, + ruleTemplate: `rule "function howto" when has_field("transaction_date") @@ -15,7 +40,77 @@ then set_field("transaction_year", new_date.year); end`, + _niceType(typeName) { + return typeName.replace(/^.*\.(.*?)$/, '$1'); + }, + + _toggleFunctionDetail(functionName) { + const newState = ObjectUtils.clone(this.state.expanded); + newState[functionName] = !newState[functionName]; + this.setState({ expanded: newState }); + }, + + _functionSignature(descriptor) { + const args = descriptor.params.map(p => { return p.optional ? `[${p.name}]` : p.name; }); + return {`${descriptor.name}(${args.join(', ')}) : ${this._niceType(descriptor.return_type)}`}; + }, + + _parameters(descriptor) { + return descriptor.params.map(p => { + return ( + + {p.name} + {this._niceType(p.type)} + {p.optional ? null : } + {p.description} + ); + }); + }, + + _renderFunctions(descriptors) { + if (!descriptors) return []; + return descriptors.map((d) => { + let details = null; + if (this.state.expanded[d.name]) { + details = ( + + + + + + + + + + + + {this._parameters(d)} + +
    ParameterTypeRequiredDescription
    + + ); + } + return ( this._toggleFunctionDetail(d.name)} style={{ cursor: 'pointer' }}> + + {this._functionSignature(d)} + {d.description} + + {details} + ); + }); + }, + + _onPageChange(newPage, pageSize) { + this.setState({ currentPage: newPage, pageSize: pageSize }); + }, + render() { + if (!this.state.functionDescriptors) { + return ; + } + + const pagedEntries = this.state.functionDescriptors.slice((this.state.currentPage - 1) * this.state.pageSize, (this.state.currentPage * this.state.pageSize) - 1); + return ( @@ -30,122 +125,28 @@ end`, - - 
    -                  {this.ruleTemplate}
-
    -     - +
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    FunctionDescription
    to_bool(any)Converts the single parameter to a boolean value using its string value.
    to_double(any, [default: double])Converts the first parameter to a double floating point value.
    to_long(any, [default: long])Converts the first parameter to a long integer value.
    to_string(any, [default: string])Converts the first parameter to its string representation.
    capitalize(value: string)Capitalizes a String changing the first letter to title case.
    uppercase(value: string, [locale: string])Converts a String to upper case.
    lowercase(value: string, [locale: string])Converts a String to lower case.
    contains(value: string, search: string, [ignore_case: boolean])Checks if a string contains another string.
    substring(value: string, start: long, [end: long])Returns a substring of value with the given start and end offsets. -
    regex(pattern: string, value: string, [group_names: array[string])Match a regular expression against a string, with matcher groups.
    parse_date(value: string, pattern: string, [timezone: string])Parses a date and time from the given string, according to a strict pattern.
    flex_parse_date(value: string, [default: DateTime], [timezone: string])Attempts to parse a date and time using the Natty date parser.
    format_date(value: DateTime, format: string, [timezone: string])Formats a date and time according to a given formatter pattern.
    parse_json(value: string)Parse a string into a JSON tree.
    to_ip(ip: string)Converts the given string to an IP object.
    cidr_match(cidr: string, ip: IpAddress)Checks whether the given IP matches a CIDR pattern.
    from_input(id: string | name: string)Checks whether the current message was received by the given input.
    route_to_stream(id: string | name: string, [message: Message])Assigns the current message to the specified stream.
    drop_message(message: Message)This currently processed message will be removed from the processing pipeline after the rule - finishes. -
    has_field(field: string, [message: Message])Checks whether the currently processed message contains the named field.
    remove_field(field: string, [message: Message])Removes the named field from the currently processed message.
    set_field(field: string, value: any, [message: Message])Sets the name field to the given value in the currently processed message.
    set_fields(fields: Map<string, any>, [message: Message])Sets multiple fields to the given values in the currently processed message.
    + + + + + + + + + {this._renderFunctions(pagedEntries)} +
    FunctionDescription
    +

    See all functions in the .

     + + 
    +                  {this.ruleTemplate}
+
    +     diff --git a/src/web/rules/RulesActions.jsx b/src/web/rules/RulesActions.jsx index 70f5bc48f230..d749b974e3e7 100644 --- a/src/web/rules/RulesActions.jsx +++ b/src/web/rules/RulesActions.jsx @@ -8,6 +8,7 @@ const RulesActions = Reflux.createActions({ update: { asyncResult: true }, parse: { asyncResult: true }, multiple: { asyncResult: true }, + loadFunctions: { asyncResult: true }, }); export default RulesActions; \ No newline at end of file diff --git a/src/web/rules/RulesStore.js b/src/web/rules/RulesStore.js index fb2217a6c0cb..b8893ce9581b 100644 --- a/src/web/rules/RulesStore.js +++ b/src/web/rules/RulesStore.js @@ -11,9 +11,10 @@ const urlPrefix = '/plugins/org.graylog.plugins.pipelineprocessor'; const RulesStore = Reflux.createStore({ listenables: [RulesActions], rules: undefined, + functionDescriptors: undefined, getInitialState() { - return { rules: this.rules }; + return { rules: this.rules, functionDescriptors: this.functionDescriptors }; }, _updateRulesState(rule) { @@ -27,7 +28,14 @@ const RulesStore = Reflux.createStore({ this.rules.push(rule); } } - this.trigger({ rules: this.rules }); + this.trigger({ rules: this.rules, functionDescriptors: this.functionDescriptors }); + }, + + _updateFunctionDescriptors(functions) { + if (functions) { + this.functionDescriptors = functions; + } + this.trigger({ rules: this.rules, functionDescriptors: this.functionDescriptors }); }, list() { @@ -39,7 +47,7 @@ const RulesStore = Reflux.createStore({ const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/rule'); return fetch('GET', url).then((response) => { this.rules = response; - this.trigger({ rules: response }); + this.trigger({ rules: response, functionDescriptors: this.functionDescriptors }); }, failCallback); }, @@ -106,7 +114,7 @@ const RulesStore = Reflux.createStore({ const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/rule/${rule.id}`); return fetch('DELETE', url).then(() => { this.rules = this.rules.filter((el) => el.id !== rule.id); - this.trigger({ rules: this.rules }); + this.trigger({ rules: this.rules, functionDescriptors: this.functionDescriptors }); UserNotification.success(`Rule "${rule.title}" was deleted successfully`); }, failCallback); }, @@ -138,6 +146,14 @@ const RulesStore = Reflux.createStore({ return promise; }, + loadFunctions() { + if (this.functionDescriptors) { + return; + } + const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/rule/functions`); + return fetch('GET', url) + .then(this._updateFunctionDescriptors); + }, }); export default RulesStore; From 32984119a4efa00abef111a1baead94784e6b4fa Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 18 Aug 2016 11:21:32 +0200 Subject: [PATCH 323/521] Use find in the regex function (#88) Do not force regular expressions passed to `regex()` to match the whole string. Fixes #35 --- .../pipelineprocessor/functions/strings/RegexMatch.java | 2 +- .../functions/FunctionsSnippetsTest.java | 1 + .../plugins/pipelineprocessor/functions/regexMatch.txt | 9 +++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java index a9ac103076b9..0185b4fd90ac 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -60,7 +60,7 @@ public RegexMatchResult evaluate(FunctionArgs args, EvaluationContext context) { (List) optionalGroupNames.optional(args, context).orElse(Collections.emptyList()); final Matcher matcher = regex.matcher(value); - final boolean matches = matcher.matches(); + final boolean matches = matcher.find(); return new RegexMatchResult(matches, matcher.toMatchResult(), groupNames); diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java index ab29816ea2d4..666bf8ead3c9 100644 --- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java +++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -299,6 +299,7 @@ public void regexMatch() { assertNotNull(message); assertTrue(message.hasField("matched_regex")); assertTrue(message.hasField("group_1")); + assertThat((String) message.getField("named_group")).isEqualTo("cd.e"); } catch (ParseException e) { Assert.fail("Should parse"); } diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt index 670d7ee103a6..07124cfca4df 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt @@ -1,8 +1,13 @@ rule "regexMatch" when - regex(".*(cde\\.)(:(\\d+))?.*", "abcde.fg").matches == true + regex("^.*(cde\\.)(:(\\d+))?.*$", "abcde.fg").matches == true && + regex(".*(cde\\.)(:(\\d+))?.*", "abcde.fg").matches == true && + regex("(cde\\.)(:(\\d+))?", "abcde.fg").matches == true && + regex("^(cde\\.)(:(\\d+))?$", "abcde.fg").matches == false then - let result = regex(".*(cd\\.e).*", "abcd.efg"); + let result = regex("(cd\\.e)", "abcd.efg"); set_field("group_1", result["0"]); + let result = regex("(cd\\.e)", "abcd.efg", ["name"]); + set_field("named_group", result["name"]); set_field("matched_regex", result.matches); end \ No newline at end of file From 70f832b8e18c94e03930ecd33162aea544e7ab5b Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 18 Aug 2016 11:34:29 +0200 Subject: [PATCH 324/521] Bumping versions to 2.1.0-beta.4 / 1.1.0-beta.4 --- package.json | 2 +- pom.xml | 2 +- .../plugins/pipelineprocessor/PipelineProcessorMetaData.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 924afce1b3a4..d971c89ebdef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "PipelineProcessor", - "version": "1.1.0-beta.3", + "version": "1.1.0-beta.4", "description": "", "repository": { "type": "git", diff --git a/pom.xml b/pom.xml index 8666b04fa498..50c1851717d2 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.4-SNAPSHOT + 2.1.0-beta.4 /usr/share/graylog-server/plugin 4.5.1 1.7.21 diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java index a3e0fb9dc966..0a1b881a6030 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java @@ -50,7 +50,7 @@ public URI getURL() { @Override public Version getVersion() { - return new Version(1, 1, 0, "beta.3"); + return new Version(1, 1, 0, "beta.4"); } @Override From 2b633f670df0d8dece0eaa923e7a582ae315d9cf Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 18 Aug 2016 09:50:06 +0000 Subject: [PATCH 325/521] [graylog-plugin-pipeline-processor-release] prepare release 1.1.0-beta.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 50c1851717d2..05ab0470afc4 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.4-SNAPSHOT + 1.1.0-beta.4 jar ${project.artifactId} From ab663a0c0f5a1cf29b270ed49affd9c2d8aab42e Mon Sep 17 00:00:00 2001 From: Gary Bot Date: Thu, 18 Aug 2016 09:50:18 +0000 Subject: [PATCH 326/521] [graylog-plugin-pipeline-processor-release] prepare for next development iteration --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 05ab0470afc4..c038eda816bb 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.graylog.plugins graylog-plugin-pipeline-processor - 1.1.0-beta.4 + 1.1.0-beta.5-SNAPSHOT jar ${project.artifactId} From 9df80a113a69270be516ab771e2e07e11e287f06 Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 18 Aug 2016 12:43:21 +0200 Subject: [PATCH 327/521] Bumping graylog dependency version to next development iteration. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c038eda816bb..6f36c697440b 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ true true true - 2.1.0-beta.4 + 2.1.0-beta.5-SNAPSHOT /usr/share/graylog-server/plugin 4.5.1 1.7.21 From eaac20b03b5257f6d7e378fce7388dc170495f7e Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 18 Aug 2016 16:54:48 +0200 Subject: [PATCH 328/521] Support DateTime comparison (#92) Add support for comparison operators on DateTime objects. Fixes #86 --- .../ast/expressions/ComparisonExpression.java | 24 +++++++++++++++++-- .../pipelineprocessor/functions/dates.txt | 10 +++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java index f0177f464f48..bfcf45688171 100644 --- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java +++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java @@ -18,6 +18,7 @@ import org.antlr.v4.runtime.Token; import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.joda.time.DateTime; public class ComparisonExpression extends BinaryExpression implements LogicalExpression { private final String operator; @@ -42,11 +43,15 @@ public boolean evaluateBool(EvaluationContext context) { final Object leftValue = this.left.evaluateUnsafe(context); final Object rightValue = this.right.evaluateUnsafe(context); + if (leftValue instanceof DateTime && rightValue instanceof DateTime) { + return compareDateTimes(operator, (DateTime) leftValue, (DateTime) rightValue); + } + if (leftValue instanceof Double || rightValue instanceof Double) { return compareDouble(operator, (double) leftValue, (double) rightValue); - } else { - return compareLong(operator, (long) leftValue, (long) rightValue); } + + return compareLong(operator, (long) leftValue, (long) rightValue); } @SuppressWarnings("Duplicates") @@ -81,6 +86,21 @@ private boolean compareDouble(String operator, double left, double right) { } } + private boolean compareDateTimes(String operator, DateTime left, DateTime right) { + switch (operator) { + case ">": + return left.isAfter(right); + case ">=": + return !left.isBefore(right); + case "<": + return left.isBefore(right); + case "<=": + return !left.isAfter(right); + default: + return false; + } + } + @Override public String toString() { return left.toString() + " " + operator + " " + right.toString(); diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt index 051fc005b101..1ba3d2bd8617 100644 --- a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt +++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt @@ -6,7 +6,15 @@ when now("CET") == now("UTC") && now("CET") == now() && flex_parse_date(value: "30th July 2010 18:03:25 ", timezone: "CET") == parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") && - format_date(flex_parse_date("30th July 2010 18:03:25"), "yyyy-MM-dd") == "2010-07-30" + format_date(flex_parse_date("30th July 2010 18:03:25"), "yyyy-MM-dd") == "2010-07-30" && + parse_date("2010-07-30T18:03:24+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") < parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:24+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") >= parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") > parse_date("2010-07-30T16:03:24Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") <= parse_date("2010-07-30T16:03:24Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") <= parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") > parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") >= parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") < parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) then trigger_test(); let date = parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ"); From d275227e7ff87bb25bfbedd0da762a64cb13ac4d Mon Sep 17 00:00:00 2001 From: Edmundo Alvarez Date: Thu, 18 Aug 2016 17:00:33 +0200 Subject: [PATCH 329/521] Make some small UI changes around RuleHelper (#90) - Update text descriptions - Hide page selector input --- src/web/rules/RuleHelper.css | 23 +++++++++++++++++++++++ src/web/rules/RuleHelper.jsx | 31 +++++++++++++++++++------------ 2 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 src/web/rules/RuleHelper.css diff --git a/src/web/rules/RuleHelper.css b/src/web/rules/RuleHelper.css new file mode 100644 index 000000000000..cca21818a08f --- /dev/null +++ b/src/web/rules/RuleHelper.css @@ -0,0 +1,23 @@ +:local(.clickableRow) { + cursor: pointer; +} + +:local(.functionTableCell) { + width: 300px; +} + +:local(.marginQuickReferenceText) { + margin-top: 5px; +} + +:local(.marginTab) { + margin-top: 10px; +} + +:local(.exampleFunction) { + white-space: pre-wrap; +} + +:local(.adjustedTableCellWidth) { + width: 1%; +} \ No newline at end of file diff --git a/src/web/rules/RuleHelper.jsx b/src/web/rules/RuleHelper.jsx index 07a61a427c63..6b5527ee215e 100644 --- a/src/web/rules/RuleHelper.jsx +++ b/src/web/rules/RuleHelper.jsx @@ -12,6 +12,8 @@ import { PaginatedList, Spinner } from 'components/common'; import DocsHelper from 'util/DocsHelper'; +import RuleHelperStyle from './RuleHelper.css'; + const RuleHelper = React.createClass({ mixins: [ Reflux.connect(RulesStore), @@ -59,9 +61,9 @@ end`, return descriptor.params.map(p => { return ( - {p.name} - {this._niceType(p.type)} - {p.optional ? null : } + {p.name} + {this._niceType(p.type)} + {p.optional ? null : } {p.description} ); }); @@ -90,9 +92,9 @@ end`, ); } - return ( this._toggleFunctionDetail(d.name)} style={{ cursor: 'pointer' }}> - - {this._functionSignature(d)} + return ( + this._toggleFunctionDetail(d.name)} className={RuleHelperStyle.clickableRow}> + {this._functionSignature(d)} {d.description} {details} @@ -115,7 +117,7 @@ end`, -

    +

    Read the {' '} to gain a better understanding of how Graylog pipeline rules work. @@ -126,8 +128,12 @@ end`, -

    - +

    + This is a list of all available functions in pipeline rules. Click on a row to see more information + about the function parameters. +

    +
    + @@ -139,11 +145,12 @@ end`,
    -

    See all functions in the .

    - 
    +                
    
+                  Do you want to see how a pipeline rule looks like? Take a look at this example:
+                
    
+                
                       {this.ruleTemplate}
                 
    
               

From 60a91867709f79228e7952a789dd754576f68d42 Mon Sep 17 00:00:00 2001
From: Kay Roepke 
Date: Fri, 19 Aug 2016 15:40:08 +0200
Subject: [PATCH 330/521] use shared classloader so other plugins can
 contribute functions (#94)

fixes #81
---
 pom.xml                                              |  8 +++++---
 .../graylog-plugin.properties                        | 12 ++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 src/main/resources/org.graylog.plugins.graylog-plugin-pipeline-processor/graylog-plugin.properties

diff --git a/pom.xml b/pom.xml
index 6f36c697440b..afb6bb5be3db 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,9 +194,11 @@
             
             
                 src/main/resources
-                
-                    **/*.properties
-                
+                
+                    **/version.properties
+                    **/graylog-plugin.properties
+                
+                true
             
         
         
diff --git a/src/main/resources/org.graylog.plugins.graylog-plugin-pipeline-processor/graylog-plugin.properties b/src/main/resources/org.graylog.plugins.graylog-plugin-pipeline-processor/graylog-plugin.properties
new file mode 100644
index 000000000000..fcebb30dca8e
--- /dev/null
+++ b/src/main/resources/org.graylog.plugins.graylog-plugin-pipeline-processor/graylog-plugin.properties
@@ -0,0 +1,12 @@
+# The plugin version
+version=${project.version}
+
+# The required Graylog server version
+graylog.version=${graylog.version}
+
+# When set to true (the default) the plugin gets a separate class loader
+# when loading the plugin. When set to false, the plugin shares a class loader
+# with other plugins that have isolated=false.
+#
+# Do not disable this unless this plugin depends on another plugin!
+isolated=false

From e8a0122bce4c6796b76b70074d88b70e3d03c0a0 Mon Sep 17 00:00:00 2001
From: Bernd Ahlers 
Date: Tue, 23 Aug 2016 12:25:24 +0200
Subject: [PATCH 331/521] Fix loading issue by removing  from
 resource

---
 pom.xml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index afb6bb5be3db..a84f39a83cb2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,10 +194,6 @@
             
             
                 src/main/resources
-                
-                    **/version.properties
-                    **/graylog-plugin.properties
-                
                 true
             
         

From 41b2b11ba85e40f2020a5a2c5cc8f0aaa953b407 Mon Sep 17 00:00:00 2001
From: Bernd Ahlers 
Date: Tue, 23 Aug 2016 12:28:36 +0200
Subject: [PATCH 332/521] Rename graylog2.{plugin-dir,version} to graylog.*

---
 pom.xml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index a84f39a83cb2..5de260401c83 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,8 +37,8 @@
         true
         true
         true
-        2.1.0-beta.5-SNAPSHOT
-        /usr/share/graylog-server/plugin
+        2.1.0-beta.5-SNAPSHOT
+        /usr/share/graylog-server/plugin
         4.5.1
         1.7.21
         2.6.2
@@ -123,7 +123,7 @@
         
             org.graylog2
             graylog2-server
-            ${graylog2.version}
+            ${graylog.version}
             provided
         
         
@@ -311,7 +311,7 @@
                             file
                             
                                 perm
-                                ${graylog2.plugin-dir}
+                                ${graylog.plugin-dir}
                                 644
                                 root
                                 root
@@ -339,7 +339,7 @@
                     root
                     
                         
-                            ${graylog2.plugin-dir}
+                            ${graylog.plugin-dir}
                             
                                 
                                     ${project.build.directory}/

From 3a07acee67a28404af1d81626862a90b19d3a829 Mon Sep 17 00:00:00 2001
From: Bernd Ahlers 
Date: Tue, 23 Aug 2016 12:28:55 +0200
Subject: [PATCH 333/521] Add missing Graylog-Plugin-Properties-Path manifest
 entry

---
 pom.xml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/pom.xml b/pom.xml
index 5de260401c83..f7a3e7df88f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -255,6 +255,18 @@
             
         
         
+            
+                org.apache.maven.plugins
+                maven-jar-plugin
+                2.6
+                
+                    
+                        
+                            ${project.groupId}.${project.artifactId}
+                        
+                    
+                
+            
             
                 maven-assembly-plugin
                 

From 8f80034256eaa514d4fd5eb984fc59b194777a68 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Tue, 23 Aug 2016 14:05:27 +0200
Subject: [PATCH 334/521] Integrate audit log (#96)

* Integrate audit log
* Replace edit with update action on pipeline connections
* Add license
---
 .../PipelineProcessorModule.java              |  3 ++
 .../PipelineProcessorAuditEventTypes.java     | 49 +++++++++++++++++++
 .../rest/PipelineConnectionsResource.java     |  3 ++
 .../rest/PipelineResource.java                |  7 +++
 .../pipelineprocessor/rest/RuleResource.java  |  8 +++
 .../rest/SimulatorResource.java               |  2 +
 6 files changed, 72 insertions(+)
 create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java

diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java
index e93bae59a2d4..ab57d1f8f934 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java
@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.pipelineprocessor;
 
+import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes;
 import org.graylog.plugins.pipelineprocessor.functions.ProcessorFunctionsModule;
 import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter;
 import org.graylog.plugins.pipelineprocessor.rest.PipelineConnectionsResource;
@@ -50,5 +51,7 @@ protected void configure() {
         installSearchResponseDecorator(searchResponseDecoratorBinder(),
                 PipelineProcessorMessageDecorator.class,
                 PipelineProcessorMessageDecorator.Factory.class);
+
+        addAuditEventTypes(PipelineProcessorAuditEventTypes.class);
     }
 }
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java b/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java
new file mode 100644
index 000000000000..b1ddb81235d4
--- /dev/null
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java
@@ -0,0 +1,49 @@
+/**
+ * This file is part of Graylog Pipeline Processor.
+ *
+ * Graylog Pipeline Processor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog Pipeline Processor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog Pipeline Processor.  If not, see .
+ */
+package org.graylog.plugins.pipelineprocessor.audit;
+
+import com.google.common.collect.ImmutableSet;
+import org.graylog2.audit.PluginAuditEventTypes;
+
+import java.util.Set;
+
+public class PipelineProcessorAuditEventTypes implements PluginAuditEventTypes {
+    private static final String NAMESPACE = "pipeline_processor:";
+
+    public static final String PIPELINE_CONNECTION_UPDATE = NAMESPACE + "pipeline_connection:update";
+    public static final String PIPELINE_CREATE = NAMESPACE + "pipeline:create";
+    public static final String PIPELINE_UPDATE = NAMESPACE + "pipeline:update";
+    public static final String PIPELINE_DELETE = NAMESPACE + "pipeline:delete";
+    public static final String RULE_CREATE = NAMESPACE + "rule:create";
+    public static final String RULE_UPDATE = NAMESPACE + "rule:update";
+    public static final String RULE_DELETE = NAMESPACE + "rule:delete";
+
+    private static final Set EVENT_TYPES = ImmutableSet.builder()
+            .add(PIPELINE_CONNECTION_UPDATE)
+            .add(PIPELINE_CREATE)
+            .add(PIPELINE_UPDATE)
+            .add(PIPELINE_DELETE)
+            .add(RULE_CREATE)
+            .add(RULE_UPDATE)
+            .add(RULE_DELETE)
+            .build();
+
+    @Override
+    public Set auditEventTypes() {
+        return EVENT_TYPES;
+    }
+}
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java
index 031cce09677e..8f86317c17ad 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java
@@ -23,9 +23,11 @@
 import io.swagger.annotations.ApiParam;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
 import org.apache.shiro.authz.annotation.RequiresPermissions;
+import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes;
 import org.graylog.plugins.pipelineprocessor.db.PipelineService;
 import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService;
 import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent;
+import org.graylog2.audit.jersey.AuditEvent;
 import org.graylog2.database.NotFoundException;
 import org.graylog2.events.ClusterEventBus;
 import org.graylog2.plugin.rest.PluginRestResource;
@@ -72,6 +74,7 @@ public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsS
     @ApiOperation(value = "Connect processing pipelines to a stream", notes = "")
     @POST
     @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_EDIT)
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CONNECTION_UPDATE)
     public PipelineConnections connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineConnections connection) throws NotFoundException {
         final String streamId = connection.streamId();
         // the default stream doesn't exist as an entity
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java
index bb9d9270558c..f69467bbde56 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java
@@ -24,11 +24,14 @@
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
 import org.apache.shiro.authz.annotation.RequiresPermissions;
 import org.graylog.plugins.pipelineprocessor.ast.Pipeline;
+import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes;
 import org.graylog.plugins.pipelineprocessor.db.PipelineDao;
 import org.graylog.plugins.pipelineprocessor.db.PipelineService;
 import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent;
 import org.graylog.plugins.pipelineprocessor.parser.ParseException;
 import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser;
+import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.database.NotFoundException;
 import org.graylog2.events.ClusterEventBus;
 import org.graylog2.plugin.rest.PluginRestResource;
@@ -78,6 +81,7 @@ public PipelineResource(PipelineService pipelineService,
     @ApiOperation(value = "Create a processing pipeline from source", notes = "")
     @POST
     @RequiresPermissions(PipelineRestPermissions.PIPELINE_CREATE)
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CREATE)
     public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException {
         final Pipeline pipeline;
         try {
@@ -101,6 +105,7 @@ public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = t
     @ApiOperation(value = "Parse a processing pipeline without saving it", notes = "")
     @POST
     @Path("/parse")
+    @NoAuditEvent("only used to parse a pipeline, no changes made in the system")
     public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException {
         final Pipeline pipeline;
         try {
@@ -143,6 +148,7 @@ public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) thr
     @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied")
     @Path("/{id}")
     @PUT
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_UPDATE)
     public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id,
                              @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException {
         checkPermission(PipelineRestPermissions.PIPELINE_EDIT, id);
@@ -169,6 +175,7 @@ public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id,
     @ApiOperation(value = "Delete a processing pipeline", notes = "It can take up to a second until the change is applied")
     @Path("/{id}")
     @DELETE
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_DELETE)
     public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException {
         checkPermission(PipelineRestPermissions.PIPELINE_DELETE, id);
 
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java
index d0e4b94bf202..d99819ba3c5e 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java
@@ -24,12 +24,15 @@
 import org.apache.shiro.authz.annotation.RequiresPermissions;
 import org.graylog.plugins.pipelineprocessor.ast.Rule;
 import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
+import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes;
 import org.graylog.plugins.pipelineprocessor.db.RuleDao;
 import org.graylog.plugins.pipelineprocessor.db.RuleService;
 import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent;
 import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry;
 import org.graylog.plugins.pipelineprocessor.parser.ParseException;
 import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser;
+import org.graylog2.audit.jersey.AuditEvent;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.database.NotFoundException;
 import org.graylog2.events.ClusterEventBus;
 import org.graylog2.plugin.rest.PluginRestResource;
@@ -83,6 +86,7 @@ public RuleResource(RuleService ruleService,
     @ApiOperation(value = "Create a processing rule from source", notes = "")
     @POST
     @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_CREATE)
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.RULE_CREATE)
     public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException {
         final Rule rule;
         try {
@@ -107,6 +111,7 @@ public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @No
     @ApiOperation(value = "Parse a processing rule without saving it", notes = "")
     @POST
     @Path("/parse")
+    @NoAuditEvent("only used to parse a rule, no changes made in the system")
     public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException {
         final Rule rule;
         try {
@@ -145,6 +150,7 @@ public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws
     @ApiOperation("Retrieve the named processing rules in bulk")
     @Path("/multiple")
     @POST
+    @NoAuditEvent("only used to get multiple pipeline rules")
     public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) {
         Collection ruleDaos = ruleService.loadNamed(rules.rules());
 
@@ -157,6 +163,7 @@ public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules)
     @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied")
     @Path("/{id}")
     @PUT
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.RULE_UPDATE)
     public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id,
                              @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException {
         checkPermission(PipelineRestPermissions.PIPELINE_RULE_EDIT, id);
@@ -185,6 +192,7 @@ public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id,
     @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied")
     @Path("/{id}")
     @DELETE
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.RULE_DELETE)
     public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException {
         checkPermission(PipelineRestPermissions.PIPELINE_RULE_DELETE, id);
         ruleService.load(id);
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java
index 36cae1aab36a..c15207c24c3b 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.common.Strings;
 import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter;
 import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTracer;
+import org.graylog2.audit.jersey.NoAuditEvent;
 import org.graylog2.database.NotFoundException;
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.rest.PluginRestResource;
@@ -62,6 +63,7 @@ public SimulatorResource(PipelineInterpreter pipelineInterpreter,
     @ApiOperation(value = "Simulate the execution of the pipeline message processor")
     @POST
     @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_READ)
+    @NoAuditEvent("only used to test pipelines, no changes made in the system")
     public SimulationResponse simulate(@ApiParam(name = "simulation", required = true) @NotNull SimulationRequest request) throws NotFoundException {
         checkPermission(RestPermissions.STREAMS_READ, request.streamId());
 

From 5a9bfda0579d49f3a6567e6dd17e4577f59b1fed Mon Sep 17 00:00:00 2001
From: Kay Roepke 
Date: Thu, 25 Aug 2016 11:26:44 +0200
Subject: [PATCH 335/521] add parse error handler for precompute args failures
 (#93)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add parse error handler for precompute args failures

allow lowercase timezone ids

* make error message look nicer

* don't hit undo before commit…

* Fix license check
---
 .../ast/exceptions/PrecomputeFailure.java     | 35 +++++++++++
 .../ast/functions/Function.java               |  5 +-
 .../dates/TimezoneAwareFunction.java          |  2 +-
 .../parser/PipelineRuleParser.java            | 14 ++++-
 .../errors/InvalidFunctionArgument.java       | 63 +++++++++++++++++++
 .../parser/PipelineRuleParserTest.java        | 38 +++++++++++
 .../parser/invalidArgumentValue.txt           |  4 ++
 7 files changed, 155 insertions(+), 6 deletions(-)
 create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java
 create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java
 create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt

diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java
new file mode 100644
index 000000000000..192a6c43d643
--- /dev/null
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java
@@ -0,0 +1,35 @@
+/**
+ * This file is part of Graylog Pipeline Processor.
+ *
+ * Graylog Pipeline Processor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog Pipeline Processor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog Pipeline Processor.  If not, see .
+ */
+package org.graylog.plugins.pipelineprocessor.ast.exceptions;
+
+public class PrecomputeFailure extends RuntimeException {
+    private final String argumentName;
+
+    public PrecomputeFailure(String argumentName, Exception cause) {
+        super(cause);
+        this.argumentName = argumentName;
+    }
+
+    public String getArgumentName() {
+        return argumentName;
+    }
+
+    @Override
+    public String getMessage() {
+        return "Unable to pre-compute argument " + getArgumentName() + ": " + getCause().getMessage();
+    }
+}
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java
index bd455c5901e8..4133f7bcac9c 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java
@@ -18,6 +18,7 @@
 
 import com.google.common.collect.ImmutableList;
 import org.graylog.plugins.pipelineprocessor.EvaluationContext;
+import org.graylog.plugins.pipelineprocessor.ast.exceptions.PrecomputeFailure;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,8 +64,8 @@ default void preprocessArgs(FunctionArgs args) {
                     args.setPreComputedValue(name, param.transform().apply(value));
                 }
             } catch (Exception exception) {
-                log.warn("Unable to precompute argument value for " + name, exception);
-                throw exception;
+                log.debug("Unable to precompute argument value for " + name, exception);
+                throw new PrecomputeFailure(name, exception);
             }
         }
 
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java
index 4a1996cca8e9..7747c805aab3 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java
@@ -33,7 +33,7 @@ public abstract class TimezoneAwareFunction extends AbstractFunction {
     public TimezoneAwareFunction() {
         timeZoneParam = ParameterDescriptor
                 .string(TIMEZONE, DateTimeZone.class)
-                .transform(DateTimeZone::forID)
+                .transform(id -> DateTimeZone.forID(id.toUpperCase()))
                 .optional()
                 .description("The timezone to apply to the date, defaults to UTC")
                 .build();
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java
index ad929986a083..8236f620fb5f 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java
@@ -36,6 +36,7 @@
 import org.graylog.plugins.pipelineprocessor.ast.Pipeline;
 import org.graylog.plugins.pipelineprocessor.ast.Rule;
 import org.graylog.plugins.pipelineprocessor.ast.Stage;
+import org.graylog.plugins.pipelineprocessor.ast.exceptions.PrecomputeFailure;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression;
@@ -68,6 +69,7 @@
 import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType;
 import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleType;
 import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleTypes;
+import org.graylog.plugins.pipelineprocessor.parser.errors.InvalidFunctionArgument;
 import org.graylog.plugins.pipelineprocessor.parser.errors.MissingRequiredParam;
 import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType;
 import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed;
@@ -357,9 +359,15 @@ public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) {
                 }
             }
 
-            final FunctionExpression expr = new FunctionExpression(
-                    ctx.getStart(), new FunctionArgs(functionRegistry.resolveOrError(name), argsMap)
-            );
+            FunctionExpression expr;
+            try {
+                expr = new FunctionExpression(
+                        ctx.getStart(), new FunctionArgs(functionRegistry.resolveOrError(name), argsMap)
+                );
+            } catch (PrecomputeFailure precomputeFailure) {
+                parseContext.addError(new InvalidFunctionArgument(ctx, function, precomputeFailure));
+                expr = new FunctionExpression(ctx.getStart(), new FunctionArgs(Function.ERROR_FUNCTION, argsMap));
+            }
 
             log.trace("FUNC: ctx {} => {}", ctx, expr);
             exprs.put(ctx, expr);
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java
new file mode 100644
index 000000000000..cfd0fdb28f56
--- /dev/null
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java
@@ -0,0 +1,63 @@
+/**
+ * This file is part of Graylog Pipeline Processor.
+ *
+ * Graylog Pipeline Processor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog Pipeline Processor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog Pipeline Processor.  If not, see .
+ */
+package org.graylog.plugins.pipelineprocessor.parser.errors;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.graylog.plugins.pipelineprocessor.ast.exceptions.PrecomputeFailure;
+import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
+import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser;
+
+public class InvalidFunctionArgument extends ParseError {
+    private final Function function;
+    private final PrecomputeFailure failure;
+
+    public InvalidFunctionArgument(RuleLangParser.FunctionCallContext ctx,
+                                   Function function,
+                                   PrecomputeFailure failure) {
+        super("invalid_function_argument", ctx);
+        this.function = function;
+        this.failure = failure;
+    }
+
+    @JsonProperty("reason")
+    @Override
+    public String toString() {
+        int paramPosition = 1;
+        for (ParameterDescriptor descriptor : function.descriptor().params()) {
+            if (descriptor.name().equals(failure.getArgumentName())) {
+                break;
+            }
+            paramPosition++;
+        }
+
+        return "Unable to pre-compute value for " + ordinal(paramPosition) + " argument " + failure.getArgumentName() + " in call to function " + function.descriptor().name() + ": " + failure.getCause().getMessage();
+    }
+
+    private static String ordinal(int i) {
+        String[] suffixes = new String[]{"th", "st", "nd", "rd", "th", "th", "th", "th", "th", "th"};
+        switch (i % 100) {
+            case 11:
+            case 12:
+            case 13:
+                return i + "th";
+            default:
+                return i + suffixes[i % 10];
+
+        }
+    }
+}
diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java
index 135b267f088a..699546b6aa5c 100644
--- a/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java
+++ b/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java
@@ -16,6 +16,7 @@
  */
 package org.graylog.plugins.pipelineprocessor.parser;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Ordering;
@@ -31,17 +32,21 @@
 import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
 import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion;
 import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion;
+import org.graylog.plugins.pipelineprocessor.functions.dates.TimezoneAwareFunction;
 import org.graylog.plugins.pipelineprocessor.functions.messages.HasField;
 import org.graylog.plugins.pipelineprocessor.functions.messages.SetField;
 import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch;
 import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType;
 import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType;
+import org.graylog.plugins.pipelineprocessor.parser.errors.InvalidFunctionArgument;
 import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType;
 import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed;
+import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError;
 import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction;
 import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable;
 import org.graylog2.plugin.Message;
 import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -253,6 +258,27 @@ public FunctionDescriptor descriptor() {
         functions.put(SetField.NAME, new SetField());
         functions.put(HasField.NAME, new HasField());
         functions.put(RegexMatch.NAME, new RegexMatch());
+        functions.put("now_in_tz", new TimezoneAwareFunction() {
+            @Override
+            protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) {
+                return DateTime.now(timezone);
+            }
+
+            @Override
+            protected String description() {
+                return "Now in the given timezone";
+            }
+
+            @Override
+            protected String getName() {
+                return "now_in_tz";
+            }
+
+            @Override
+            protected ImmutableList params() {
+                return ImmutableList.of();
+            }
+        });
         functionRegistry = new FunctionRegistry(functions);
     }
 
@@ -467,6 +493,18 @@ public void indexedAccessWrongIndexType() {
         }
     }
 
+    @Test
+    public void invalidArgumentValue() {
+        try {
+            parser.parseRule(ruleForTest(), false);
+        } catch (ParseException e) {
+            assertEquals(1, e.getErrors().size());
+            final ParseError parseError = Iterables.getOnlyElement(e.getErrors());
+            assertEquals("Unable to pre-compute value for 1st argument timezone in call to function now_in_tz: The datetime zone id '123' is not recognised", parseError.toString());
+            assertEquals(InvalidFunctionArgument.class, parseError.getClass());
+        }
+    }
+
     public static class CustomObject {
         private final String id;
 
diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt
new file mode 100644
index 000000000000..f1d96cfa30ee
--- /dev/null
+++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt
@@ -0,0 +1,4 @@
+rule "invalid arg"
+when now_in_tz("123")  // this isn't a valid tz
+then
+end
\ No newline at end of file

From 83a04074c3b885a9f4329583e9526425460010d5 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Thu, 25 Aug 2016 13:13:29 +0200
Subject: [PATCH 336/521] Bumping versions to 2.1.0-rc.1 / 1.1.0-rc.1

---
 package.json                                                    | 2 +-
 pom.xml                                                         | 2 +-
 .../plugins/pipelineprocessor/PipelineProcessorMetaData.java    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index d971c89ebdef..e738fee82c46 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "PipelineProcessor",
-  "version": "1.1.0-beta.4",
+  "version": "1.1.0-rc.1",
   "description": "",
   "repository": {
     "type": "git",
diff --git a/pom.xml b/pom.xml
index f7a3e7df88f0..07c7fe63802e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,7 +37,7 @@
         true
         true
         true
-        2.1.0-beta.5-SNAPSHOT
+        2.1.0-rc.1
         /usr/share/graylog-server/plugin
         4.5.1
         1.7.21
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java
index 0a1b881a6030..b2425c1f9407 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMetaData.java
@@ -50,7 +50,7 @@ public URI getURL() {
 
     @Override
     public Version getVersion() {
-        return new Version(1, 1, 0, "beta.4");
+        return new Version(1, 1, 0, "rc.1");
     }
 
     @Override

From b47f06b20b681063bb9501d1830c1801701f7050 Mon Sep 17 00:00:00 2001
From: Gary Bot 
Date: Thu, 25 Aug 2016 11:26:49 +0000
Subject: [PATCH 337/521] [graylog-plugin-pipeline-processor-release] prepare
 release 1.1.0-rc.1

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 07c7fe63802e..619a13c1f59f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
 
     org.graylog.plugins
     graylog-plugin-pipeline-processor
-    1.1.0-beta.5-SNAPSHOT
+    1.1.0-rc.1
     jar
 
     ${project.artifactId}

From 560383f893b1e017f8cfb3dd0513fc43c5e66c67 Mon Sep 17 00:00:00 2001
From: Gary Bot 
Date: Thu, 25 Aug 2016 11:27:01 +0000
Subject: [PATCH 338/521] [graylog-plugin-pipeline-processor-release] prepare
 for next development iteration

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 619a13c1f59f..20b4b420a153 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
 
     org.graylog.plugins
     graylog-plugin-pipeline-processor
-    1.1.0-rc.1
+    1.1.0-rc.2-SNAPSHOT
     jar
 
     ${project.artifactId}

From c97483795306e0fcd949f58ceb64e46c5c407ec8 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Thu, 25 Aug 2016 14:05:49 +0200
Subject: [PATCH 339/521] Bumping graylog dependency version to next
 development iteration.

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 20b4b420a153..042f43d1c6e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -37,7 +37,7 @@
         true
         true
         true
-        2.1.0-rc.1
+        2.1.0-rc.2-SNAPSHOT
         /usr/share/graylog-server/plugin
         4.5.1
         1.7.21

From 71f5b31ea8f0eb96b8a973e6f1bf9b8e2c1c0f03 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Tue, 30 Aug 2016 16:20:57 +0200
Subject: [PATCH 340/521] Fix page size in function list (#97)

`slice()` doesn't include the end element in the array that returns, so
we were rendering 9 functions instead of 10.
---
 src/web/rules/RuleHelper.jsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/web/rules/RuleHelper.jsx b/src/web/rules/RuleHelper.jsx
index 6b5527ee215e..30bcb02fb678 100644
--- a/src/web/rules/RuleHelper.jsx
+++ b/src/web/rules/RuleHelper.jsx
@@ -111,7 +111,7 @@ end`,
       return ;
     }
 
-    const pagedEntries = this.state.functionDescriptors.slice((this.state.currentPage - 1) * this.state.pageSize, (this.state.currentPage * this.state.pageSize) - 1);
+    const pagedEntries = this.state.functionDescriptors.slice((this.state.currentPage - 1) * this.state.pageSize, this.state.currentPage * this.state.pageSize);
 
     return (
       

From 11361e34e4e19a2b83a586abe4bf543b6a57e38e Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Tue, 30 Aug 2016 17:36:16 +0200
Subject: [PATCH 341/521] Add '@Test' annotation to keyValue test

Refs #77
---
 .../pipelineprocessor/functions/FunctionsSnippetsTest.java       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java
index 666bf8ead3c9..96bcb80eb45e 100644
--- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java
+++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java
@@ -530,6 +530,7 @@ public void fieldPrefixSuffix() {
         assertThat(message.getField("field2_suff")).isEqualTo("10");
     }
 
+    @Test
     public void keyValue() {
         final Rule rule = parser.parseRule(ruleForTest(), true);
 

From a0eb8210f221c8f2ec3382001d84aba7e792ef06 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Tue, 30 Aug 2016 18:10:02 +0200
Subject: [PATCH 342/521] Correct trim value argument description

---
 .../plugins/pipelineprocessor/functions/strings/KeyValue.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java
index 4acb543db410..8737c77140f9 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java
@@ -63,7 +63,7 @@ public KeyValue() {
         trimValueCharactersParam = string("trim_value_chars", CharMatcher.class)
                 .transform(CharMatcher::anyOf)
                 .optional()
-                .description("The characters to trim from keys, default is not to trim")
+                .description("The characters to trim from values, default is not to trim")
                 .build();
     }
 

From 7e92759684d8f39bb115ecc0446281ceefc2f0f8 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Thu, 1 Sep 2016 15:08:18 +0200
Subject: [PATCH 343/521] Update version to 1.2.0-SNAPSHOT

---
 package.json | 2 +-
 pom.xml      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index e738fee82c46..865bae2b34ff 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "PipelineProcessor",
-  "version": "1.1.0-rc.1",
+  "version": "1.2.0-SNAPSHOT",
   "description": "",
   "repository": {
     "type": "git",
diff --git a/pom.xml b/pom.xml
index 042f43d1c6e5..fac09468a7ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
 
     org.graylog.plugins
     graylog-plugin-pipeline-processor
-    1.1.0-rc.2-SNAPSHOT
+    1.2.0-SNAPSHOT
     jar
 
     ${project.artifactId}
@@ -37,7 +37,7 @@
         true
         true
         true
-        2.1.0-rc.2-SNAPSHOT
+        2.2.0-SNAPSHOT
         /usr/share/graylog-server/plugin
         4.5.1
         1.7.21

From fcd9e895ff4e2ac903d770b278b04c8a47c4aeb8 Mon Sep 17 00:00:00 2001
From: Kay Roepke 
Date: Tue, 13 Sep 2016 12:21:33 +0200
Subject: [PATCH 344/521] use case insensitive lookup for timezone IDs (#102)

* use case insensitive lookup for timezone IDs

simply upper-casing timezone IDs failed for strings like 'Europe/Moscow'. Unfortunately the forID function is case sensitive.

fixes #100

* override the millis provider to stabilize test

* remove unused statement and fix import
---
 .../functions/dates/TimezoneAwareFunction.java     | 13 ++++++++++---
 .../functions/FunctionsSnippetsTest.java           | 14 ++++++++++++++
 .../pipelineprocessor/functions/timezones.txt      | 11 +++++++++++
 3 files changed, 35 insertions(+), 3 deletions(-)
 create mode 100644 src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt

diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java
index 7747c805aab3..59b215480cd7 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java
@@ -17,6 +17,8 @@
 package org.graylog.plugins.pipelineprocessor.functions.dates;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import org.graylog.plugins.pipelineprocessor.EvaluationContext;
 import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction;
 import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
@@ -25,15 +27,20 @@
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.util.Locale;
+
 public abstract class TimezoneAwareFunction extends AbstractFunction {
 
-    public static final String TIMEZONE = "timezone";
+    private static final String TIMEZONE = "timezone";
+    private static final ImmutableMap UPPER_ZONE_MAP = Maps.uniqueIndex(
+            DateTimeZone.getAvailableIDs(),
+            input -> input != null ? input.toUpperCase(Locale.ENGLISH) : "UTC");
     private final ParameterDescriptor timeZoneParam;
 
-    public TimezoneAwareFunction() {
+    protected TimezoneAwareFunction() {
         timeZoneParam = ParameterDescriptor
                 .string(TIMEZONE, DateTimeZone.class)
-                .transform(id -> DateTimeZone.forID(id.toUpperCase()))
+                .transform(id -> DateTimeZone.forID(UPPER_ZONE_MAP.get(id)))
                 .optional()
                 .description("The timezone to apply to the date, defaults to UTC")
                 .build();
diff --git a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java
index 96bcb80eb45e..69db51a2e5c2 100644
--- a/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java
+++ b/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java
@@ -562,4 +562,18 @@ public void keyValueFailure() {
 
         assertThat(context.hasEvaluationErrors()).isTrue();
     }
+
+    @Test
+    public void timezones() {
+        final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH);
+        DateTimeUtils.setCurrentMillisProvider(clock);
+        try {
+            final Rule rule = parser.parseRule(ruleForTest(), true);
+            evaluateRule(rule);
+
+            assertThat(actionsTriggered.get()).isTrue();
+        } finally {
+            DateTimeUtils.setCurrentMillisSystem();
+        }
+    }
 }
diff --git a/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt
new file mode 100644
index 000000000000..370a8e627d56
--- /dev/null
+++ b/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt
@@ -0,0 +1,11 @@
+// now() is fixed, test uses different millisprovider!
+
+rule "timezones"
+when
+  now("CET") == now("UTC") &&
+  now("utc") == now("UTC") &&
+  now("Europe/Moscow") == now("europe/moscow") &&
+  now("europe/MoSCOw") == now("msk")
+then
+  trigger_test();
+end
\ No newline at end of file

From da4ac840c5ada8e0be5c3cef8b800d9f2ce9b9d4 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Tue, 6 Sep 2016 16:54:01 +0200
Subject: [PATCH 345/521] Display boolean values in pipeline simulator (#99)

Convert field values into strings for displaying.

Fixes #54
(cherry picked from commit 18a31177aedb47a14fd6db7c7400e6e9d5a3c1c0)
---
 src/web/simulator/SimulationChanges.jsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/web/simulator/SimulationChanges.jsx b/src/web/simulator/SimulationChanges.jsx
index 6ffb659e47b3..d96368d7666f 100644
--- a/src/web/simulator/SimulationChanges.jsx
+++ b/src/web/simulator/SimulationChanges.jsx
@@ -29,7 +29,7 @@ const SimulationChanges = React.createClass({
 
   _formatFieldValue(field, value, isAdded, isRemoved) {
     const className = (isAdded ? 'added-field' : (isRemoved ? 'removed-field' : ''));
-    return 
    {value}
    ;
+    return 
    {String(value)}
    ;
   },
 
   _formatAddedFields(originalMessage, processedMessage) {

From 699c3e350ac716bac1c428cb4811e1891ec8f843 Mon Sep 17 00:00:00 2001
From: Edmundo Alvarez 
Date: Wed, 28 Sep 2016 15:35:21 +0200
Subject: [PATCH 346/521] Simplify pipeline processor UI (#106)

Fixes #104

* Use pipelines overview page as entry point

* Add connected streams to pipeline overview page

* Move pipeline connection lists to its own component

* Add API method to connect streams to a pipeline

Migrating pipeline connections to the pipeline page also means that
we need to create connections in the reverse order: several streams
connected to a pipeline. I added a new API method for it, as I think the
old method can still be useful for someone using the API
programatically.

* Format PipelineConnectionsActions

* Set connections from pipeline details view

* Use nicer word break in timeline

* Cleanup after moving connections to pipeline view

* Add link to manage rules to rule details page

* Adapt pipeline simulator to recent changes

Now the stream selection needs to be done in the simulate page, as we
don't know before which stream to use.
---
 .../rest/PipelineConnectionsResource.java     |  72 +++++++-
 .../rest/PipelineReverseConnections.java      |  40 +++++
 src/web/index.jsx                             |   6 +-
 src/web/pipeline-connections/Connection.jsx   |  76 --------
 .../pipeline-connections/ConnectionForm.jsx   | 167 ------------------
 .../PipelineConnections.jsx                   | 102 -----------
 .../PipelineConnectionsPage.jsx               |  94 ----------
 src/web/pipelines/Pipeline.css                |  10 +-
 src/web/pipelines/Pipeline.jsx                |  58 +++++-
 .../PipelineConnectionsActions.js             |   5 +-
 src/web/pipelines/PipelineConnectionsForm.jsx | 104 +++++++++++
 src/web/pipelines/PipelineConnectionsList.jsx |  34 ++++
 .../PipelineConnectionsStore.js               |  40 ++++-
 src/web/pipelines/PipelineDetailsPage.jsx     |  39 +++-
 src/web/pipelines/PipelinesOverviewPage.jsx   |  10 +-
 .../pipelines/ProcessingTimelineComponent.css |   6 +
 .../pipelines/ProcessingTimelineComponent.jsx |  35 +++-
 src/web/rules/Rule.jsx                        |   6 +-
 src/web/rules/RulesPage.jsx                   |   4 -
 src/web/simulator/ProcessorSimulator.jsx      |  46 ++++-
 src/web/simulator/SimulatorPage.jsx           |  53 ++----
 21 files changed, 479 insertions(+), 528 deletions(-)
 create mode 100644 src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java
 delete mode 100644 src/web/pipeline-connections/Connection.jsx
 delete mode 100644 src/web/pipeline-connections/ConnectionForm.jsx
 delete mode 100644 src/web/pipeline-connections/PipelineConnections.jsx
 delete mode 100644 src/web/pipeline-connections/PipelineConnectionsPage.jsx
 rename src/web/{pipeline-connections => pipelines}/PipelineConnectionsActions.js (53%)
 create mode 100644 src/web/pipelines/PipelineConnectionsForm.jsx
 create mode 100644 src/web/pipelines/PipelineConnectionsList.jsx
 rename src/web/{pipeline-connections => pipelines}/PipelineConnectionsStore.js (58%)

diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java
index 8f86317c17ad..403bad89cd7d 100644
--- a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java
@@ -34,6 +34,8 @@
 import org.graylog2.shared.rest.resources.RestResource;
 import org.graylog2.shared.security.RestPermissions;
 import org.graylog2.streams.StreamService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import javax.validation.constraints.NotNull;
@@ -54,6 +56,7 @@
 @Produces(MediaType.APPLICATION_JSON)
 @RequiresAuthentication
 public class PipelineConnectionsResource extends RestResource implements PluginRestResource {
+    private static final Logger LOG = LoggerFactory.getLogger(PipelineConnectionsResource.class);
 
     private final PipelineStreamConnectionsService connectionsService;
     private final PipelineService pipelineService;
@@ -73,6 +76,7 @@ public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsS
 
     @ApiOperation(value = "Connect processing pipelines to a stream", notes = "")
     @POST
+    @Path("/to_stream")
     @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_EDIT)
     @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CONNECTION_UPDATE)
     public PipelineConnections connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineConnections connection) throws NotFoundException {
@@ -87,9 +91,65 @@ public PipelineConnections connectPipelines(@ApiParam(name = "Json body", requir
             checkPermission(PipelineRestPermissions.PIPELINE_READ, s);
             pipelineService.load(s);
         }
-        final PipelineConnections save = connectionsService.save(connection);
-        clusterBus.post(PipelineConnectionsChangedEvent.create(save.streamId(), save.pipelineIds()));
-        return save;
+        return savePipelineConnections(connection);
+    }
+
+    @ApiOperation(value = "Connect streams to a processing pipeline", notes = "")
+    @POST
+    @Path("/to_pipeline")
+    @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_EDIT)
+    @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CONNECTION_UPDATE)
+    public Set connectStreams(@ApiParam(name = "Json body", required = true) @NotNull PipelineReverseConnections connection) throws NotFoundException {
+        final String pipelineId = connection.pipelineId();
+        final Set updatedConnections = Sets.newHashSet();
+
+        // verify the pipeline exists
+        checkPermission(PipelineRestPermissions.PIPELINE_READ, pipelineId);
+        pipelineService.load(pipelineId);
+
+        // get all connections where the pipeline was present
+        final Set pipelineConnections = connectionsService.loadAll().stream()
+                .filter(p -> p.pipelineIds().contains(pipelineId))
+                .collect(Collectors.toSet());
+
+        // remove deleted pipeline connections
+        for (PipelineConnections pipelineConnection : pipelineConnections) {
+            if (!connection.streamIds().contains(pipelineConnection.streamId())) {
+                final Set pipelines = pipelineConnection.pipelineIds();
+                pipelines.remove(connection.pipelineId());
+                pipelineConnection.toBuilder().pipelineIds(pipelines).build();
+
+                updatedConnections.add(pipelineConnection);
+                savePipelineConnections(pipelineConnection);
+                LOG.debug("Deleted stream {} connection with pipeline {}", pipelineConnection.streamId(), pipelineId);
+            }
+        }
+
+        // update pipeline connections
+        for (String streamId : connection.streamIds()) {
+            // verify the stream exist
+            if (!streamId.equalsIgnoreCase("default")) {
+                checkPermission(RestPermissions.STREAMS_READ, streamId);
+                streamService.load(streamId);
+            }
+
+            PipelineConnections updatedConnection;
+            try {
+                updatedConnection = connectionsService.load(streamId);
+            } catch (NotFoundException e) {
+                updatedConnection = PipelineConnections.create(null, streamId, Sets.newHashSet());
+            }
+
+            final Set pipelines = updatedConnection.pipelineIds();
+            pipelines.add(pipelineId);
+            updatedConnection.toBuilder().pipelineIds(pipelines).build();
+
+            updatedConnections.add(updatedConnection);
+            savePipelineConnections(updatedConnection);
+            LOG.debug("Added stream {} connection with pipeline {}", streamId, pipelineId);
+        }
+
+        return updatedConnections;
     }
 
     @ApiOperation("Get pipeline connections for the given stream")
@@ -142,4 +202,10 @@ public Set getAll() throws NotFoundException {
         return filteredConnections;
     }
 
+    private PipelineConnections savePipelineConnections(PipelineConnections connection) {
+        final PipelineConnections save = connectionsService.save(connection);
+        clusterBus.post(PipelineConnectionsChangedEvent.create(save.streamId(), save.pipelineIds()));
+        return save;
+    }
+
 }
diff --git a/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java
new file mode 100644
index 000000000000..99921079a442
--- /dev/null
+++ b/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java
@@ -0,0 +1,40 @@
+/**
+ * This file is part of Graylog Pipeline Processor.
+ *
+ * Graylog Pipeline Processor is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog Pipeline Processor is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog Pipeline Processor.  If not, see .
+ */
+package org.graylog.plugins.pipelineprocessor.rest;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+
+import java.util.Set;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class PipelineReverseConnections {
+    @JsonProperty
+    public abstract String pipelineId();
+
+    @JsonProperty
+    public abstract Set streamIds();
+
+    @JsonCreator
+    public static PipelineReverseConnections create(@JsonProperty("pipeline_id") String pipelineId,
+                                                    @JsonProperty("stream_ids") Set streamIds) {
+        return new AutoValue_PipelineReverseConnections(pipelineId, streamIds);
+    }
+}
diff --git a/src/web/index.jsx b/src/web/index.jsx
index 6d8c41cacc65..abc2049d1b8c 100644
--- a/src/web/index.jsx
+++ b/src/web/index.jsx
@@ -5,18 +5,16 @@ import packageJson from '../../package.json';
 import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin';
 import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage';
 import PipelineDetailsPage from 'pipelines/PipelineDetailsPage';
-import PipelineConnectionsPage from 'pipeline-connections/PipelineConnectionsPage';
 import SimulatorPage from 'simulator/SimulatorPage';
 import RulesPage from 'rules/RulesPage';
 import RuleDetailsPage from 'rules/RuleDetailsPage';
 
 PluginStore.register(new PluginManifest(packageJson, {
   routes: [
-    { path: '/system/pipelines', component: PipelineConnectionsPage },
-    { path: '/system/pipelines/overview', component: PipelinesOverviewPage },
+    { path: '/system/pipelines', component: PipelinesOverviewPage },
     { path: '/system/pipelines/rules', component: RulesPage },
     { path: '/system/pipelines/rules/:ruleId', component: RuleDetailsPage },
-    { path: '/system/pipelines/simulate/:streamId', component: SimulatorPage },
+    { path: '/system/pipelines/simulate', component: SimulatorPage },
     { path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage },
   ],
 
diff --git a/src/web/pipeline-connections/Connection.jsx b/src/web/pipeline-connections/Connection.jsx
deleted file mode 100644
index c31b008afa54..000000000000
--- a/src/web/pipeline-connections/Connection.jsx
+++ /dev/null
@@ -1,76 +0,0 @@
-import React from 'react';
-import { Button, Col } from 'react-bootstrap';
-import { LinkContainer } from 'react-router-bootstrap';
-
-import { DataTable, EntityListItem, Timestamp } from 'components/common';
-import ConnectionForm from './ConnectionForm';
-
-import Routes from 'routing/Routes';
-
-const Connection = React.createClass({
-  propTypes: {
-    stream: React.PropTypes.object.isRequired,
-    pipelines: React.PropTypes.array.isRequired,
-    onUpdate: React.PropTypes.func.isRequired,
-  },
-
-  _pipelineHeaderFormatter(header) {
-    return {header};
-  },
-
-  _pipelineRowFormatter(pipeline) {
-    return (
-      
-        
-          {pipeline.title}
-        
-        {pipeline.description}
-        
-        
-      
-    );
-  },
-
-  _formatPipelines(pipelines) {
-    const headers = ['Title', 'Description', 'Created', 'Last modified'];
-
-    return (
-      
-    );
-  },
-
-  render() {
-    const actions = [
-      
-        
-      ,
-      ,
-    ];
-
-    const content = (
-      
-        {this._formatPipelines(this.props.pipelines)}
-      
-    );
-
-    return (
-      
-    );
-  },
-});
-
-export default Connection;
diff --git a/src/web/pipeline-connections/ConnectionForm.jsx b/src/web/pipeline-connections/ConnectionForm.jsx
deleted file mode 100644
index c15b05aab4b0..000000000000
--- a/src/web/pipeline-connections/ConnectionForm.jsx
+++ /dev/null
@@ -1,167 +0,0 @@
-import React, { PropTypes } from 'react';
-import Reflux from 'reflux';
-import { Input, Button } from 'react-bootstrap';
-import { LinkContainer } from 'react-router-bootstrap';
-
-import { Select, SelectableList } from 'components/common';
-import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm';
-import ObjectUtils from 'util/ObjectUtils';
-
-import PipelinesStore from 'pipelines/PipelinesStore';
-
-import Routes from 'routing/Routes';
-
-const ConnectionForm = React.createClass({
-  propTypes: {
-    stream: PropTypes.object,
-    connection: PropTypes.object,
-    streams: PropTypes.array,
-    create: PropTypes.bool,
-    save: PropTypes.func.isRequired,
-    validateConnection: PropTypes.func.isRequired,
-  },
-
-  mixins: [Reflux.connect(PipelinesStore)],
-
-  getDefaultProps() {
-    return {
-      streams: [],
-      connection: {
-        stream: undefined,
-        pipelines: [],
-      },
-    };
-  },
-
-  getInitialState() {
-    const connection = ObjectUtils.clone(this.props.connection);
-    return {
-      // when editing, take the connection that's been passed in
-      connection: {
-        stream: connection.stream,
-        pipelines: this._getFormattedOptions(connection.pipelines),
-      },
-    };
-  },
-
-  openModal() {
-    this.refs.modal.open();
-  },
-
-  _onStreamChange(newStream) {
-    const connection = ObjectUtils.clone(this.state.connection);
-    connection.stream = this.props.streams.filter(s => s.id === newStream)[0];
-    this.setState({ connection });
-  },
-
-  _onConnectionsChange(newRules) {
-    const connection = ObjectUtils.clone(this.state.connection);
-    connection.pipelines = newRules;
-    this.setState({ connection });
-  },
-
-  _closeModal() {
-    this.refs.modal.close();
-  },
-
-  _saved() {
-    this._closeModal();
-    if (this.props.create) {
-      this.setState(this.getInitialState());
-    }
-  },
-
-  _save() {
-    const connection = this.state.connection;
-    const filteredConnection = {};
-    filteredConnection.stream = connection.stream.id;
-    filteredConnection.pipelines = connection.pipelines.map(p => p.value);
-    this.props.save(filteredConnection, this._saved);
-  },
-
-  _getFormattedStreams(streams) {
-    if (!streams) {
-      return [];
-    }
-
-    return streams.map(s => {
-      return { value: s.id, label: s.title };
-    });
-  },
-
-  _getFormattedOptions(pipelines) {
-    if (!pipelines) {
-      return [];
-    }
-
-    return pipelines
-      .map(pipeline => {
-        return { value: pipeline.id, label: pipeline.title };
-      });
-  },
-
-  _getFilteredFormattedOptions(pipelines) {
-    return this._getFormattedOptions(pipelines)
-      // Remove already selected options
-      .filter(pipeline => !this.state.connection.pipelines.some(p => p.value === pipeline.value));
-  },
-
-  render() {
-    let triggerButtonContent;
-    if (this.props.create) {
-      triggerButtonContent = 'Add new connection';
-    } else {
-      triggerButtonContent = Edit connections;
-    }
-
-    let streamSelector;
-    if (this.props.create) {
-      const streamHelp = (
-        
-          Select the stream you want to connect pipelines to, or create one in the{' '}
-          Streams page.
-        
-      );
-      streamSelector = (
-        
-          
-              
-            
-          
-        
-      
-    );
-  },
-});
-
-export default ConnectionForm;
diff --git a/src/web/pipeline-connections/PipelineConnections.jsx b/src/web/pipeline-connections/PipelineConnections.jsx
deleted file mode 100644
index 99290cbc4ac8..000000000000
--- a/src/web/pipeline-connections/PipelineConnections.jsx
+++ /dev/null
@@ -1,102 +0,0 @@
-import React from 'react';
-import { Row, Col } from 'react-bootstrap';
-import naturalSort from 'javascript-natural-sort';
-import { LinkContainer } from 'react-router-bootstrap';
-
-import { EntityList, TypeAheadDataFilter } from 'components/common';
-import Connection from './Connection';
-import ConnectionForm from './ConnectionForm';
-import Routes from 'routing/Routes';
-
-const PipelineConnections = React.createClass({
-  propTypes: {
-    pipelines: React.PropTypes.array.isRequired,
-    streams: React.PropTypes.array.isRequired,
-    connections: React.PropTypes.array.isRequired,
-    onConnectionsChange: React.PropTypes.func.isRequired,
-  },
-
-  getInitialState() {
-    return {
-      filteredStreams: this.props.streams.slice(),
-    };
-  },
-
-  _updateFilteredStreams(filteredStreams) {
-    this.setState({ filteredStreams });
-  },
-
-  _updateConnection(newConnection, callback) {
-    this.props.onConnectionsChange(newConnection, callback);
-  },
-
-  _isConnectionWithPipelines(connection) {
-    return connection.pipeline_ids.length > 0;
-  },
-
-  render() {
-    const filteredStreamsMap = {};
-    this.state.filteredStreams.forEach(s => {
-      filteredStreamsMap[s.id] = s;
-    });
-
-    const formattedConnections = this.props.connections
-      .filter(c => this.state.filteredStreams.some(s => s.id === c.stream_id && this._isConnectionWithPipelines(c)))
-      .sort((c1, c2) => naturalSort(filteredStreamsMap[c1.stream_id].title, filteredStreamsMap[c2.stream_id].title))
-      .map(c => {
-        return (
-           s.id === c.stream_id)[0]}
-                      pipelines={this.props.pipelines.filter(p => c.pipeline_ids.indexOf(p.id) !== -1)}
-                      onUpdate={this._updateConnection} />
-        );
-      });
-
-    const filteredStreams = this.props.streams.filter(s => !this.props.connections.some(c => c.stream_id === s.id && this._isConnectionWithPipelines(c)));
-    const pipelinesNo = this.props.pipelines.length;
-    let noItemsText;
-    if (pipelinesNo === 0) {
-      noItemsText = (
-        
-          There are no pipeline connections. You can start by creating your{' '}
-          
-            pipeline rules
-          {' '}
-          and then putting them together in a{' '}
-          
-            pipeline
-          !
-        
-      );
-    } else {
-      noItemsText = (
-        
-          There are no pipeline connections. Click on "Add new connection" to connect your pipelines to a stream.
-        
-      );
-    }
-
-    return (
-      
    
-        
-          
-            
-          
-          
-            
    
-              
-            
    
-          
-            
-        
-      
    
-    );
-  },
-});
-
-export default PipelineConnections;
diff --git a/src/web/pipeline-connections/PipelineConnectionsPage.jsx b/src/web/pipeline-connections/PipelineConnectionsPage.jsx
deleted file mode 100644
index 237c80782bef..000000000000
--- a/src/web/pipeline-connections/PipelineConnectionsPage.jsx
+++ /dev/null
@@ -1,94 +0,0 @@
-import React from 'react';
-import Reflux from 'reflux';
-import { Button, Row, Col } from 'react-bootstrap';
-import { LinkContainer } from 'react-router-bootstrap';
-
-import DocumentationLink from 'components/support/DocumentationLink';
-import { PageHeader, Spinner } from 'components/common';
-import PipelineConnections from './PipelineConnections';
-
-import PipelinesActions from 'pipelines/PipelinesActions';
-import PipelinesStore from 'pipelines/PipelinesStore';
-import PipelineConnectionsActions from 'pipeline-connections/PipelineConnectionsActions';
-import PipelineConnectionsStore from 'pipeline-connections/PipelineConnectionsStore';
-
-import StoreProvider from 'injection/StoreProvider';
-const StreamsStore = StoreProvider.getStore('Streams');
-
-import DocsHelper from 'util/DocsHelper';
-import Routes from 'routing/Routes';
-
-const PipelineConnectionsPage = React.createClass({
-  mixins: [Reflux.connect(PipelinesStore), Reflux.connect(PipelineConnectionsStore)],
-
-  getInitialState() {
-    return {
-      streams: undefined,
-    };
-  },
-
-  componentDidMount() {
-    PipelinesActions.list();
-    PipelineConnectionsActions.list();
-
-    StreamsStore.listStreams().then((streams) => {
-      streams.push({
-        id: 'default',
-        title: 'Default',
-        description: 'Stream used by default for messages not matching another stream.',
-      });
-      this.setState({ streams });
-    });
-  },
-
-  _updateConnections(connections, callback) {
-    PipelineConnectionsActions.update(connections);
-    callback();
-  },
-
-  _isLoading() {
-    return !this.state.pipelines || !this.state.streams || !this.state.connections;
-  },
-
-  render() {
-    let content;
-    if (this._isLoading()) {
-      content = ;
-    } else {
-      content = (
-        
-      );
-    }
-    return (
-      
-        
-          
-            Pipelines let you process messages sent to Graylog. Here you can select which streams will be used{' '}
-            as input for the different pipelines you configure.
-          
-          
-            Read more about Graylog pipelines in the .
-          
-
-          
-            
-              
-            
-             
-            
-              
-            
-          
-        
-
-        
-          
-            {content}
-          
-        
-      );
-  },
-});
-
-export default PipelineConnectionsPage;
diff --git a/src/web/pipelines/Pipeline.css b/src/web/pipelines/Pipeline.css
index 282078ce9fc6..faeb0b0e28b4 100644
--- a/src/web/pipelines/Pipeline.css
+++ b/src/web/pipelines/Pipeline.css
@@ -13,4 +13,12 @@ dl.pipeline-dl > dt:after {
 
 dl.pipeline-dl > dd {
     margin-left: 100px;
-}
\ No newline at end of file
+}
+
+.row-margin-top {
+    margin-top: 10px;
+}
+
+.description-margin-top {
+    margin-top: 5px;
+}
diff --git a/src/web/pipelines/Pipeline.jsx b/src/web/pipelines/Pipeline.jsx
index 5491013cb285..1c471daf6c54 100644
--- a/src/web/pipelines/Pipeline.jsx
+++ b/src/web/pipelines/Pipeline.jsx
@@ -1,16 +1,24 @@
-import React, { PropTypes } from 'react';
-import { Row, Col } from 'react-bootstrap';
+import React from 'react';
+import { Button, Row, Col } from 'react-bootstrap';
+import { LinkContainer } from 'react-router-bootstrap';
 
-import { EntityList } from 'components/common';
+import { EntityList, Pluralize } from 'components/common';
 import Stage from './Stage';
 import StageForm from './StageForm';
 import PipelineDetails from './PipelineDetails';
+import PipelineConnectionsForm from './PipelineConnectionsForm';
+import PipelineConnectionsList from './PipelineConnectionsList';
+
+import Routes from 'routing/Routes';
 
 const Pipeline = React.createClass({
   propTypes: {
-    pipeline: PropTypes.object.isRequired,
-    onStagesChange: PropTypes.func.isRequired,
-    onPipelineChange: PropTypes.func.isRequired,
+    pipeline: React.PropTypes.object.isRequired,
+    connections: React.PropTypes.array.isRequired,
+    streams: React.PropTypes.array.isRequired,
+    onConnectionsChange: React.PropTypes.func.isRequired,
+    onStagesChange: React.PropTypes.func.isRequired,
+    onPipelineChange: React.PropTypes.func.isRequired,
   },
 
   componentDidMount() {
@@ -46,6 +54,18 @@ const Pipeline = React.createClass({
     };
   },
 
+  _formatConnectedStreams(streams) {
+    const formattedStreams = streams.map(s => `"${s.title}"`);
+    const streamList = streams.length > 1 ? [formattedStreams.slice(0, -1).join(', '), formattedStreams.slice(-1)].join(' and ') : formattedStreams[0];
+    return (
+      
+        This pipeline is processing messages from the{' '}
+        {' '}
+        {streamList}.
+      
+    );
+  },
+
   _formatStage(stage, maxStage) {
     return (
        Math.max(max, currentStage.stage), -Infinity);
     const formattedStages = pipeline.stages
       .sort((s1, s2) => s1.stage - s2.stage)
@@ -62,14 +83,33 @@ const Pipeline = React.createClass({
 
     return (
       
    
-        
-        
+        
+        
+          
+            
    
+              
+                
+              
+               
+              
+            
    
+            
    Pipeline connections
    
+            
    
+              
+            
    
+            
    
+          
+            
+        
           
             
    
               
             
    
             
    Pipeline Stages
    
-            
    
+            
    
               Stages are groups of conditions and actions which need to run in order, and provide the necessary{' '}
               control flow to decide whether or not to run the rest of a pipeline.
             
    
diff --git a/src/web/pipeline-connections/PipelineConnectionsActions.js b/src/web/pipelines/PipelineConnectionsActions.js
similarity index 53%
rename from src/web/pipeline-connections/PipelineConnectionsActions.js
rename to src/web/pipelines/PipelineConnectionsActions.js
index 9bd43ed44269..0d7de6d6a579 100644
--- a/src/web/pipeline-connections/PipelineConnectionsActions.js
+++ b/src/web/pipelines/PipelineConnectionsActions.js
@@ -1,8 +1,9 @@
 import Reflux from 'reflux';
 
 const PipelineConnectionsActions = Reflux.createActions({
-  'list': {asyncResult: true},
-  'update': {asyncResult: true},
+  list: { asyncResult: true },
+  connectToStream: { asyncResult: true },
+  connectToPipeline: { asyncResult: true },
 });
 
 export default PipelineConnectionsActions;
diff --git a/src/web/pipelines/PipelineConnectionsForm.jsx b/src/web/pipelines/PipelineConnectionsForm.jsx
new file mode 100644
index 000000000000..0de9c46493d1
--- /dev/null
+++ b/src/web/pipelines/PipelineConnectionsForm.jsx
@@ -0,0 +1,104 @@
+import React, { PropTypes } from 'react';
+import { Input, Button } from 'react-bootstrap';
+import { LinkContainer } from 'react-router-bootstrap';
+import naturalSort from 'javascript-natural-sort';
+
+import { SelectableList } from 'components/common';
+import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm';
+
+import Routes from 'routing/Routes';
+
+const PipelineConnectionsForm = React.createClass({
+  propTypes: {
+    pipeline: PropTypes.object.isRequired,
+    connections: PropTypes.array.isRequired,
+    streams: PropTypes.array.isRequired,
+    save: PropTypes.func.isRequired,
+  },
+
+  getInitialState() {
+    return {
+      connectedStreams: this._getFormattedStreams(this._getConnectedStreams(this.props.pipeline, this.props.connections, this.props.streams)),
+    };
+  },
+
+  openModal() {
+    this.refs.modal.open();
+  },
+
+  _onStreamsChange(newStreams) {
+    this.setState({ connectedStreams: newStreams.sort((s1, s2) => naturalSort(s1.label, s2.label)) });
+  },
+
+  _closeModal() {
+    this.refs.modal.close();
+  },
+
+  _resetForm() {
+    this.setState(this.getInitialState());
+  },
+
+  _saved() {
+    this._closeModal();
+  },
+
+  _save() {
+    const streamIds = this.state.connectedStreams.map(cs => cs.value);
+    const newConnection = {
+      pipeline: this.props.pipeline.id,
+      streams: streamIds,
+    };
+
+    this.props.save(newConnection, this._saved);
+  },
+
+  _getConnectedStreams(pipeline, connections, streams) {
+    return connections
+      .filter(c => c.pipeline_ids && c.pipeline_ids.includes(pipeline.id)) // Get connections for this pipeline
+      .filter(c => streams.some(s => s.id === c.stream_id)) // Filter out deleted streams
+      .map(c => this.props.streams.find(s => s.id === c.stream_id));
+  },
+
+  _getFormattedStreams(streams) {
+    return streams
+      .map(s => {
+        return { value: s.id, label: s.title };
+      })
+      .sort((s1, s2) => naturalSort(s1.label, s2.label));
+  },
+
+  _getFilteredStreams(streams) {
+    return streams.filter(s => !this.state.connectedStreams.some(cs => cs.value.toLowerCase() === s.id.toLowerCase()));
+  },
+
+  render() {
+    const streamsHelp = (
+      
+        Select the streams you want to connect this pipeline, or create one in the{' '}
+        Streams page.
+      
+    );
+
+    return (
+      
+        
+        Edit connections for {this.props.pipeline.title}}
+                            onSubmitForm={this._save} onCancel={this._resetForm} submitButtonText="Save">
+          
    
+            
+              
+            
+          
    
+        
+      
+    );
+  },
+});
+
+export default PipelineConnectionsForm;
diff --git a/src/web/pipelines/PipelineConnectionsList.jsx b/src/web/pipelines/PipelineConnectionsList.jsx
new file mode 100644
index 000000000000..3195dfd531f8
--- /dev/null
+++ b/src/web/pipelines/PipelineConnectionsList.jsx
@@ -0,0 +1,34 @@
+import React from 'react';
+import naturalSort from 'javascript-natural-sort';
+
+const PipelineConnectionsList = React.createClass({
+  propTypes: {
+    pipeline: React.PropTypes.object.isRequired,
+    connections: React.PropTypes.array.isRequired,
+    streams: React.PropTypes.array.isRequired,
+    streamsFormatter: React.PropTypes.func.isRequired,
+    noConnectionsMessage: React.PropTypes.any,
+  },
+
+  getDefaultProps() {
+    return {
+      noConnectionsMessage: 'Pipeline not connected to any streams',
+    };
+  },
+
+  render() {
+    const streamsUsingPipeline = this.props.connections
+      .filter(c => c.pipeline_ids && c.pipeline_ids.includes(this.props.pipeline.id)) // Get connections for this pipeline
+      .filter(c => this.props.streams.some(s => s.id === c.stream_id)) // Filter out deleted streams
+      .map(c => this.props.streams.find(s => s.id === c.stream_id))
+      .sort((s1, s2) => naturalSort(s1.title, s2.title));
+
+    return (
+      
+        {streamsUsingPipeline.length === 0 ? this.props.noConnectionsMessage : this.props.streamsFormatter(streamsUsingPipeline)}
+      
+    );
+  },
+});
+
+export default PipelineConnectionsList;
diff --git a/src/web/pipeline-connections/PipelineConnectionsStore.js b/src/web/pipelines/PipelineConnectionsStore.js
similarity index 58%
rename from src/web/pipeline-connections/PipelineConnectionsStore.js
rename to src/web/pipelines/PipelineConnectionsStore.js
index ac4163ce49f5..01377bd619f9 100644
--- a/src/web/pipeline-connections/PipelineConnectionsStore.js
+++ b/src/web/pipelines/PipelineConnectionsStore.js
@@ -26,16 +26,12 @@ const PipelineConnectionsStore = Reflux.createStore({
     const promise = fetch('GET', url);
     promise.then((response) => {
       this.connections = response;
-      this.trigger({connections: response});
+      this.trigger({ connections: response });
     }, failCallback);
   },
 
-  update(connection) {
-    const failCallback = (error) => {
-      UserNotification.error('Updating pipeline connections failed with status: ' + error.message,
-        'Could not update pipeline connections');
-    };
-    const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/connections');
+  connectToStream(connection) {
+    const url = URLUtils.qualifyUrl(urlPrefix + '/system/pipelines/connections/to_stream');
     const updatedConnection = {
       stream_id: connection.stream,
       pipeline_ids: connection.pipelines,
@@ -52,7 +48,35 @@ const PipelineConnectionsStore = Reflux.createStore({
         this.trigger({connections: this.connections});
         UserNotification.success(`Pipeline connections updated successfully`);
       },
-      failCallback);
+      this._failUpdateCallback);
+  },
+
+  connectToPipeline(reverseConnection) {
+    const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/connections/to_pipeline`);
+    const updatedConnection = {
+      pipeline_id: reverseConnection.pipeline,
+      stream_ids: reverseConnection.streams,
+    };
+    const promise = fetch('POST', url, updatedConnection);
+    promise.then(
+      response => {
+        response.forEach(connection => {
+          if (this.connections.filter(c => c.stream_id === connection.stream_id)[0]) {
+            this.connections = this.connections.map(c => (c.stream_id === connection.stream_id ? connection : c));
+          } else {
+            this.connections.push(connection);
+          }
+        });
+
+        this.trigger({ connections: this.connections });
+        UserNotification.success('Pipeline connections updated successfully');
+      },
+      this._failUpdateCallback);
+  },
+
+  _failUpdateCallback(error) {
+    UserNotification.error(`Updating pipeline connections failed with status: ${error.message}`,
+      'Could not update pipeline connections');
   },
 });
 
diff --git a/src/web/pipelines/PipelineDetailsPage.jsx b/src/web/pipelines/PipelineDetailsPage.jsx
index 562bccf56322..61819505deaa 100644
--- a/src/web/pipelines/PipelineDetailsPage.jsx
+++ b/src/web/pipelines/PipelineDetailsPage.jsx
@@ -12,6 +12,11 @@ import ObjectUtils from 'util/ObjectUtils';
 import PipelinesActions from 'pipelines/PipelinesActions';
 import PipelinesStore from 'pipelines/PipelinesStore';
 import RulesStore from 'rules/RulesStore';
+import PipelineConnectionsActions from './PipelineConnectionsActions';
+import PipelineConnectionsStore from './PipelineConnectionsStore';
+
+import StoreProvider from 'injection/StoreProvider';
+const StreamsStore = StoreProvider.getStore('Streams');
 
 import Routes from 'routing/Routes';
 
@@ -19,19 +24,36 @@ function filterPipeline(state) {
   return state.pipelines ? state.pipelines.filter(p => p.id === this.props.params.pipelineId)[0] : undefined;
 }
 
+function filterConnections(state) {
+  if (!state.connections) {
+    return undefined;
+  }
+  return state.connections.filter(c => c.pipeline_ids && c.pipeline_ids.includes(this.props.params.pipelineId));
+}
+
 const PipelineDetailsPage = React.createClass({
   propTypes: {
     params: React.PropTypes.object.isRequired,
     history: React.PropTypes.object.isRequired,
   },
 
-  mixins: [Reflux.connectFilter(PipelinesStore, 'pipeline', filterPipeline)],
+  mixins: [Reflux.connectFilter(PipelinesStore, 'pipeline', filterPipeline), Reflux.connectFilter(PipelineConnectionsStore, 'connections', filterConnections)],
 
   componentDidMount() {
     if (!this._isNewPipeline(this.props.params.pipelineId)) {
       PipelinesActions.get(this.props.params.pipelineId);
     }
     RulesStore.list();
+    PipelineConnectionsActions.list();
+
+    StreamsStore.listStreams().then((streams) => {
+      streams.push({
+        id: 'default',
+        title: 'Default',
+        description: 'Stream used by default for messages not matching another stream.',
+      });
+      this.setState({ streams });
+    });
   },
 
   componentWillReceiveProps(nextProps) {
@@ -40,6 +62,11 @@ const PipelineDetailsPage = React.createClass({
     }
   },
 
+  _onConnectionsChange(updatedConnections, callback) {
+    PipelineConnectionsActions.connectToPipeline(updatedConnections);
+    callback();
+  },
+
   _onStagesChange(newStages, callback) {
     const newPipeline = ObjectUtils.clone(this.state.pipeline);
     newPipeline.stages = newStages;
@@ -69,7 +96,7 @@ const PipelineDetailsPage = React.createClass({
   },
 
   _isLoading() {
-    return !this._isNewPipeline(this.props.params.pipelineId) && !this.state.pipeline;
+    return !this._isNewPipeline(this.props.params.pipelineId) && !this.state.pipeline || !this.state.connections || !this.state.streams;
   },
 
   render() {
@@ -88,7 +115,11 @@ const PipelineDetailsPage = React.createClass({
     if (this._isNewPipeline(this.props.params.pipelineId)) {
       content = ;
     } else {
-      content = ;
+      content = (
+        
+      );
     }
 
     return (
@@ -104,7 +135,7 @@ const PipelineDetailsPage = React.createClass({
 
           
             
-              
+              
             
             {' '}
             
diff --git a/src/web/pipelines/PipelinesOverviewPage.jsx b/src/web/pipelines/PipelinesOverviewPage.jsx
index 8fb03744f00c..577db1c47771 100644
--- a/src/web/pipelines/PipelinesOverviewPage.jsx
+++ b/src/web/pipelines/PipelinesOverviewPage.jsx
@@ -3,9 +3,11 @@ import { Row, Col, Button } from 'react-bootstrap';
 import { LinkContainer } from 'react-router-bootstrap';
 
 import { PageHeader } from 'components/common';
+import DocumentationLink from 'components/support/DocumentationLink';
 import ProcessingTimelineComponent from './ProcessingTimelineComponent';
 
 import Routes from 'routing/Routes';
+import DocsHelper from 'util/DocsHelper';
 
 const PipelinesOverviewPage = React.createClass({
   render() {
@@ -17,14 +19,14 @@ const PipelinesOverviewPage = React.createClass({
             rules are evaluated and applied. Messages can go through one or more stages.
           
           
-            Click on a pipeline name to view more about it and edit its stages.
+            Read more about Graylog pipelines in the .
           
 
           
-            
-              
+            
+              
             
-            {' '}
+             
             
               
             
diff --git a/src/web/pipelines/ProcessingTimelineComponent.css b/src/web/pipelines/ProcessingTimelineComponent.css
index 114d637d5970..592b4d322aa1 100644
--- a/src/web/pipelines/ProcessingTimelineComponent.css
+++ b/src/web/pipelines/ProcessingTimelineComponent.css
@@ -23,4 +23,10 @@
     text-overflow: ellipsis;
     white-space: nowrap;
     width: 300px;
+}
+
+.stream-list {
+    max-width: 150px;
+    width: 150px;
+    word-wrap: break-word;
 }
\ No newline at end of file
diff --git a/src/web/pipelines/ProcessingTimelineComponent.jsx b/src/web/pipelines/ProcessingTimelineComponent.jsx
index 6f41e4b41bcc..47bb84d9d88d 100644
--- a/src/web/pipelines/ProcessingTimelineComponent.jsx
+++ b/src/web/pipelines/ProcessingTimelineComponent.jsx
@@ -6,18 +6,34 @@ import naturalSort from 'javascript-natural-sort';
 
 import { DataTable, Spinner } from 'components/common';
 import { MetricContainer, CounterRate } from 'components/metrics';
+import PipelineConnectionsList from './PipelineConnectionsList';
 
 import PipelinesActions from 'pipelines/PipelinesActions';
 import PipelinesStore from 'pipelines/PipelinesStore';
+import PipelineConnectionsActions from './PipelineConnectionsActions';
+import PipelineConnectionsStore from './PipelineConnectionsStore';
+
+import StoreProvider from 'injection/StoreProvider';
+const StreamsStore = StoreProvider.getStore('Streams');
 
 import Routes from 'routing/Routes';
 
 const ProcessingTimelineComponent = React.createClass({
-  mixins: [Reflux.connect(PipelinesStore)],
+  mixins: [Reflux.connect(PipelinesStore), Reflux.connect(PipelineConnectionsStore)],
 
   componentDidMount() {
     this.style.use();
     PipelinesActions.list();
+    PipelineConnectionsActions.list();
+
+    StreamsStore.listStreams().then((streams) => {
+      streams.push({
+        id: 'default',
+        title: 'Default',
+        description: 'Stream used by default for messages not matching another stream.',
+      });
+      this.setState({ streams });
+    });
   },
 
   componentWillUnmount() {
@@ -45,6 +61,10 @@ const ProcessingTimelineComponent = React.createClass({
     return {header};
   },
 
+  _formatConnectedStreams(streams) {
+    return streams.map(s => s.title).join(', ');
+  },
+
   _formatStages(pipeline, stages) {
     const formattedStages = [];
     const stageNumbers = stages.map(stage => stage.stage);
@@ -75,6 +95,11 @@ const ProcessingTimelineComponent = React.createClass({
             
           
         
+        
+          Not connected} />
+        
         {this._formatStages(pipeline, pipeline.stages)}
         
           
@@ -95,8 +120,12 @@ const ProcessingTimelineComponent = React.createClass({
     };
   },
 
+  _isLoading() {
+    return !this.state.pipelines || !this.state.streams || !this.state.connections;
+  },
+
   render() {
-    if (!this.state.pipelines) {
+    if (this._isLoading()) {
       return ;
     }
 
@@ -121,7 +150,7 @@ const ProcessingTimelineComponent = React.createClass({
 
     this.usedStages = this._calculateUsedStages(this.state.pipelines);
 
-    const headers = ['Pipeline', 'Processing Timeline', 'Actions'];
+    const headers = ['Pipeline', 'Connected to Streams', 'Processing Timeline', 'Actions'];
     return (
       
    
         {addNewPipelineButton}
diff --git a/src/web/rules/Rule.jsx b/src/web/rules/Rule.jsx
index b5be36527544..3cd08bb62c2a 100644
--- a/src/web/rules/Rule.jsx
+++ b/src/web/rules/Rule.jsx
@@ -46,11 +46,11 @@ const Rule = React.createClass({
 
           
             
-              
+              
             
              
-            
-              
+            
+              
             
           
         
diff --git a/src/web/rules/RulesPage.jsx b/src/web/rules/RulesPage.jsx
index 913539009b19..88bccf6694c8 100644
--- a/src/web/rules/RulesPage.jsx
+++ b/src/web/rules/RulesPage.jsx
@@ -39,10 +39,6 @@ const RulesPage = React.createClass({
 
           
             
-              
-            
-             
-            
               
             
           
diff --git a/src/web/simulator/ProcessorSimulator.jsx b/src/web/simulator/ProcessorSimulator.jsx
index ba9e1e7dd0d0..a6f88c290baa 100644
--- a/src/web/simulator/ProcessorSimulator.jsx
+++ b/src/web/simulator/ProcessorSimulator.jsx
@@ -1,6 +1,8 @@
 import React from 'react';
-import { Col, Row } from 'react-bootstrap';
+import { Col, Input, Row } from 'react-bootstrap';
+import naturalSort from 'javascript-natural-sort';
 
+import { Select } from 'components/common';
 import RawMessageLoader from 'components/messageloaders/RawMessageLoader';
 import SimulationResults from './SimulationResults';
 
@@ -10,12 +12,13 @@ import SimulatorStore from './SimulatorStore';
 
 const ProcessorSimulator = React.createClass({
   propTypes: {
-    stream: React.PropTypes.object.isRequired,
+    streams: React.PropTypes.array.isRequired,
   },
 
   getInitialState() {
     return {
       message: undefined,
+      stream: this.props.streams.find(s => s.id.toLowerCase() === 'default'),
       simulation: undefined,
       loading: false,
       error: undefined,
@@ -26,7 +29,7 @@ const ProcessorSimulator = React.createClass({
     this.setState({ message: message, simulation: undefined, loading: true, error: undefined });
 
     SimulatorActions.simulate
-      .triggerPromise(this.props.stream, message.fields, options.inputId)
+      .triggerPromise(this.state.stream, message.fields, options.inputId)
       .then(
         response => {
           this.setState({ simulation: response, loading: false });
@@ -37,7 +40,30 @@ const ProcessorSimulator = React.createClass({
       );
   },
 
+  _getFormattedStreams(streams) {
+    if (!streams) {
+      return [];
+    }
+
+    return streams
+      .map(s => {
+        return { value: s.id, label: s.title };
+      })
+      .sort((s1, s2) => naturalSort(s1.label, s2.label));
+  },
+
+  _onStreamSelect(selectedStream) {
+    const stream = this.props.streams.find(s => s.id.toLowerCase() === selectedStream.toLowerCase());
+    this.setState({ stream: stream });
+  },
+
   render() {
+    const streamHelp = (
+      
+        Select a stream to use during simulation, the Default stream is used by default.
+      
+    );
+
     return (
       
    
         
@@ -45,12 +71,22 @@ const ProcessorSimulator = React.createClass({
             
    Load a message
    
             
    
               Build an example message that will be used in the simulation.{' '}
-              No real messages stored in Graylog will be changed. All actions are purely simulated on the temporary input you provide below.
+              No real messages stored in Graylog will be changed. All actions are purely simulated on the
+                temporary input you provide below.
             
    
+            
+              
+                
+