diff --git a/graylog-project-parent/pom.xml b/graylog-project-parent/pom.xml index 9bd81ea8dc14..a065e1191876 100644 --- a/graylog-project-parent/pom.xml +++ b/graylog-project-parent/pom.xml @@ -495,6 +495,21 @@ validation-api ${validation-api.version} + + org.antlr + antlr4-runtime + ${antlr.version} + + + org.jooq + jool + ${jool.version} + + + com.squareup + javapoet + ${javapoet.version} + diff --git a/graylog2-server/pom.xml b/graylog2-server/pom.xml index a59eeb55641d..b41b36f69a87 100644 --- a/graylog2-server/pom.xml +++ b/graylog2-server/pom.xml @@ -485,6 +485,18 @@ javax.validation validation-api + + org.antlr + antlr4-runtime + + + org.jooq + jool + + + com.squareup + javapoet + nl.jqno.equalsverifier @@ -686,6 +698,18 @@ com.mycila license-maven-plugin + + org.antlr + antlr4-maven-plugin + + + antlr + + antlr4 + + + + diff --git a/graylog2-server/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 b/graylog2-server/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 new file mode 100644 index 000000000000..cb20c6717f6c --- /dev/null +++ b/graylog2-server/src/main/antlr4/org/graylog/plugins/pipelineprocessor/parser/RuleLang.g4 @@ -0,0 +1,435 @@ +/* + Parts of the grammar are derived from the Java.g4 grammar at https://github.com/antlr/grammars-v4/blob/master/java/Java.g4 + Those parts are under the following license: + + [The "BSD licence"] + Copyright (c) 2013 Terence Parr, Sam Harwell + All rights reserved. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +grammar RuleLang; + +@header { +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +} + +file + : ( ruleDeclaration + | pipelineDeclaration + )+ + EOF + ; + +pipelineDecls + : pipelineDeclaration+ EOF + ; + +pipeline + : pipelineDeclaration EOF + ; + +pipelineDeclaration + : Pipeline name=String + stageDeclaration+ + End + ; + +stageDeclaration + : Stage stage=Integer Match modifier=(All|Either) + ruleRef* + ; + +ruleRef + : Rule name=String ';'? + ; + +ruleDecls + : ruleDeclaration+ EOF + ; + +ruleDeclaration + : Rule name=String + When condition=expression + (Then actions=statement*)? + End + ; + +expression + : '(' expression ')' # ParenExpr + | literal # LiteralPrimary + | functionCall # Func + | Identifier # Identifier + | '[' (expression (',' expression)*)* ']' # ArrayLiteralExpr + | '{' (propAssignment (',' propAssignment)*)* '}' # MapLiteralExpr + | MessageRef '.' field=expression # MessageRef + | fieldSet=expression '.' field=expression # Nested + | array=expression '[' index=expression ']' # IndexedAccess + | sign=('+'|'-') expr=expression # SignedExpression + | Not expression # Not + | left=expression mult=('*'|'/'|'%') right=expression # Multiplication + | left=expression add=('+'|'-') right=expression # Addition + | left=expression comparison=('<=' | '>=' | '>' | '<') right=expression # Comparison + | left=expression equality=('==' | '!=') right=expression # Equality + | left=expression and=And right=expression # And + | left=expression or=Or right=expression # Or + ; + +propAssignment + : Identifier ':' expression + ; + +statement + : functionCall ';' # FuncStmt + | Let varName=Identifier '=' expression ';' # VarAssignStmt + | ';' # EmptyStmt + ; + +functionCall + : funcName=Identifier '(' arguments? ')' + ; + +arguments + : propAssignment (',' propAssignment)* # NamedArgs + | expression (',' expression)* # PositionalArgs + ; + +literal + : Integer # Integer + | Float # Float + | Char # Char + | String # String + | Boolean # Boolean + ; + +// Lexer + +All : A L L; +Either: E I T H E R; +And : A N D | '&&'; +Or: O R | '||'; +Not: N O T | '!'; +Pipeline: P I P E L I N E; +Rule: R U L E; +During: D U R I N G; +Stage: S T A G E; +When: W H E N; +Then: T H E N; +End: E N D; +Let: L E T; +Match: M A T C H; +MessageRef: '$message'; + +Boolean + : 'true'|'false' + ; + +// Integer literals + +Integer + : DecimalIntegerLiteral + | HexIntegerLiteral + | OctalIntegerLiteral + | BinaryIntegerLiteral + ; + +fragment +DecimalIntegerLiteral + : Sign? DecimalNumeral IntegerTypeSuffix? + ; + +fragment +HexIntegerLiteral + : Sign? HexNumeral IntegerTypeSuffix? + ; + +fragment +OctalIntegerLiteral + : Sign? OctalNumeral IntegerTypeSuffix? + ; + +fragment +BinaryIntegerLiteral + : Sign? BinaryNumeral IntegerTypeSuffix? + ; + +fragment +IntegerTypeSuffix + : [lL] + ; + +fragment +DecimalNumeral + : '0' + | NonZeroDigit (Digits? | Underscores Digits) + ; + +fragment +Digits + : Digit (DigitOrUnderscore* Digit)? + ; + +fragment +Digit + : '0' + | NonZeroDigit + ; + +fragment +NonZeroDigit + : [1-9] + ; + +fragment +DigitOrUnderscore + : Digit + | '_' + ; + +fragment +Underscores + : '_'+ + ; + +fragment +HexNumeral + : '0' [xX] HexDigits + ; + +fragment +HexDigits + : HexDigit (HexDigitOrUnderscore* HexDigit)? + ; + +fragment +HexDigit + : [0-9a-fA-F] + ; + +fragment +HexDigitOrUnderscore + : HexDigit + | '_' + ; + +fragment +OctalNumeral + : '0' Underscores? OctalDigits + ; + +fragment +OctalDigits + : OctalDigit (OctalDigitOrUnderscore* OctalDigit)? + ; + +fragment +OctalDigit + : [0-7] + ; + +fragment +OctalDigitOrUnderscore + : OctalDigit + | '_' + ; + +fragment +BinaryNumeral + : '0' [bB] BinaryDigits + ; + +fragment +BinaryDigits + : BinaryDigit (BinaryDigitOrUnderscore* BinaryDigit)? + ; + +fragment +BinaryDigit + : [01] + ; + +fragment +BinaryDigitOrUnderscore + : BinaryDigit + | '_' + ; + +// Floats +Float + : Sign? DecimalFloatingPointLiteral + | Sign? HexadecimalFloatingPointLiteral + ; + +fragment +DecimalFloatingPointLiteral + : Digits '.' Digits? ExponentPart? FloatTypeSuffix? + | '.' Digits ExponentPart? FloatTypeSuffix? + | Digits ExponentPart FloatTypeSuffix? + | Digits FloatTypeSuffix + ; + +fragment +ExponentPart + : ExponentIndicator SignedInteger + ; + +fragment +ExponentIndicator + : [eE] + ; + +fragment +SignedInteger + : Sign? Digits + ; + +fragment +Sign + : [+-] + ; + +fragment +FloatTypeSuffix + : [fFdD] + ; + +fragment +HexadecimalFloatingPointLiteral + : HexSignificand BinaryExponent FloatTypeSuffix? + ; + +fragment +HexSignificand + : HexNumeral '.'? + | '0' [xX] HexDigits? '.' HexDigits + ; + +fragment +BinaryExponent + : BinaryExponentIndicator SignedInteger + ; + +fragment +BinaryExponentIndicator + : [pP] + ; + +// Char + +Char + : '\'' SingleCharacter '\'' + | '\'' EscapeSequence '\'' + ; + +fragment +SingleCharacter + : ~['\\] + ; + +// String literals +String + : '"' StringCharacters? '"' + ; +fragment +StringCharacters + : StringCharacter+ + ; +fragment +StringCharacter + : ~["\\] + | EscapeSequence + ; +// §3.10.6 Escape Sequences for Character and String Literals +fragment +EscapeSequence + : '\\' [btnfr"'\\] + | OctalEscape + | UnicodeEscape + ; + +fragment +OctalEscape + : '\\' OctalDigit + | '\\' OctalDigit OctalDigit + | '\\' ZeroToThree OctalDigit OctalDigit + ; + +fragment +UnicodeEscape + : '\\' 'u' HexDigit HexDigit HexDigit HexDigit + ; + +fragment +ZeroToThree + : [0-3] + ; + +Identifier + : [a-zA-Z_] [a-zA-Z_0-9]* + | '`' ~['`']+ '`' + ; + + +// +// Whitespace and comments +// + +WS : [ \t\r\n\u000C]+ -> skip + ; + +COMMENT + : '/*' .*? '*/' -> skip + ; + +LINE_COMMENT + : '//' ~[\r\n]* -> skip + ; + + +// to support case insensitive keywords + +fragment A : [aA]; +fragment B : [bB]; +fragment C : [cC]; +fragment D : [dD]; +fragment E : [eE]; +fragment F : [fF]; +fragment G : [gG]; +fragment H : [hH]; +fragment I : [iI]; +fragment J : [jJ]; +fragment K : [kK]; +fragment L : [lL]; +fragment M : [mM]; +fragment N : [nN]; +fragment O : [oO]; +fragment P : [pP]; +fragment Q : [qQ]; +fragment R : [rR]; +fragment S : [sS]; +fragment T : [tT]; +fragment U : [uU]; +fragment V : [vV]; +fragment W : [wW]; +fragment X : [xX]; +fragment Y : [yY]; +fragment Z : [zZ]; diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java new file mode 100644 index 000000000000..d24dbd3a18ae --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java @@ -0,0 +1,170 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog2.plugin.EmptyMessages; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.MessageCollection; +import org.graylog2.plugin.Messages; +import org.joda.time.DateTime; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class EvaluationContext { + + private static final EvaluationContext EMPTY_CONTEXT = new EvaluationContext() { + @Override + public void addCreatedMessage(Message newMessage) { + // cannot add messages to empty context + } + + @Override + public void define(String identifier, Class type, Object value) { + // cannot define any variables in empty context + } + }; + + @Nonnull + private final Message message; + @Nullable + private Map ruleVars; + @Nullable + private List createdMessages; + @Nullable + private List evalErrors; + + private EvaluationContext() { + this(new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"))); // first Graylog release + } + + public EvaluationContext(@Nonnull Message message) { + this.message = message; + } + + public void define(String identifier, Class type, Object value) { + if (ruleVars == null) { + ruleVars = Maps.newHashMap(); + } + ruleVars.put(identifier, new TypedValue(type, value)); + } + + public Message currentMessage() { + return message; + } + + public TypedValue get(String identifier) { + if (ruleVars == null) { + throw new IllegalStateException("Use of undeclared variable " + identifier); + } + return ruleVars.get(identifier); + } + + public Messages createdMessages() { + if (createdMessages == null) { + return new EmptyMessages(); + } + return new MessageCollection(createdMessages); + } + + public void addCreatedMessage(Message newMessage) { + if (createdMessages == null) { + createdMessages = Lists.newArrayList(); + } + createdMessages.add(newMessage); + } + + public void clearCreatedMessages() { + if (createdMessages != null) { + createdMessages.clear(); + } + } + + public static EvaluationContext emptyContext() { + return EMPTY_CONTEXT; + } + + public void addEvaluationError(int line, int charPositionInLine, @Nullable FunctionDescriptor descriptor, Throwable e) { + if (evalErrors == null) { + evalErrors = Lists.newArrayList(); + } + evalErrors.add(new EvalError(line, charPositionInLine, descriptor, e)); + } + + public boolean hasEvaluationErrors() { + return evalErrors != null; + } + + public List evaluationErrors() { + return evalErrors == null ? Collections.emptyList() : Collections.unmodifiableList(evalErrors); + } + + public class TypedValue { + private final Class type; + private final Object value; + + public TypedValue(Class type, Object value) { + this.type = type; + this.value = value; + } + + public Class getType() { + return type; + } + + public Object getValue() { + return value; + } + } + + public static class EvalError { + private final int line; + private final int charPositionInLine; + @Nullable + private final FunctionDescriptor descriptor; + private final Throwable throwable; + + public EvalError(int line, int charPositionInLine, @Nullable FunctionDescriptor descriptor, Throwable throwable) { + this.line = line; + this.charPositionInLine = charPositionInLine; + this.descriptor = descriptor; + this.throwable = throwable; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + if (descriptor != null) { + sb.append("In call to function '").append(descriptor.name()).append("' at "); + } else { + sb.append("At "); + } + return sb.append(line) + .append(":") + .append(charPositionInLine) + .append(" an exception was thrown: ") + .append(throwable.getMessage()) + .toString(); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineConfig.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineConfig.java new file mode 100644 index 000000000000..da177687c210 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineConfig.java @@ -0,0 +1,30 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import com.github.joschi.jadconfig.Parameter; + +import org.graylog2.plugin.PluginConfigBean; + +public class PipelineConfig implements PluginConfigBean { + + @Parameter("cached_stageiterators") + private boolean cachedStageIterators = true; + + @Parameter("generate_native_code") + private boolean generateNativeCode = false; +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java new file mode 100644 index 000000000000..67e0405d2823 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorMessageDecorator.java @@ -0,0 +1,133 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.inject.assistedinject.Assisted; + +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.processors.ConfigurationStateUpdater; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; +import org.graylog2.decorators.Decorator; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.configuration.ConfigurationRequest; +import org.graylog2.plugin.configuration.fields.ConfigurationField; +import org.graylog2.plugin.configuration.fields.DropdownField; +import org.graylog2.plugin.decorators.SearchResponseDecorator; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; +import org.graylog2.rest.resources.search.responses.SearchResponse; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +public class PipelineProcessorMessageDecorator implements SearchResponseDecorator { + private static final String CONFIG_FIELD_PIPELINE = "pipeline"; + + private final PipelineInterpreter pipelineInterpreter; + private final ConfigurationStateUpdater pipelineStateUpdater; + private final ImmutableSet pipelines; + + public interface Factory extends SearchResponseDecorator.Factory { + @Override + PipelineProcessorMessageDecorator create(Decorator decorator); + + @Override + Config getConfig(); + + @Override + Descriptor getDescriptor(); + } + + public static class Config implements SearchResponseDecorator.Config { + private final PipelineService pipelineService; + + @Inject + public Config(PipelineService pipelineService) { + this.pipelineService = pipelineService; + } + + @Override + public ConfigurationRequest getRequestedConfiguration() { + final Map pipelineOptions = this.pipelineService.loadAll().stream() + .sorted((o1, o2) -> o1.title().compareTo(o2.title())) + .collect(Collectors.toMap(PipelineDao::id, PipelineDao::title)); + return new ConfigurationRequest() {{ + addField(new DropdownField(CONFIG_FIELD_PIPELINE, + "Pipeline", + "", + pipelineOptions, + "Which pipeline to use for message decoration", + ConfigurationField.Optional.NOT_OPTIONAL)); + }}; + }; + } + + public static class Descriptor extends SearchResponseDecorator.Descriptor { + public Descriptor() { + super("Pipeline Processor Decorator", "http://docs.graylog.org/en/2.0/pages/pipelines.html", "Pipeline Processor Decorator"); + } + } + + @Inject + public PipelineProcessorMessageDecorator(PipelineInterpreter pipelineInterpreter, + ConfigurationStateUpdater pipelineStateUpdater, + @Assisted Decorator decorator) { + this.pipelineInterpreter = pipelineInterpreter; + this.pipelineStateUpdater = pipelineStateUpdater; + final String pipelineId = (String)decorator.config().get(CONFIG_FIELD_PIPELINE); + if (Strings.isNullOrEmpty(pipelineId)) { + this.pipelines = ImmutableSet.of(); + } else { + this.pipelines = ImmutableSet.of(pipelineId); + } + } + + @Override + public SearchResponse apply(SearchResponse searchResponse) { + final List results = new ArrayList<>(); + if (pipelines.isEmpty()) { + return searchResponse; + } + searchResponse.messages().forEach((inMessage) -> { + final Message message = new Message(inMessage.message()); + final List additionalCreatedMessages = pipelineInterpreter.processForPipelines(message, + pipelines, + new NoopInterpreterListener(), + pipelineStateUpdater.getLatestState()); + + results.add(ResultMessageSummary.create(inMessage.highlightRanges(), message.getFields(), inMessage.index())); + additionalCreatedMessages.forEach((additionalMessage) -> { + // TODO: pass proper highlight ranges. Need to rebuild them for new messages. + results.add(ResultMessageSummary.create( + ImmutableMultimap.of(), + additionalMessage.getFields(), + "[created from decorator]" + )); + }); + }); + + return searchResponse.toBuilder().messages(results).build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java new file mode 100644 index 000000000000..38bbdcd52cea --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import com.google.inject.assistedinject.FactoryModuleBuilder; + +import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes; +import org.graylog.plugins.pipelineprocessor.functions.ProcessorFunctionsModule; +import org.graylog.plugins.pipelineprocessor.periodical.LegacyDefaultStreamMigration; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnectionsResource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineResource; +import org.graylog.plugins.pipelineprocessor.rest.PipelineRestPermissions; +import org.graylog.plugins.pipelineprocessor.rest.RuleResource; +import org.graylog.plugins.pipelineprocessor.rest.SimulatorResource; +import org.graylog2.plugin.PluginConfigBean; +import org.graylog2.plugin.PluginModule; + +import java.util.Collections; +import java.util.Set; + +public class PipelineProcessorModule extends PluginModule { + @Override + protected void configure() { + addPeriodical(LegacyDefaultStreamMigration.class); + + addMessageProcessor(PipelineInterpreter.class, PipelineInterpreter.Descriptor.class); + addRestResource(RuleResource.class); + addRestResource(PipelineResource.class); + addRestResource(PipelineConnectionsResource.class); + addRestResource(SimulatorResource.class); + addPermissions(PipelineRestPermissions.class); + + install(new ProcessorFunctionsModule()); + + installSearchResponseDecorator(searchResponseDecoratorBinder(), + PipelineProcessorMessageDecorator.class, + PipelineProcessorMessageDecorator.Factory.class); + + install(new FactoryModuleBuilder().build(PipelineInterpreter.State.Factory.class)); + + addAuditEventTypes(PipelineProcessorAuditEventTypes.class); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java new file mode 100644 index 000000000000..8531475ecdbd --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Pipeline.java @@ -0,0 +1,106 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; +import com.google.auto.value.AutoValue; +import com.google.auto.value.extension.memoized.Memoized; +import com.google.common.collect.Sets; +import org.graylog2.shared.metrics.MetricUtils; + +import javax.annotation.Nullable; +import java.util.SortedSet; + +@AutoValue +public abstract class Pipeline { + + private String metricName; + private transient Meter executed; + + @Nullable + public abstract String id(); + public abstract String name(); + public abstract SortedSet stages(); + + public static Builder builder() { + return new AutoValue_Pipeline.Builder(); + } + + public static Pipeline empty(String name) { + return builder().name(name).stages(Sets.newTreeSet()).build(); + } + + public abstract Builder toBuilder(); + + public Pipeline withId(String id) { + return toBuilder().id(id).build(); + } + + @Memoized + public abstract int hashCode(); + + /** + * Register the metrics attached to this pipeline. + * + * @param metricRegistry the registry to add the metrics to + */ + public void registerMetrics(MetricRegistry metricRegistry) { + if (id() != null) { + metricName = MetricRegistry.name(Pipeline.class, id(), "executed"); + executed = metricRegistry.meter(metricName); + } + } + + /** + * The metric filter matching all metrics that have been registered by this pipeline. + * Commonly used to remove the relevant metrics from the registry upon deletion of the pipeline. + * + * @return the filter matching this pipeline's metrics + */ + public MetricFilter metricsFilter() { + if (id() == null) { + return (name, metric) -> false; + } + return new MetricUtils.SingleMetricFilter(metricName); + + } + public void markExecution() { + if (executed != null) { + executed.mark(); + } + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Pipeline build(); + + public abstract Builder id(String id); + + public abstract Builder name(String name); + + public abstract Builder stages(SortedSet stages); + } + + public String toString() { + final StringBuilder sb = new StringBuilder("Pipeline "); + sb.append("'").append(name()).append("'"); + sb.append(" (").append(id()).append(")"); + return sb.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java new file mode 100644 index 000000000000..714a504d7b3e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Rule.java @@ -0,0 +1,218 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; + +import org.antlr.v4.runtime.CommonToken; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.codegen.GeneratedRule; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.reflections.ReflectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import javax.annotation.Nullable; + +@AutoValue +public abstract class Rule { + private static final Logger LOG = LoggerFactory.getLogger(Rule.class); + + private transient Set metricNames = Sets.newHashSet(); + + private transient Meter globalExecuted; + private transient Meter localExecuted; + private transient Meter globalFailed; + private transient Meter localFailed; + private transient Meter globalMatched; + private transient Meter localMatched; + private transient Meter globalNotMatched; + private transient Meter localNotMatched; + + @Nullable + public abstract String id(); + + public abstract String name(); + + public abstract LogicalExpression when(); + + public abstract Collection then(); + + @Nullable + public abstract Class generatedRuleClass(); + + @Nullable + public abstract GeneratedRule generatedRule(); + + public static Builder builder() { + return new AutoValue_Rule.Builder(); + } + + public abstract Builder toBuilder(); + + public Rule withId(String id) { + return toBuilder().id(id).build(); + } + + public static Rule alwaysFalse(String name) { + return builder().name(name).when(new BooleanExpression(new CommonToken(-1), false)).then(Collections.emptyList()).build(); + } + + /** + * Register the metrics attached to this pipeline. + * + * @param metricRegistry the registry to add the metrics to + */ + public void registerMetrics(MetricRegistry metricRegistry, String pipelineId, String stageId) { + if (id() == null) { + LOG.debug("Not registering metrics for unsaved rule {}", name()); + return; + } + if (id() != null) { + globalExecuted = registerGlobalMeter(metricRegistry, "executed"); + localExecuted = registerLocalMeter(metricRegistry, pipelineId, stageId, "executed"); + + globalFailed = registerGlobalMeter(metricRegistry, "failed"); + localFailed = registerLocalMeter(metricRegistry, pipelineId, stageId, "failed"); + + globalMatched = registerGlobalMeter(metricRegistry, "matched"); + localMatched = registerLocalMeter(metricRegistry, pipelineId, stageId, "matched"); + + globalNotMatched = registerGlobalMeter(metricRegistry, "not-matched"); + localNotMatched = registerLocalMeter(metricRegistry, pipelineId, stageId, "not-matched"); + + } + } + + private Meter registerGlobalMeter(MetricRegistry metricRegistry, String type) { + final String name = MetricRegistry.name(Rule.class, id(), type); + metricNames.add(name); + return metricRegistry.meter(name); + } + + private Meter registerLocalMeter(MetricRegistry metricRegistry, + String pipelineId, + String stageId, String type) { + final String name = MetricRegistry.name(Rule.class, id(), pipelineId, stageId, type); + metricNames.add(name); + return metricRegistry.meter(name); + } + + /** + * The metric filter matching all metrics that have been registered by this pipeline. + * Commonly used to remove the relevant metrics from the registry upon deletion of the pipeline. + * + * @return the filter matching this pipeline's metrics + */ + public MetricFilter metricsFilter() { + if (id() == null) { + return (name, metric) -> false; + } + return (name, metric) -> metricNames.contains(name); + + } + + public void markExecution() { + if (id() != null) { + globalExecuted.mark(); + localExecuted.mark(); + } + } + + public void markMatch() { + if (id() != null) { + globalMatched.mark(); + localMatched.mark(); + } + } + + public void markNonMatch() { + if (id() != null) { + globalNotMatched.mark(); + localNotMatched.mark(); + } + } + + public void markFailure() { + if (id() != null) { + globalFailed.mark(); + localFailed.mark(); + } + } + + /** + * Creates a copy of this Rule with a new instance of the generated rule class if present. + * + * This prevents sharing instances across threads, which is not supported for performance reasons. + * Otherwise the generated code would need to be thread safe, adding to the runtime overhead. + * Instead we buy speed by spending more memory. + * + * @param functionRegistry the registered functions of the system + * @return a copy of this rule with a new instance of its generated code + */ + public Rule invokableCopy(FunctionRegistry functionRegistry) { + final Builder builder = toBuilder(); + final Class ruleClass = generatedRuleClass(); + if (ruleClass != null) { + try { + //noinspection unchecked + final Set constructors = ReflectionUtils.getConstructors(ruleClass); + final Constructor onlyElement = Iterables.getOnlyElement(constructors); + final GeneratedRule instance = (GeneratedRule) onlyElement.newInstance(functionRegistry); + builder.generatedRule(instance); + } catch (IllegalAccessException | InstantiationException | InvocationTargetException e) { + LOG.warn("Unable to generate code for rule {}: {}", id(), e); + } + } + return builder.build(); + } + + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder id(String id); + public abstract Builder name(String name); + public abstract Builder when(LogicalExpression condition); + public abstract Builder then(Collection actions); + public abstract Builder generatedRuleClass(@Nullable Class klass); + public abstract Builder generatedRule(GeneratedRule instance); + + public abstract Rule build(); + } + + + public String toString() { + final StringBuilder sb = new StringBuilder("Rule "); + sb.append("'").append(name()).append("'"); + sb.append(" (").append(id()).append(")"); + return sb.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstBaseListener.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstBaseListener.java new file mode 100644 index 000000000000..5c55d4f22322 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstBaseListener.java @@ -0,0 +1,390 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import org.graylog.plugins.pipelineprocessor.ast.expressions.AdditionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ConstantExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MultiplicationExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NumericExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.SignedExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.UnaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.statements.FunctionStatement; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; + +public class RuleAstBaseListener implements RuleAstListener { + @Override + public void enterRule(Rule rule) { + + } + + @Override + public void exitRule(Rule rule) { + + } + + @Override + public void enterWhen(Rule rule) { + + } + + @Override + public void exitWhen(Rule rule) { + + } + + @Override + public void enterThen(Rule rule) { + + } + + @Override + public void exitThen(Rule rule) { + + } + + @Override + public void enterStatement(Statement statement) { + + } + + @Override + public void exitStatement(Statement statement) { + + } + + @Override + public void enterFunctionCallStatement(FunctionStatement func) { + + } + + @Override + public void exitFunctionCallStatement(FunctionStatement func) { + + } + + @Override + public void enterVariableAssignStatement(VarAssignStatement assign) { + + } + + @Override + public void exitVariableAssignStatement(VarAssignStatement assign) { + + } + + @Override + public void enterAddition(AdditionExpression expr) { + + } + + @Override + public void exitAddition(AdditionExpression expr) { + + } + + @Override + public void enterAnd(AndExpression expr) { + + } + + @Override + public void exitAnd(AndExpression expr) { + + } + + @Override + public void enterArrayLiteral(ArrayLiteralExpression expr) { + + } + + @Override + public void exitArrayLiteral(ArrayLiteralExpression expr) { + + } + + @Override + public void enterBinary(BinaryExpression expr) { + + } + + @Override + public void exitBinary(BinaryExpression expr) { + + } + + @Override + public void enterBoolean(BooleanExpression expr) { + + } + + @Override + public void exitBoolean(BooleanExpression expr) { + + } + + @Override + public void enterBooleanFuncWrapper(BooleanValuedFunctionWrapper expr) { + + } + + @Override + public void exitBooleanFuncWrapper(BooleanValuedFunctionWrapper expr) { + + } + + @Override + public void enterComparison(ComparisonExpression expr) { + + } + + @Override + public void exitComparison(ComparisonExpression expr) { + + } + + @Override + public void enterConstant(ConstantExpression expr) { + + } + + @Override + public void exitConstant(ConstantExpression expr) { + + } + + @Override + public void enterDouble(DoubleExpression expr) { + + } + + @Override + public void exitDouble(DoubleExpression expr) { + + } + + @Override + public void enterEquality(EqualityExpression expr) { + + } + + @Override + public void exitEquality(EqualityExpression expr) { + + } + + @Override + public void enterFieldAccess(FieldAccessExpression expr) { + + } + + @Override + public void exitFieldAccess(FieldAccessExpression expr) { + + } + + @Override + public void enterFieldRef(FieldRefExpression expr) { + + } + + @Override + public void exitFieldRef(FieldRefExpression expr) { + + } + + @Override + public void enterFunctionCall(FunctionExpression expr) { + + } + + @Override + public void exitFunctionCall(FunctionExpression expr) { + + } + + @Override + public void enterIndexedAccess(IndexedAccessExpression expr) { + + } + + @Override + public void exitIndexedAccess(IndexedAccessExpression expr) { + + } + + @Override + public void enterLogical(LogicalExpression expr) { + + } + + @Override + public void exitLogical(LogicalExpression expr) { + + } + + @Override + public void enterLong(LongExpression expr) { + + } + + @Override + public void exitLong(LongExpression expr) { + + } + + @Override + public void enterMapLiteral(MapLiteralExpression expr) { + + } + + @Override + public void exitMapLiteral(MapLiteralExpression expr) { + + } + + @Override + public void enterMessageRef(MessageRefExpression expr) { + + } + + @Override + public void exitMessageRef(MessageRefExpression expr) { + + } + + @Override + public void enterMultiplication(MultiplicationExpression expr) { + + } + + @Override + public void exitMultiplication(MultiplicationExpression expr) { + + } + + @Override + public void enterNot(NotExpression expr) { + + } + + @Override + public void exitNot(NotExpression expr) { + + } + + @Override + public void enterNumeric(NumericExpression expr) { + + } + + @Override + public void exitNumeric(NumericExpression expr) { + + } + + @Override + public void enterOr(OrExpression expr) { + + } + + @Override + public void exitOr(OrExpression expr) { + + } + + @Override + public void enterSigned(SignedExpression expr) { + + } + + @Override + public void exitSigned(SignedExpression expr) { + + } + + @Override + public void enterString(StringExpression expr) { + + } + + @Override + public void exitString(StringExpression expr) { + + } + + @Override + public void enterUnary(UnaryExpression expr) { + + } + + @Override + public void exitUnary(UnaryExpression expr) { + + } + + @Override + public void enterVariableReference(VarRefExpression expr) { + + } + + @Override + public void exitVariableReference(VarRefExpression expr) { + + } + + @Override + public void enterEveryExpression(Expression expr) { + + } + + @Override + public void exitEveryExpression(Expression expr) { + + } + + @Override + public void enterFunctionArg(FunctionExpression functionExpression, Expression expression) { + + } + + @Override + public void exitFunctionArg(Expression expression) { + + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstListener.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstListener.java new file mode 100644 index 000000000000..fde3d8b67deb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstListener.java @@ -0,0 +1,189 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import org.graylog.plugins.pipelineprocessor.ast.expressions.AdditionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ConstantExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MultiplicationExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NumericExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.SignedExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.UnaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.statements.FunctionStatement; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; + +/** + * Consider using RuleAstBaseListener to only implement the callbacks relevant to you. + */ +public interface RuleAstListener { + void enterRule(Rule rule); + + void exitRule(Rule rule); + + void enterWhen(Rule rule); + + void exitWhen(Rule rule); + + void enterThen(Rule rule); + + void exitThen(Rule rule); + + void enterStatement(Statement statement); + + void exitStatement(Statement statement); + + void enterFunctionCallStatement(FunctionStatement func); + + void exitFunctionCallStatement(FunctionStatement func); + + void enterVariableAssignStatement(VarAssignStatement assign); + + void exitVariableAssignStatement(VarAssignStatement assign); + + void enterAddition(AdditionExpression expr); + + void exitAddition(AdditionExpression expr); + + void enterAnd(AndExpression expr); + + void exitAnd(AndExpression expr); + + void enterArrayLiteral(ArrayLiteralExpression expr); + + void exitArrayLiteral(ArrayLiteralExpression expr); + + void enterBinary(BinaryExpression expr); + + void exitBinary(BinaryExpression expr); + + void enterBoolean(BooleanExpression expr); + + void exitBoolean(BooleanExpression expr); + + void enterBooleanFuncWrapper(BooleanValuedFunctionWrapper expr); + + void exitBooleanFuncWrapper(BooleanValuedFunctionWrapper expr); + + void enterComparison(ComparisonExpression expr); + + void exitComparison(ComparisonExpression expr); + + void enterConstant(ConstantExpression expr); + + void exitConstant(ConstantExpression expr); + + void enterDouble(DoubleExpression expr); + + void exitDouble(DoubleExpression expr); + + void enterEquality(EqualityExpression expr); + + void exitEquality(EqualityExpression expr); + + void enterFieldAccess(FieldAccessExpression expr); + + void exitFieldAccess(FieldAccessExpression expr); + + void enterFieldRef(FieldRefExpression expr); + + void exitFieldRef(FieldRefExpression expr); + + void enterFunctionCall(FunctionExpression expr); + + void exitFunctionCall(FunctionExpression expr); + + void enterIndexedAccess(IndexedAccessExpression expr); + + void exitIndexedAccess(IndexedAccessExpression expr); + + void enterLogical(LogicalExpression expr); + + void exitLogical(LogicalExpression expr); + + void enterLong(LongExpression expr); + + void exitLong(LongExpression expr); + + void enterMapLiteral(MapLiteralExpression expr); + + void exitMapLiteral(MapLiteralExpression expr); + + void enterMessageRef(MessageRefExpression expr); + + void exitMessageRef(MessageRefExpression expr); + + void enterMultiplication(MultiplicationExpression expr); + + void exitMultiplication(MultiplicationExpression expr); + + void enterNot(NotExpression expr); + + void exitNot(NotExpression expr); + + void enterNumeric(NumericExpression expr); + + void exitNumeric(NumericExpression expr); + + void enterOr(OrExpression expr); + + void exitOr(OrExpression expr); + + void enterSigned(SignedExpression expr); + + void exitSigned(SignedExpression expr); + + void enterString(StringExpression expr); + + void exitString(StringExpression expr); + + void enterUnary(UnaryExpression expr); + + void exitUnary(UnaryExpression expr); + + void enterVariableReference(VarRefExpression expr); + + void exitVariableReference(VarRefExpression expr); + + void enterEveryExpression(Expression expr); + + void exitEveryExpression(Expression expr); + + void enterFunctionArg(FunctionExpression functionExpression, Expression expression); + + void exitFunctionArg(Expression expression); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstWalker.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstWalker.java new file mode 100644 index 000000000000..90740dc1b77d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/RuleAstWalker.java @@ -0,0 +1,270 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import org.graylog.plugins.pipelineprocessor.ast.expressions.AdditionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ConstantExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MultiplicationExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NumericExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.SignedExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.UnaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.statements.FunctionStatement; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; + +import java.util.Collection; + +public class RuleAstWalker { + + public void walk(RuleAstListener listener, Rule rule) { + listener.enterRule(rule); + + listener.enterWhen(rule); + + walkExpression(listener, rule.when()); + + listener.exitWhen(rule); + + listener.enterThen(rule); + + walkStatements(listener, rule.then()); + + listener.exitThen(rule); + + listener.exitRule(rule); + } + + private void walkExpression(RuleAstListener listener, Expression expr) { + listener.enterEveryExpression(expr); + triggerAbstractEnter(listener, expr); + switch (expr.nodeType()) { + case ADD: + listener.enterAddition((AdditionExpression) expr); + visitChildren(listener, expr); + listener.exitAddition((AdditionExpression) expr); + break; + case AND: + listener.enterAnd((AndExpression) expr); + visitChildren(listener, expr); + listener.exitAnd((AndExpression) expr); + break; + case ARRAY_LITERAL: + listener.enterArrayLiteral((ArrayLiteralExpression) expr); + visitChildren(listener, expr); + listener.exitArrayLiteral((ArrayLiteralExpression) expr); + break; + case BINARY: + // special, handled as wrapper type in triggerAbstractEnter/Exit + break; + case BOOLEAN: + listener.enterBoolean((BooleanExpression) expr); + visitChildren(listener, expr); + listener.exitBoolean((BooleanExpression) expr); + break; + case BOOLEAN_FUNC_WRAPPER: + listener.enterBooleanFuncWrapper((BooleanValuedFunctionWrapper) expr); + visitChildren(listener, expr); + listener.exitBooleanFuncWrapper((BooleanValuedFunctionWrapper) expr); + break; + case COMPARISON: + listener.enterComparison((ComparisonExpression) expr); + visitChildren(listener, expr); + listener.exitComparison((ComparisonExpression) expr); + break; + case CONSTANT: + // special, handled as wrapper type in triggerAbstractEnter/Exit + break; + case DOUBLE: + listener.enterDouble((DoubleExpression) expr); + visitChildren(listener, expr); + listener.exitDouble((DoubleExpression) expr); + break; + case EQUALITY: + listener.enterEquality((EqualityExpression) expr); + visitChildren(listener, expr); + listener.exitEquality((EqualityExpression) expr); + break; + case FIELD_ACCESS: + listener.enterFieldAccess((FieldAccessExpression) expr); + visitChildren(listener, expr); + listener.exitFieldAccess((FieldAccessExpression) expr); + break; + case FIELD_REF: + listener.enterFieldRef((FieldRefExpression) expr); + visitChildren(listener, expr); + listener.exitFieldRef((FieldRefExpression) expr); + break; + case FUNCTION: + listener.enterFunctionCall((FunctionExpression) expr); + // special case, we want to wrap each function argument's expressing into its own + // callback, so we can generate statements for them. + expr.children().forEach(expression -> { + listener.enterFunctionArg((FunctionExpression) expr, expression); + walkExpression(listener, expression); + listener.exitFunctionArg(expression); + }); + + listener.exitFunctionCall((FunctionExpression) expr); + break; + case INDEXED_ACCESS: + listener.enterIndexedAccess((IndexedAccessExpression) expr); + visitChildren(listener, expr); + listener.exitIndexedAccess((IndexedAccessExpression) expr); + break; + case LOGICAL: + // special, handled as wrapper type in triggerAbstractEnter/Exit + break; + case LONG: + listener.enterLong((LongExpression) expr); + visitChildren(listener, expr); + listener.exitLong((LongExpression) expr); + break; + case MAP_LITERAL: + listener.enterMapLiteral((MapLiteralExpression) expr); + visitChildren(listener, expr); + listener.exitMapLiteral((MapLiteralExpression) expr); + break; + case MESSAGE: + listener.enterMessageRef((MessageRefExpression) expr); + visitChildren(listener, expr); + listener.exitMessageRef((MessageRefExpression) expr); + break; + case MULT: + listener.enterMultiplication((MultiplicationExpression) expr); + visitChildren(listener, expr); + listener.exitMultiplication((MultiplicationExpression) expr); + break; + case NOT: + listener.enterNot((NotExpression) expr); + visitChildren(listener, expr); + listener.exitNot((NotExpression) expr); + break; + case NUMERIC: + // special, handled as wrapper type in triggerAbstractEnter/Exit + break; + case OR: + listener.enterOr((OrExpression) expr); + visitChildren(listener, expr); + listener.exitOr((OrExpression) expr); + break; + case SIGNED: + listener.enterSigned((SignedExpression) expr); + visitChildren(listener, expr); + listener.exitSigned((SignedExpression) expr); + break; + case STRING: + listener.enterString((StringExpression) expr); + visitChildren(listener, expr); + listener.exitString((StringExpression) expr); + break; + case UNARY: + // special, handled as wrapper type in triggerAbstractEnter/Exit + break; + case VAR_REF: + listener.enterVariableReference((VarRefExpression) expr); + visitChildren(listener, expr); + listener.exitVariableReference((VarRefExpression) expr); + break; + } + triggerAbstractExit(listener, expr); + listener.exitEveryExpression(expr); + } + + private void triggerAbstractEnter(RuleAstListener listener, Expression expr) { + + if (expr instanceof BinaryExpression) { + listener.enterBinary((BinaryExpression) expr); + + } else if (expr instanceof UnaryExpression) { // must not be first in "else if" because "binary is instanceof unary" + listener.enterUnary((UnaryExpression) expr); + } + // for the others we trigger regardless whether it's a binary or unary expr + if (expr instanceof LogicalExpression) { + listener.enterLogical((LogicalExpression) expr); + } + if (expr instanceof NumericExpression) { + listener.enterNumeric((NumericExpression) expr); + } + if (expr instanceof ConstantExpression) { + listener.enterConstant((ConstantExpression) expr); + } + } + + private void triggerAbstractExit(RuleAstListener listener, Expression expr) { + if (expr instanceof BinaryExpression) { + listener.exitBinary((BinaryExpression) expr); + + } else if (expr instanceof UnaryExpression) { // must not be first in "else if" because "binary is instanceof unary" + listener.exitUnary((UnaryExpression) expr); + } + // for the others we trigger regardless whether it's a binary or unary expr + if (expr instanceof LogicalExpression) { + listener.exitLogical((LogicalExpression) expr); + } + if (expr instanceof NumericExpression) { + listener.exitNumeric((NumericExpression) expr); + } + if (expr instanceof ConstantExpression) { + listener.exitConstant((ConstantExpression) expr); + } + } + + private void visitChildren(RuleAstListener listener, Expression expr) { + expr.children().forEach(expression -> walkExpression(listener, expression)); + } + + private void walkStatements(RuleAstListener listener, Collection statements) { + statements.forEach(statement -> { + listener.enterStatement(statement); + + if (statement instanceof FunctionStatement) { + FunctionStatement func = (FunctionStatement) statement; + listener.enterFunctionCallStatement(func); + walkExpression(listener, func.getFunctionExpression()); + listener.exitFunctionCallStatement(func); + } else if (statement instanceof VarAssignStatement) { + VarAssignStatement assign = (VarAssignStatement) statement; + listener.enterVariableAssignStatement(assign); + walkExpression(listener, assign.getValueExpression()); + listener.exitVariableAssignStatement(assign); + } + + listener.exitStatement(statement); + }); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java new file mode 100644 index 000000000000..1ef998d08913 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/Stage.java @@ -0,0 +1,111 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; +import com.google.auto.value.AutoValue; +import org.graylog2.shared.metrics.MetricUtils; + +import java.util.List; + +import static com.codahale.metrics.MetricRegistry.name; + +@AutoValue +public abstract class Stage implements Comparable { + private List rules; + // not an autovalue property, because it introduces a cycle in hashCode() and we have no way of excluding it + private transient Pipeline pipeline; + private transient Meter executed; + private transient String meterName; + + public abstract int stage(); + public abstract boolean matchAll(); + public abstract List ruleReferences(); + + public List getRules() { + return rules; + } + + public void setRules(List rules) { + this.rules = rules; + } + + public static Builder builder() { + return new AutoValue_Stage.Builder(); + } + + public abstract Builder toBuilder(); + + @Override + public int compareTo(@SuppressWarnings("NullableProblems") Stage other) { + return Integer.compare(stage(), other.stage()); + } + + /** + * Register the metrics attached to this stage. + * + * @param metricRegistry the registry to add the metrics to + */ + public void registerMetrics(MetricRegistry metricRegistry, String pipelineId) { + meterName = name(Pipeline.class, pipelineId, "stage", String.valueOf(stage()), "executed"); + executed = metricRegistry.meter(meterName); + } + + /** + * The metric filter matching all metrics that have been registered by this pipeline. + * Commonly used to remove the relevant metrics from the registry upon deletion of the pipeline. + * + * @return the filter matching this pipeline's metrics + */ + public MetricFilter metricsFilter() { + if (meterName == null) { + return (name, metric) -> false; + } + return new MetricUtils.SingleMetricFilter(meterName); + + } + public void markExecution() { + if (executed != null) { + executed.mark(); + } + } + + public Pipeline getPipeline() { + return pipeline; + } + + public void setPipeline(Pipeline pipeline) { + this.pipeline = pipeline; + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Stage build(); + + public abstract Builder stage(int stageNumber); + + public abstract Builder matchAll(boolean mustMatchAll); + + public abstract Builder ruleReferences(List ruleRefs); + } + + public String toString() { + return "Stage " + stage(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java new file mode 100644 index 000000000000..117a62865e4a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java @@ -0,0 +1,38 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.exceptions; + +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; + +public class FunctionEvaluationException extends LocationAwareEvalException { + private final FunctionExpression functionExpression; + private final Exception exception; + + public FunctionEvaluationException(FunctionExpression functionExpression, Exception exception) { + super(functionExpression.getStartToken(), exception); + this.functionExpression = functionExpression; + this.exception = exception; + } + + public FunctionExpression getFunctionExpression() { + return functionExpression; + } + + public Exception getException() { + return exception; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java new file mode 100644 index 000000000000..3ec325cd5e5d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java @@ -0,0 +1,32 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.exceptions; + +import org.antlr.v4.runtime.Token; + +public class LocationAwareEvalException extends RuntimeException { + private final Token startToken; + + public LocationAwareEvalException(Token startToken, Throwable cause) { + super(cause); + this.startToken = startToken; + } + + public Token getStartToken() { + return startToken; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java new file mode 100644 index 000000000000..1ff53bb94ffa --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java @@ -0,0 +1,35 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.exceptions; + +public class PrecomputeFailure extends RuntimeException { + private final String argumentName; + + public PrecomputeFailure(String argumentName, Exception cause) { + super(cause); + this.argumentName = argumentName; + } + + public String getArgumentName() { + return argumentName; + } + + @Override + public String getMessage() { + return "Unable to pre-compute argument " + getArgumentName() + ": " + getCause().getMessage(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AdditionExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AdditionExpression.java new file mode 100644 index 000000000000..28f6efe9cdcf --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AdditionExpression.java @@ -0,0 +1,132 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.Period; + +import javax.annotation.Nullable; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class AdditionExpression extends BinaryExpression implements NumericExpression { + private final boolean isPlus; + private Class type = Void.class; + + public AdditionExpression(Token start, Expression left, Expression right, boolean isPlus) { + super(start, left, right); + this.isPlus = isPlus; + } + + @Override + public boolean isIntegral() { + return getType().equals(Long.class); + } + + @Override + public long evaluateLong(EvaluationContext context) { + return (long) firstNonNull(evaluateUnsafe(context), 0); + } + + @Override + public double evaluateDouble(EvaluationContext context) { + return (double) firstNonNull(evaluateUnsafe(context), 0d); + } + + @Nullable + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final Object leftValue = left.evaluateUnsafe(context); + final Object rightValue = right.evaluateUnsafe(context); + + // special case for date arithmetic + final boolean leftDate = DateTime.class.equals(leftValue.getClass()); + final boolean leftPeriod = Period.class.equals(leftValue.getClass()); + final boolean rightDate = DateTime.class.equals(rightValue.getClass()); + final boolean rightPeriod = Period.class.equals(rightValue.getClass()); + + if (leftDate && rightPeriod) { + final DateTime date = (DateTime) leftValue; + final Period period = (Period) rightValue; + + return isPlus() ? date.plus(period) : date.minus(period); + } else if (leftPeriod && rightDate) { + final DateTime date = (DateTime) rightValue; + final Period period = (Period) leftValue; + + return isPlus() ? date.plus(period) : date.minus(period); + } else if (leftPeriod && rightPeriod) { + final Period period1 = (Period) leftValue; + final Period period2 = (Period) rightValue; + + return isPlus() ? period1.plus(period2) : period1.minus(period2); + } else if (leftDate && rightDate) { + // the most uncommon, this is only defined for - really and means "interval between them" + // because adding two dates makes no sense + if (isPlus()) { + // makes no sense to compute and should be handles in the parser already + return null; + } + final DateTime left = (DateTime) leftValue; + final DateTime right = (DateTime) rightValue; + + if (left.isBefore(right)) { + return new Duration(left, right); + } else { + return new Duration(right, left); + } + } + if (isIntegral()) { + final long l = (long) leftValue; + final long r = (long) rightValue; + if (isPlus) { + return l + r; + } else { + return l - r; + } + } else { + final double l = (double) leftValue; + final double r = (double) rightValue; + if (isPlus) { + return l + r; + } else { + return l - r; + } + } + } + + public boolean isPlus() { + return isPlus; + } + + @Override + public Class getType() { + return type; + } + + public void setType(Class type) { + this.type = type; + } + + @Override + public String toString() { + return left.toString() + (isPlus ? " + " : " - ") + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java new file mode 100644 index 000000000000..fe9e10999063 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java @@ -0,0 +1,47 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class AndExpression extends BinaryExpression implements LogicalExpression { + public AndExpression(Token start, Expression left, + Expression right) { + super(start, left, right); + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + return ((LogicalExpression)left).evaluateBool(context) && ((LogicalExpression)right).evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public String toString() { + return left.toString() + " AND " + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java new file mode 100644 index 000000000000..c77b01c78449 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java @@ -0,0 +1,62 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.util.List; +import java.util.stream.Collectors; + +public class ArrayLiteralExpression extends BaseExpression { + private final List elements; + + public ArrayLiteralExpression(Token start, List elements) { + super(start); + this.elements = elements; + } + + @Override + public boolean isConstant() { + return elements.stream().allMatch(Expression::isConstant); + } + + @Override + public List evaluateUnsafe(EvaluationContext context) { + return elements.stream() + .map(expression -> expression.evaluateUnsafe(context)) + .collect(Collectors.toList()); + } + + @Override + public Class getType() { + return List.class; + } + + @Override + public String toString() { + return "[" + Joiner.on(", ").join(elements) + "]"; + } + + @Override + public Iterable children() { + return ImmutableList.copyOf(elements); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java new file mode 100644 index 000000000000..0868c58f63ed --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; + +public abstract class BaseExpression implements Expression { + + private final Token startToken; + + public BaseExpression(Token startToken) { + this.startToken = startToken; + } + + @Override + public Token getStartToken() { + return startToken; + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java new file mode 100644 index 000000000000..84f71edf6318 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java @@ -0,0 +1,48 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.collect.ImmutableList; + +import org.antlr.v4.runtime.Token; + +public abstract class BinaryExpression extends UnaryExpression { + + protected Expression left; + + public BinaryExpression(Token start, Expression left, Expression right) { + super(start, right); + this.left = left; + } + + @Override + public boolean isConstant() { + return left.isConstant() && right.isConstant(); + } + + public Expression left() { + return left; + } + + public void left(Expression left) { + this.left = left; + } + @Override + public Iterable children() { + return ImmutableList.of(left, right); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java new file mode 100644 index 000000000000..55fb7b67b9bd --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java @@ -0,0 +1,45 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class BooleanExpression extends ConstantExpression implements LogicalExpression { + private final boolean value; + + public BooleanExpression(Token start, boolean value) { + super(start, Boolean.class); + this.value = value; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return value; + } + + + @Override + public boolean evaluateBool(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return Boolean.toString(value); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java new file mode 100644 index 000000000000..ba1c26b638f7 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanValuedFunctionWrapper.java @@ -0,0 +1,69 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.util.Collections; + +public class BooleanValuedFunctionWrapper extends BaseExpression implements LogicalExpression { + private final Expression expr; + + public BooleanValuedFunctionWrapper(Token start, Expression expr) { + super(start); + this.expr = expr; + if (!expr.getType().equals(Boolean.class)) { + throw new IllegalArgumentException("expr must be of boolean type"); + } + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + final Object value = expr.evaluateUnsafe(context); + return value != null && (Boolean) value; + } + + @Override + public boolean isConstant() { + return expr.isConstant(); + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public Class getType() { + return expr.getType(); + } + + public Expression expression() { + return expr; + } + + @Override + public String toString() { + return expr.toString(); + } + + @Override + public Iterable children() { + return Collections.singleton(expr); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java new file mode 100644 index 000000000000..5bb6e5174aca --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ComparisonExpression.java @@ -0,0 +1,112 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.joda.time.DateTime; + +public class ComparisonExpression extends BinaryExpression implements LogicalExpression { + private final String operator; + + public ComparisonExpression(Token start, Expression left, Expression right, String operator) { + super(start, left, right); + this.operator = operator; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + + final Object leftValue = this.left.evaluateUnsafe(context); + final Object rightValue = this.right.evaluateUnsafe(context); + if (leftValue instanceof DateTime && rightValue instanceof DateTime) { + return compareDateTimes(operator, (DateTime) leftValue, (DateTime) rightValue); + } + + if (leftValue instanceof Double || rightValue instanceof Double) { + return compareDouble(operator, (double) leftValue, (double) rightValue); + } + + return compareLong(operator, (long) leftValue, (long) rightValue); + } + + @SuppressWarnings("Duplicates") + private boolean compareLong(String operator, long left, long right) { + switch (operator) { + case ">": + return left > right; + case ">=": + return left >= right; + case "<": + return left < right; + case "<=": + return left <= right; + default: + return false; + } + } + + @SuppressWarnings("Duplicates") + private boolean compareDouble(String operator, double left, double right) { + switch (operator) { + case ">": + return left > right; + case ">=": + return left >= right; + case "<": + return left < right; + case "<=": + return left <= right; + default: + return false; + } + } + + private boolean compareDateTimes(String operator, DateTime left, DateTime right) { + switch (operator) { + case ">": + return left.isAfter(right); + case ">=": + return !left.isBefore(right); + case "<": + return left.isBefore(right); + case "<=": + return !left.isAfter(right); + default: + return false; + } + } + + public String getOperator() { + return operator; + } + + @Override + public String toString() { + return left.toString() + " " + operator + " " + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java new file mode 100644 index 000000000000..6f11e63e970f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; + +import java.util.Collections; + +public abstract class ConstantExpression extends BaseExpression { + + private final Class type; + + protected ConstantExpression(Token start, Class type) { + super(start); + this.type = type; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public Class getType() { + return type; + } + + @Override + public Iterable children() { + return Collections.emptySet(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java new file mode 100644 index 000000000000..030d131a969b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class DoubleExpression extends ConstantExpression implements NumericExpression { + private final double value; + + public DoubleExpression(Token start, double value) { + super(start, Double.class); + this.value = value; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return Double.toString(value); + } + + @Override + public boolean isIntegral() { + return false; + } + + @Override + public long evaluateLong(EvaluationContext context) { + return (long) value; + } + + @Override + public double evaluateDouble(EvaluationContext context) { + return value; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java new file mode 100644 index 000000000000..fa4cae814fd1 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java @@ -0,0 +1,88 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EqualityExpression extends BinaryExpression implements LogicalExpression { + private static final Logger log = LoggerFactory.getLogger(EqualityExpression.class); + + private final boolean checkEquality; + + public EqualityExpression(Token start, Expression left, Expression right, boolean checkEquality) { + super(start, left, right); + this.checkEquality = checkEquality; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + final Object left = this.left.evaluateUnsafe(context); + final Object right = this.right.evaluateUnsafe(context); + if (left == null) { + log.warn("left expression evaluated to null, returning false: {}", this.left); + return false; + } + final boolean equals; + // sigh: DateTime::equals takes the chronology into account, so identical instants expressed in different timezones are not equal + if (left instanceof DateTime && right instanceof DateTime) { + equals = ((DateTime) left).isEqual((DateTime) right); + } else { + equals = left.equals(right); + } + + if (log.isTraceEnabled()) { + traceEquality(left, right, equals, checkEquality); + } + if (checkEquality) { + return equals; + } + return !equals; + } + + private void traceEquality(Object left, + Object right, + boolean equals, + boolean checkEquality) { + log.trace(checkEquality + ? "[{}] {} == {} : {} == {}" + : "[{}] {} != {} : {} != {}", + checkEquality == equals, this.left, this.right, left, right); + } + + public boolean isCheckEquality() { + return checkEquality; + } + + @Override + public String toString() { + return left.toString() + (checkEquality ? " == " : " != ") + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java new file mode 100644 index 000000000000..04de2614e17f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/Expression.java @@ -0,0 +1,122 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.FunctionEvaluationException; + +import java.util.Map; + +import javax.annotation.Nullable; + +import static org.graylog2.shared.utilities.ExceptionUtils.getRootCause; + +public interface Expression { + + boolean isConstant(); + + Token getStartToken(); + + @Nullable + default Object evaluate(EvaluationContext context) { + try { + return evaluateUnsafe(context); + } catch (FunctionEvaluationException fee) { + context.addEvaluationError(fee.getStartToken().getLine(), + fee.getStartToken().getCharPositionInLine(), + fee.getFunctionExpression().getFunction().descriptor(), + getRootCause(fee)); + } catch (Exception e) { + context.addEvaluationError(getStartToken().getLine(), getStartToken().getCharPositionInLine(), null, getRootCause(e)); + } + return null; + } + + Class getType(); + + /** + * This method is allowed to throw exceptions. The outside world is supposed to call evaluate instead. + */ + @Nullable + Object evaluateUnsafe(EvaluationContext context); + + /** + * This method is allowed to throw exceptions and evaluates the expression in an empty context. + * It is only useful for the interpreter/code generator to evaluate constant expressions to their effective value. + * + * @return the value of the expression in an empty context + */ + default Object evaluateUnsafe() { + return evaluateUnsafe(EvaluationContext.emptyContext()); + } + + Iterable children(); + + default Type nodeType() { + return Type.fromClass(this.getClass()); + } + + // helper to aid switching over the available expression node types + enum Type { + ADD(AdditionExpression.class), + AND(AndExpression.class), + ARRAY_LITERAL(ArrayLiteralExpression.class), + BINARY(BinaryExpression.class), + BOOLEAN(BooleanExpression.class), + BOOLEAN_FUNC_WRAPPER(BooleanValuedFunctionWrapper.class), + COMPARISON(ComparisonExpression.class), + CONSTANT(ConstantExpression.class), + DOUBLE(DoubleExpression.class), + EQUALITY(EqualityExpression.class), + FIELD_ACCESS(FieldAccessExpression.class), + FIELD_REF(FieldRefExpression.class), + FUNCTION(FunctionExpression.class), + INDEXED_ACCESS(IndexedAccessExpression.class), + LOGICAL(LogicalExpression.class), + LONG(LongExpression.class), + MAP_LITERAL(MapLiteralExpression.class), + MESSAGE(MessageRefExpression.class), + MULT(MultiplicationExpression.class), + NOT(NotExpression.class), + NUMERIC(NumericExpression.class), + OR(OrExpression.class), + SIGNED(SignedExpression.class), + STRING(StringExpression.class), + UNARY(UnaryExpression.class), + VAR_REF(VarRefExpression.class); + + static Map classMap; + + static { + classMap = Maps.uniqueIndex(Iterators.forArray(Type.values()), type -> type.klass); + } + + private final Class klass; + + Type(Class expressionClass) { + klass = expressionClass; + } + + static Type fromClass(Class klass) { + return classMap.get(klass); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java new file mode 100644 index 000000000000..1e97108df966 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java @@ -0,0 +1,102 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.base.CaseFormat; +import com.google.common.collect.ImmutableList; +import org.antlr.v4.runtime.Token; +import org.apache.commons.beanutils.PropertyUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; + +public class FieldAccessExpression extends BaseExpression { + private static final Logger log = LoggerFactory.getLogger(FieldAccessExpression.class); + + private final Expression object; + private final Expression field; + + public FieldAccessExpression(Token start, Expression object, Expression field) { + super(start); + this.object = object; + this.field = field; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final Object bean = this.object.evaluateUnsafe(context); + final Object fieldValue = field.evaluateUnsafe(context); + if (bean == null || fieldValue == null) { + return null; + } + final String fieldName = fieldValue.toString(); + + // First try to access the field using the given field name + final Object property = getProperty(bean, fieldName); + if (property == null) { + // If the given field name does not work, try to convert it to camel case to make JSON-like access + // to fields possible. Example: "geo.location.metro_code" => "geo.getLocation().getMetroCode()" + return getProperty(bean, CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, fieldName)); + } + return property; + } + + private Object getProperty(Object bean, String fieldName) { + try { + Object property = PropertyUtils.getProperty(bean, fieldName); + if (property == null) { + // in case the bean is a Map, try again with a simple property, it might be masked by the Map + property = PropertyUtils.getSimpleProperty(bean, fieldName); + } + log.debug("[field access] property {} of bean {}: {}", fieldName, bean.getClass().getTypeName(), property); + return property; + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + log.debug("Unable to read property {} from {}", fieldName, bean); + return null; + } + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return object.toString() + "." + field.toString(); + } + + public Expression object() { + return object; + } + + public Expression field() { + return field; + } + + @Override + public Iterable children() { + return ImmutableList.of(object, field); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java new file mode 100644 index 000000000000..ffe14b1390f4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java @@ -0,0 +1,62 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.util.Collections; + +public class FieldRefExpression extends BaseExpression { + private final String variableName; + private final Expression fieldExpr; + + public FieldRefExpression(Token start, String variableName, Expression fieldExpr) { + super(start); + this.variableName = variableName; + this.fieldExpr = fieldExpr; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return variableName; + } + + @Override + public Class getType() { + return String.class; + } + + @Override + public String toString() { + return variableName; + } + + public String fieldName() { + return variableName; + } + + @Override + public Iterable children() { + return Collections.emptySet(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java new file mode 100644 index 000000000000..15592c6fb179 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java @@ -0,0 +1,95 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.base.Joiner; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.FunctionEvaluationException; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.LocationAwareEvalException; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; + +import java.util.Map; +import java.util.stream.Collectors; + +public class FunctionExpression extends BaseExpression { + private final FunctionArgs args; + private final Function function; + private final FunctionDescriptor descriptor; + + public FunctionExpression(Token start, FunctionArgs args) { + super(start); + this.args = args; + this.function = args.getFunction(); + this.descriptor = this.function.descriptor(); + + // precomputes all constant arguments to avoid dynamically recomputing trees on every invocation + this.function.preprocessArgs(args); + } + + public Function getFunction() { + return function; + } + + public FunctionArgs getArgs() { + return args; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + try { + return descriptor.returnType().cast(function.evaluate(args, context)); + } catch (LocationAwareEvalException laee) { + // the exception already has a location from the input source, simply propagate it. + throw laee; + } catch (Exception e) { + // we need to wrap the original exception to retain the position in the tree where the exception originated + throw new FunctionEvaluationException(this, e); + } + } + + @Override + public Class getType() { + return descriptor.returnType(); + } + + @Override + public String toString() { + String argsString = ""; + if (args != null) { + argsString = Joiner.on(", ") + .withKeyValueSeparator(": ") + .join(args.getArgs().entrySet().stream() + .sorted((e1, e2) -> e1.getKey().compareTo(e2.getKey())) + .iterator()); + } + return descriptor.name() + "(" + argsString + ")"; + } + + @Override + public Iterable children() { + return args.getArgs().entrySet().stream().map(Map.Entry::getValue).collect(Collectors.toList()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java new file mode 100644 index 000000000000..10bfba333417 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpression.java @@ -0,0 +1,94 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.primitives.Ints; +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.lang.reflect.Array; +import java.util.List; +import java.util.Map; + +public class IndexedAccessExpression extends BaseExpression { + private final Expression indexableObject; + private final Expression index; + + public IndexedAccessExpression(Token start, Expression indexableObject, Expression index) { + super(start); + this.indexableObject = indexableObject; + this.index = index; + } + + @Override + public boolean isConstant() { + return indexableObject.isConstant() && index.isConstant(); + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final Object idxObj = this.index.evaluateUnsafe(context); + final Object indexable = indexableObject.evaluateUnsafe(context); + if (idxObj == null || indexable == null) { + return null; + } + + if (idxObj instanceof Long) { + int idx = Ints.saturatedCast((long) idxObj); + if (indexable.getClass().isArray()) { + return Array.get(indexable, idx); + } else if (indexable instanceof List) { + return ((List) indexable).get(idx); + } else if (indexable instanceof Iterable) { + return Iterables.get((Iterable) indexable, idx); + } + throw new IllegalArgumentException("Object '" + indexable + "' is not an Array, List or Iterable."); + } else if (idxObj instanceof String) { + final String idx = idxObj.toString(); + if (indexable instanceof Map) { + return ((Map) indexable).get(idx); + } + throw new IllegalArgumentException("Object '" + indexable + "' is not a Map."); + } + throw new IllegalArgumentException("Index '" + idxObj + "' is not a Long or String."); + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return indexableObject.toString() + "[" + index.toString() + "]"; + } + + public Expression getIndexableObject() { + return indexableObject; + } + + public Expression getIndex() { + return index; + } + + @Override + public Iterable children() { + return ImmutableList.of(indexableObject, index); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java new file mode 100644 index 000000000000..608c0e7eda67 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java @@ -0,0 +1,24 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface LogicalExpression extends Expression { + + boolean evaluateBool(EvaluationContext context); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java new file mode 100644 index 000000000000..ad49fd6822e8 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class LongExpression extends ConstantExpression implements NumericExpression { + private final long value; + + public LongExpression(Token start, long value) { + super(start, Long.class); + this.value = value; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return Long.toString(value); + } + + @Override + public boolean isIntegral() { + return true; + } + + @Override + public long evaluateLong(EvaluationContext context) { + return value; + } + + @Override + public double evaluateDouble(EvaluationContext context) { + return value; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java new file mode 100644 index 000000000000..f55aaba64f65 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java @@ -0,0 +1,70 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.jooq.lambda.Seq; +import org.jooq.lambda.tuple.Tuple2; + +import java.util.HashMap; +import java.util.Map; + +public class MapLiteralExpression extends BaseExpression { + private final HashMap map; + + public MapLiteralExpression(Token start, HashMap map) { + super(start); + this.map = map; + } + + @Override + public boolean isConstant() { + return map.values().stream().allMatch(Expression::isConstant); + } + + @Override + public Map evaluateUnsafe(EvaluationContext context) { + // evaluate all values for each key and return the resulting map + return Seq.seq(map) + .map(entry -> entry.map2(value -> value.evaluateUnsafe(context))) + .toMap(Tuple2::v1, Tuple2::v2); + } + + @Override + public Class getType() { + return Map.class; + } + + @Override + public String toString() { + return "{" + Joiner.on(", ").withKeyValueSeparator(":").join(map) + "}"; + } + + public Iterable> entries() { + return ImmutableSet.copyOf(map.entrySet()); + } + + @Override + public Iterable children() { + return ImmutableList.copyOf(map.values()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java new file mode 100644 index 000000000000..056dee312e80 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java @@ -0,0 +1,64 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import java.util.Collections; + +public class MessageRefExpression extends BaseExpression { + private final Expression fieldExpr; + + public MessageRefExpression(Token start, Expression fieldExpr) { + super(start); + this.fieldExpr = fieldExpr; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final Object fieldName = fieldExpr.evaluateUnsafe(context); + if (fieldName == null) { + return null; + } + return context.currentMessage().getField(fieldName.toString()); + } + + @Override + public Class getType() { + return Object.class; + } + + @Override + public String toString() { + return "$message." + fieldExpr.toString(); + } + + public Expression getFieldExpr() { + return fieldExpr; + } + + @Override + public Iterable children() { + return Collections.singleton(fieldExpr); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MultiplicationExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MultiplicationExpression.java new file mode 100644 index 000000000000..c055a152052d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MultiplicationExpression.java @@ -0,0 +1,104 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import javax.annotation.Nullable; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class MultiplicationExpression extends BinaryExpression implements NumericExpression { + private final char operator; + private Class type; + + public MultiplicationExpression(Token start, Expression left, Expression right, char operator) { + super(start, left, right); + this.operator = operator; + } + + @Override + public boolean isIntegral() { + return getType().equals(Long.class); + } + + @Override + public long evaluateLong(EvaluationContext context) { + return (long) firstNonNull(evaluateUnsafe(context), 0); + } + + @Override + public double evaluateDouble(EvaluationContext context) { + return (double) firstNonNull(evaluateUnsafe(context), 0d); + } + + @SuppressWarnings("Duplicates") + @Nullable + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final Object leftValue = left.evaluateUnsafe(context); + final Object rightValue = right.evaluateUnsafe(context); + + if (isIntegral()) { + long l = (long) leftValue; + long r = (long) rightValue; + switch (operator) { + case '*': + return l * r; + case '/': + return l / r; + case '%': + return l % r; + default: + throw new IllegalStateException("Invalid operator, this is a bug."); + } + } else { + final double l = (double) leftValue; + final double r = (double) rightValue; + + switch (operator) { + case '*': + return l * r; + case '/': + return l / r; + case '%': + return l % r; + default: + throw new IllegalStateException("Invalid operator, this is a bug."); + } + } + } + + public char getOperator() { + return operator; + } + + @Override + public Class getType() { + return type; + } + + public void setType(Class type) { + this.type = type; + } + + @Override + public String toString() { + return left.toString() + " " + operator + " " + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java new file mode 100644 index 000000000000..d799a8839905 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class NotExpression extends UnaryExpression implements LogicalExpression { + public NotExpression(Token start, Expression right) { + super(start, right); + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + return !((LogicalExpression)right).evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public String toString() { + return "NOT " + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java new file mode 100644 index 000000000000..18200704210c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java @@ -0,0 +1,28 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface NumericExpression extends Expression { + + boolean isIntegral(); + + long evaluateLong(EvaluationContext context); + + double evaluateDouble(EvaluationContext context); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java new file mode 100644 index 000000000000..87922fd05501 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java @@ -0,0 +1,47 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class OrExpression extends BinaryExpression implements LogicalExpression { + public OrExpression(Token start, Expression left, + Expression right) { + super(start, left, right); + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return evaluateBool(context); + } + + @Override + public boolean evaluateBool(EvaluationContext context) { + return ((LogicalExpression)left).evaluateBool(context) || ((LogicalExpression)right).evaluateBool(context); + } + + @Override + public Class getType() { + return Boolean.class; + } + + @Override + public String toString() { + return left.toString() + " OR " + right.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/SignedExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/SignedExpression.java new file mode 100644 index 000000000000..247fed85223f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/SignedExpression.java @@ -0,0 +1,73 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +import javax.annotation.Nullable; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class SignedExpression extends UnaryExpression implements NumericExpression { + private final boolean isPlus; + + public SignedExpression(Token start, Expression right, boolean isPlus) { + super(start, right); + this.isPlus = isPlus; + } + + @Override + public boolean isIntegral() { + return getType().equals(Long.class); + } + + @Override + public long evaluateLong(EvaluationContext context) { + return (long) firstNonNull(evaluateUnsafe(context), 0); + } + + @Override + public double evaluateDouble(EvaluationContext context) { + return (double) firstNonNull(evaluateUnsafe(context), 0d); + } + + @Nullable + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final Object value = right.evaluateUnsafe(context); + + if (value instanceof Long) { + long number = (long) value; + return isPlus ? +number : -number; + } else if (value instanceof Double) { + double number = (double) value; + return isPlus ? +number : -number; + } + // nothing we could handle, the type checker should've caught it + throw new IllegalArgumentException("Value of '" + right.toString() + "' is not a number: " + value); + } + + @Override + public String toString() { + return (isPlus ? " + " : " - ") + right.toString(); + } + + public boolean isPlus() { + return isPlus; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java new file mode 100644 index 000000000000..d648ee16b62c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java @@ -0,0 +1,40 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public class StringExpression extends ConstantExpression { + + private final String value; + + public StringExpression(Token start, String value) { + super(start, String.class); + this.value = value; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return value; + } + + @Override + public String toString() { + return '"' + value + '"'; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java new file mode 100644 index 000000000000..5b008f88edbf --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/UnaryExpression.java @@ -0,0 +1,68 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.errors.SyntaxError; + +import java.util.Collections; + +public abstract class UnaryExpression extends BaseExpression { + + protected Expression right; + + public UnaryExpression(Token start, Expression right) { + super(start); + this.right = requireNonNull(right, start); + } + + private static Expression requireNonNull(Expression expression, Token token) { + if (expression != null) { + return expression; + } else { + final int line = token.getLine(); + final int positionInLine = token.getCharPositionInLine(); + final String msg = "Invalid expression (line: " + line + ", column: " + positionInLine + ")"; + final SyntaxError syntaxError = new SyntaxError(token.getText(), line, positionInLine, msg, null); + throw new ParseException(Collections.singleton(syntaxError)); + } + } + + @Override + public boolean isConstant() { + return right.isConstant(); + } + + @Override + public Class getType() { + return right.getType(); + } + + public Expression right() { + return right; + } + + public void right(Expression right) { + this.right = right; + } + + @Override + public Iterable children() { + return Collections.singleton(right); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java new file mode 100644 index 000000000000..47497ab08149 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/VarRefExpression.java @@ -0,0 +1,77 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +public class VarRefExpression extends BaseExpression { + private static final Logger log = LoggerFactory.getLogger(VarRefExpression.class); + private final String identifier; + private final Expression varExpr; + private Class type = Object.class; + + public VarRefExpression(Token start, String identifier, Expression varExpr) { + super(start); + this.identifier = identifier; + this.varExpr = varExpr; + } + + @Override + public boolean isConstant() { + return varExpr != null && varExpr.isConstant(); + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + final EvaluationContext.TypedValue typedValue = context.get(identifier); + if (typedValue != null) { + return typedValue.getValue(); + } + log.error("Unable to retrieve value for variable {}", identifier); + return null; + } + + @Override + public Class getType() { + return type; + } + + @Override + public String toString() { + return identifier; + } + + public String varName() { + return identifier; + } + + public Expression varExpr() { return varExpr; } + + public void setType(Class type) { + this.type = type; + } + + @Override + public Iterable children() { + return Collections.emptySet(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java new file mode 100644 index 000000000000..6a45249aef38 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java @@ -0,0 +1,33 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +/** + * Helper Function implementation which evaluates and memoizes all constant FunctionArgs. + * + * @param the return type + */ +public abstract class AbstractFunction implements Function { + + @Override + public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { + return arg.evaluateUnsafe(EvaluationContext.emptyContext()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java new file mode 100644 index 000000000000..c38abeae63fb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java @@ -0,0 +1,90 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.PrecomputeFailure; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public interface Function { + + Logger log = LoggerFactory.getLogger(Function.class); + + Function ERROR_FUNCTION = new AbstractFunction() { + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + return null; + } + + @Override + public void preprocessArgs(FunctionArgs args) { + // intentionally left blank + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("__unresolved_function") + .returnType(Void.class) + .params(ImmutableList.of()) + .build(); + } + }; + + default void preprocessArgs(FunctionArgs args) { + for (Map.Entry e : args.getConstantArgs().entrySet()) { + final String name = e.getKey(); + try { + final Object value = preComputeConstantArgument(args, name, e.getValue()); + if (value != null) { + //noinspection unchecked + final ParameterDescriptor param = (ParameterDescriptor) args.param(name); + if (param == null) { + throw new IllegalStateException("Unknown parameter " + name + "! Cannot continue."); + } + args.setPreComputedValue(name, param.transform().apply(value)); + } + } catch (Exception exception) { + log.debug("Unable to precompute argument value for " + name, exception); + throw new PrecomputeFailure(name, exception); + } + } + + } + + /** + * Implementations should provide a non-null value for each argument they wish to pre-compute. + *
+ * Examples include compile a Pattern from a regex string, which will never change during the lifetime of the function. + * If any part of the expression tree depends on external values this method will not be called, e.g. if the regex depends on a message field. + * @param args the function args for this functions, usually you don't need this + * @param name the name of the argument to potentially precompute + * @param arg the expression tree for the argument + * @return the precomputed value for the argument or null if the value should be dynamically calculated for each invocation + */ + Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg); + + T evaluate(FunctionArgs args, EvaluationContext context); + + FunctionDescriptor descriptor(); + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java new file mode 100644 index 000000000000..38df9974ba9a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java @@ -0,0 +1,87 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import com.google.common.collect.Maps; + +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class FunctionArgs { + + @Nonnull + private final Map args; + + private final Map constantValues = Maps.newHashMap(); + private final Function function; + private final FunctionDescriptor descriptor; + + public FunctionArgs(Function func, Map args) { + function = func; + descriptor = function.descriptor(); + this.args = firstNonNull(args, Collections.emptyMap()); + } + + @Nonnull + public Map getArgs() { + return args; + } + + @Nonnull + public Map getConstantArgs() { + return args.entrySet().stream() + .filter(e -> e != null && e.getValue() != null && e.getValue().isConstant()) + .filter(e -> !(e.getValue() instanceof VarRefExpression)) // do not eagerly touch variables + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + public boolean isPresent(String key) { + return args.containsKey(key); + } + + @Nullable + public Expression expression(String key) { + return args.get(key); + } + + public Object getPreComputedValue(String name) { + return constantValues.get(name); + } + + public void setPreComputedValue(@Nonnull String name, @Nonnull Object value) { + Objects.requireNonNull(value); + constantValues.put(name, value); + } + + public Function getFunction() { + return function; + } + + public ParameterDescriptor param(String name) { + return descriptor.param(name); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java new file mode 100644 index 000000000000..e0bf7d5df8e2 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java @@ -0,0 +1,82 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; + +import javax.annotation.Nullable; + +@AutoValue +@JsonAutoDetect +public abstract class FunctionDescriptor { + + @JsonProperty + public abstract String name(); + + @JsonProperty + public abstract boolean pure(); + + @JsonProperty + public abstract Class returnType(); + + @JsonProperty + public abstract ImmutableList params(); + + @JsonIgnore + public abstract ImmutableMap paramMap(); + + @JsonIgnore + public ParameterDescriptor param(String name) { + return paramMap().get(name); + } + + @JsonProperty + @Nullable + public abstract String description(); + + public static Builder builder() { + //noinspection unchecked + return new AutoValue_FunctionDescriptor.Builder().pure(false); + } + + @AutoValue.Builder + public static abstract class Builder { + abstract FunctionDescriptor autoBuild(); + + public FunctionDescriptor build() { + return paramMap(Maps.uniqueIndex(params(), ParameterDescriptor::name)) + .autoBuild(); + } + + public abstract Builder name(String name); + public abstract Builder pure(boolean pure); + public abstract Builder returnType(Class type); + public Builder params(ParameterDescriptor... params) { + return params(ImmutableList.builder().add(params).build()); + } + public abstract Builder params(ImmutableList params); + public abstract Builder paramMap(ImmutableMap map); + public abstract ImmutableList params(); + public abstract Builder description(@Nullable String description); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java new file mode 100644 index 000000000000..96abb79e3fe2 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/ParameterDescriptor.java @@ -0,0 +1,154 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.functions; + +import com.google.auto.value.AutoValue; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +import java.util.Optional; + +import javax.annotation.Nullable; + +@AutoValue +@JsonAutoDetect +public abstract class ParameterDescriptor { + + @JsonProperty + public abstract Class type(); + + @JsonProperty + public abstract Class transformedType(); + + @JsonProperty + public abstract String name(); + + @JsonProperty + public abstract boolean optional(); + + @JsonIgnore + public abstract java.util.function.Function transform(); + + @JsonProperty + @Nullable + public abstract String description(); + + public static Builder param() { + return new AutoValue_ParameterDescriptor.Builder().optional(false); + } + + public static Builder string(String name) { + return string(name, String.class); + } + + public static Builder string(String name, Class transformedClass) { + return ParameterDescriptor.param().type(String.class).transformedType(transformedClass).name(name); + } + + public static Builder object(String name) { + return object(name, Object.class); + } + + public static Builder object(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Object.class).transformedType(transformedClass).name(name); + } + + public static Builder integer(String name) { + return integer(name, Long.class); + } + + public static Builder integer(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Long.class).transformedType(transformedClass).name(name); + } + + public static Builder floating(String name) { + return floating(name, Double.class); + } + public static Builder floating(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Double.class).transformedType(transformedClass).name(name); + } + + public static Builder bool(String name) { + return bool(name, Boolean.class); + } + + public static Builder bool(String name, Class transformedClass) { + return ParameterDescriptor.param().type(Boolean.class).transformedType(transformedClass).name(name); + } + + public static Builder type(String name, Class typeClass) { + return type(name, typeClass, typeClass); + } + + public static Builder type(String name, Class typeClass, Class transformedClass) { + return ParameterDescriptor.param().type(typeClass).transformedType(transformedClass).name(name); + } + + @Nullable + public R required(FunctionArgs args, EvaluationContext context) { + final Object precomputedValue = args.getPreComputedValue(name()); + if (precomputedValue != null) { + return transformedType().cast(precomputedValue); + } + final Expression valueExpr = args.expression(name()); + if (valueExpr == null) { + return null; + } + final Object value = valueExpr.evaluateUnsafe(context); + return transformedType().cast(transform().apply(type().cast(value))); + } + + public Optional optional(FunctionArgs args, EvaluationContext context) { + return Optional.ofNullable(required(args, context)); + } + + @AutoValue.Builder + public static abstract class Builder { + public abstract Builder type(Class type); + public abstract Builder transformedType(Class type); + public abstract Builder name(String name); + public abstract Builder optional(boolean optional); + + public Builder optional() { + return optional(true); + } + + public abstract Builder description(String description); + + abstract ParameterDescriptor autoBuild(); + public ParameterDescriptor build() { + try { + transform(); + } catch (IllegalStateException ignored) { + // unfortunately there's no "hasTransform" method in autovalue + //noinspection unchecked + transform((java.util.function.Function) java.util.function.Function.identity()); + } + return autoBuild(); + } + + public abstract Builder transform(@Nullable java.util.function.Function transform); + @Nullable + public abstract java.util.function.Function transform(); + + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java new file mode 100644 index 000000000000..38c4bfde5120 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.statements; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +public class FunctionStatement implements Statement { + + private final Expression functionExpression; + + public FunctionStatement(Expression functionExpression) { + this.functionExpression = functionExpression; + } + + @Override + public Object evaluate(EvaluationContext context) { + return functionExpression.evaluate(context); + } + + public Expression getFunctionExpression() { + return functionExpression; + } + + @Override + public String toString() { + return functionExpression.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java new file mode 100644 index 000000000000..fea3c2193b50 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java @@ -0,0 +1,25 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.statements; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface Statement { + + // TODO should this have a return value at all? + Object evaluate(EvaluationContext context); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java new file mode 100644 index 000000000000..df9863bb6619 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java @@ -0,0 +1,50 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.statements; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +public class VarAssignStatement implements Statement { + private final String name; + private final Expression expr; + + public VarAssignStatement(String name, Expression expr) { + this.name = name; + this.expr = expr; + } + + @Override + public Void evaluate(EvaluationContext context) { + final Object result = expr.evaluate(context); + context.define(name, expr.getType(), result); + return null; + } + + public String getName() { + return name; + } + + public Expression getValueExpression() { + return expr; + } + + @Override + public String toString() { + return "let " + name + " = " + expr.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java new file mode 100644 index 000000000000..185422d872de --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java @@ -0,0 +1,49 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.audit; + +import com.google.common.collect.ImmutableSet; +import org.graylog2.audit.PluginAuditEventTypes; + +import java.util.Set; + +public class PipelineProcessorAuditEventTypes implements PluginAuditEventTypes { + private static final String NAMESPACE = "pipeline_processor:"; + + public static final String PIPELINE_CONNECTION_UPDATE = NAMESPACE + "pipeline_connection:update"; + public static final String PIPELINE_CREATE = NAMESPACE + "pipeline:create"; + public static final String PIPELINE_UPDATE = NAMESPACE + "pipeline:update"; + public static final String PIPELINE_DELETE = NAMESPACE + "pipeline:delete"; + public static final String RULE_CREATE = NAMESPACE + "rule:create"; + public static final String RULE_UPDATE = NAMESPACE + "rule:update"; + public static final String RULE_DELETE = NAMESPACE + "rule:delete"; + + private static final Set EVENT_TYPES = ImmutableSet.builder() + .add(PIPELINE_CONNECTION_UPDATE) + .add(PIPELINE_CREATE) + .add(PIPELINE_UPDATE) + .add(PIPELINE_DELETE) + .add(RULE_CREATE) + .add(RULE_UPDATE) + .add(RULE_DELETE) + .build(); + + @Override + public Set auditEventTypes() { + return EVENT_TYPES; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/CodeGenerator.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/CodeGenerator.java new file mode 100644 index 000000000000..5060f235e03a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/CodeGenerator.java @@ -0,0 +1,736 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Primitives; + +import com.squareup.javapoet.AnnotationSpec; +import com.squareup.javapoet.ClassName; +import com.squareup.javapoet.CodeBlock; +import com.squareup.javapoet.FieldSpec; +import com.squareup.javapoet.JavaFile; +import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterizedTypeName; +import com.squareup.javapoet.TypeName; +import com.squareup.javapoet.TypeSpec; + +import org.apache.commons.beanutils.PropertyUtils; +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.RuleAstBaseListener; +import org.graylog.plugins.pipelineprocessor.ast.RuleAstWalker; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AdditionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ConstantExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MultiplicationExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.SignedExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; +import org.graylog.plugins.pipelineprocessor.codegen.compiler.JavaCompiler; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.Period; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.beans.FeatureDescriptor; +import java.beans.PropertyDescriptor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.lang.model.element.Modifier; + +import static com.google.common.base.MoreObjects.firstNonNull; + +public class CodeGenerator { + private static final Logger log = LoggerFactory.getLogger(CodeGenerator.class); + private final Provider compilerProvider; + + @Inject + public CodeGenerator(Provider compilerProvider) { + this.compilerProvider = compilerProvider; + } + + public static String sourceCodeForRule(Rule rule) { + final JavaPoetListener javaPoetListener = new JavaPoetListener(); + new RuleAstWalker().walk(javaPoetListener, rule); + return javaPoetListener.getSource(); + } + + @SuppressWarnings("unchecked") + public Class generateCompiledRule(Rule rule, PipelineClassloader ruleClassloader) { + if (rule.id() == null) { + throw new IllegalArgumentException("Rules must have an id to generate code for them"); + } + final String sourceCode = sourceCodeForRule(rule); + try { + if (log.isTraceEnabled()) { + log.trace("Sourcecode:\n{}", sourceCode); + } + return (Class) compilerProvider.get().loadFromString(ruleClassloader, "org.graylog.plugins.pipelineprocessor.$dynamic.rules.rule$" + rule.id() , sourceCode); + } catch (ClassNotFoundException e) { + log.error("Unable to compile code\n{}", sourceCode); + return null; + } + + } + + private static class JavaPoetListener extends RuleAstBaseListener { + public static final Set> OPERATOR_SAFE_TYPES = Sets.union(Primitives.allPrimitiveTypes(), Primitives.allWrapperTypes()); + private long counter = 0; + private IdentityHashMap codeSnippet = new IdentityHashMap<>(); + + private TypeSpec.Builder classFile; + private JavaFile generatedFile; + private MethodSpec.Builder when; + private MethodSpec.Builder then; + + // points to either when or then + private MethodSpec.Builder currentMethod; + + /** + * the unique set of function references in this rule + */ + private Set functionMembers = Sets.newHashSet(); + private Set hoistedExpressionMembers = Sets.newHashSet(); + private Set functionArgsHolderTypes = Sets.newHashSet(); + private MethodSpec.Builder constructorBuilder; + private CodeBlock.Builder lateConstructorBlock; + private CodeBlock.Builder hoistedConstantExpressions; + private Set functionReferences = Sets.newHashSet(); + + public String getSource() { + return generatedFile.toString(); + } + + @Override + public void enterRule(Rule rule) { + // generates a new ephemeral unique class name for each generated rule. Only valid for the runtime of the jvm + classFile = TypeSpec.classBuilder("rule$" + rule.id()) + .addSuperinterface(GeneratedRule.class) + .addModifiers(Modifier.FINAL, Modifier.PUBLIC) + .addAnnotation(AnnotationSpec.builder(SuppressWarnings.class) + .addMember("value", "$S", "unchecked") + .build() + ) + .addMethod(MethodSpec.methodBuilder("name") + .returns(String.class) + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .addStatement("return $S", rule.name()) + .build() + ); + constructorBuilder = MethodSpec.constructorBuilder() + .addModifiers(Modifier.PUBLIC) + .addParameter(FunctionRegistry.class, "functionRegistry"); + lateConstructorBlock = CodeBlock.builder(); + hoistedConstantExpressions = CodeBlock.builder(); + } + + @Override + public void exitRule(Rule rule) { + // create fields for each used function + classFile.addFields(functionMembers); + // create fields for hoisted constant expressions + classFile.addFields(hoistedExpressionMembers); + // TODO these can be shared and should potentially created by an AnnotationProcessor for each defined function instead of every rule + classFile.addTypes(functionArgsHolderTypes); + + // resolve functions (but only do so once for each function) + constructorBuilder.addStatement("// resolve used functions"); + functionReferences.forEach(block -> constructorBuilder.addStatement("$L", block)); + // add initializers for fields that depend on the functions being set + constructorBuilder.addStatement("// function parameters"); + constructorBuilder.addCode(lateConstructorBlock.build()); + // all the expressions/statements that are constant at compile time + constructorBuilder.addStatement("// constant expressions"); + constructorBuilder.addCode(hoistedConstantExpressions.build()); + + + classFile.addMethod(constructorBuilder.build()); + + generatedFile = JavaFile.builder("org.graylog.plugins.pipelineprocessor.$dynamic.rules", classFile.build()) + .build(); + } + + @Override + public void enterWhen(Rule rule) { + when = MethodSpec.methodBuilder("when") + .addModifiers(Modifier.PUBLIC) + .addAnnotation(Override.class) + .returns(boolean.class) + .addParameter(EvaluationContext.class, "context", Modifier.FINAL); + currentMethod = when; + } + + @Override + public void exitWhen(Rule rule) { + final CodeBlock result = codeSnippet.getOrDefault(rule.when(), CodeBlock.of("$$when")); + when.addStatement("return $L", result); + + classFile.addMethod(when.build()); + // sanity to catch errors earlier + currentMethod = null; + } + + @Override + public void enterThen(Rule rule) { + then = MethodSpec.methodBuilder("then") + .addModifiers(Modifier.PUBLIC) + .addAnnotation(Override.class) + .addParameter(EvaluationContext.class, "context", Modifier.FINAL); + currentMethod = then; + } + + @Override + public void exitThen(Rule rule) { + classFile.addMethod(then.build()); + // sanity to catch errors earlier + currentMethod = null; + } + + @Override + public void exitAnd(AndExpression expr) { + final CodeBlock left = codeSnippet.get(expr.left()); + final CodeBlock right = codeSnippet.get(expr.right()); + + codeSnippet.put(expr, CodeBlock.of("($L && $L)", blockOrMissing(left, expr.left()), blockOrMissing(right, expr.right()))); + } + + @Override + public void exitOr(OrExpression expr) { + final CodeBlock left = codeSnippet.get(expr.left()); + final CodeBlock right = codeSnippet.get(expr.right()); + + codeSnippet.put(expr, CodeBlock.of("($L || $L)", blockOrMissing(left, expr.left()), blockOrMissing(right, expr.right()))); + } + + @Override + public void exitNot(NotExpression expr) { + final CodeBlock right = codeSnippet.get(expr.right()); + + codeSnippet.put(expr, CodeBlock.of("!$L", blockOrMissing(right, expr.right()))); + } + + @Override + public void exitFieldRef(FieldRefExpression expr) { + codeSnippet.put(expr, CodeBlock.of("$L", expr.fieldName())); + } + + @Override + public void exitFieldAccess(FieldAccessExpression expr) { + final CodeBlock object = codeSnippet.get(expr.object()); + final CodeBlock field = codeSnippet.get(expr.field()); + + final Object objectRef = blockOrMissing(object, expr.object()); + + final Expression o = expr.object(); + final PropertyDescriptor[] propertyDescriptors = PropertyUtils.getPropertyDescriptors(o.getType()); + final ImmutableMap propertyByName = Maps.uniqueIndex(Iterators.forArray(propertyDescriptors), FeatureDescriptor::getName); + + final String fieldName = field.toString(); + final CodeBlock block; + if (propertyByName.containsKey(fieldName)) { + // we have the property, resolve the read method name for it + final PropertyDescriptor descriptor = propertyByName.get(fieldName); + final String methodName = descriptor.getReadMethod().getName(); + block = CodeBlock.of("$L.$L()", objectRef, methodName); + } else if (o instanceof Map) { + // there wasn't any property, but the object is a Map, translate into .get() call + block = CodeBlock.of("$L.get($S)", objectRef, field); + } else { + // this is basically an error, because we expected either a property to match or a map lookup. + log.warn("Unable to determine field accessor for property {}", field); + block = CodeBlock.of("null"); + } + + codeSnippet.put(expr, block); + } + + @Override + public void exitFunctionCall(FunctionExpression expr) { + final String functionValueVarName = subExpressionName(); + final FunctionDescriptor function = expr.getFunction().descriptor(); + + final String mangledFunctionName = functionReference(function); + final String mangledFuncArgsHolder = functionArgsHolder(function); + + // evaluate all the parameters (the parser made sure all required fields are given) + final FunctionArgs args = expr.getArgs(); + final CodeBlock.Builder argAssignment = CodeBlock.builder(); + args.getArgs().forEach((name, argExpr) -> { + final Object varRef = blockOrMissing(codeSnippet.get(argExpr), argExpr); + // hoist constant argument evaluation + CodeBlock.Builder target = argExpr.isConstant() ? hoistedConstantExpressions : argAssignment; + target.addStatement("$L.setAndTransform$$$L($L)", + mangledFuncArgsHolder, + name, + varRef); + }); + currentMethod.addCode(argAssignment.build()); + + // actually invoke the function + CodeBlock functionInvocation = CodeBlock.of("$L.evaluate($L, context)", mangledFunctionName, mangledFuncArgsHolder); + + // don't create intermediate values for void functions (set_fields et al) + if (Void.class.equals(function.returnType())) { + currentMethod.addStatement("$L", functionInvocation); + } else { + currentMethod.addStatement("$T $L = $L", ClassName.get(function.returnType()), functionValueVarName, functionInvocation); + } + // create a field/initializer block for the function reference + functionMembers.add( + FieldSpec.builder(expr.getFunction().getClass(), mangledFunctionName, Modifier.PRIVATE, Modifier.FINAL) + .build()); + functionReferences.add(CodeBlock.of("$L = ($T) functionRegistry.resolve($S)", + mangledFunctionName, + expr.getFunction().getClass(), + function.name())); + codeSnippet.put(expr, CodeBlock.of("$L", functionValueVarName)); + } + + @Nonnull + private String functionArgsHolder(FunctionDescriptor function) { + // create the argument holder for the function invocation (and create the holder class if it doesn't exist yet) + final String functionArgsClassname = functionArgsHolderClass(function); + final String functionArgsMember = functionReference(function) + "$" + subExpressionName(); + classFile.addField(FieldSpec.builder( + ClassName.bestGuess(functionArgsClassname), functionArgsMember, Modifier.PRIVATE) + .build()); + + lateConstructorBlock.addStatement("$L = new $L()", functionArgsMember, functionArgsClassname); + + return functionArgsMember; + } + + @Nonnull + private String functionArgsHolderClass(FunctionDescriptor functionDescriptor) { + final String funcReferenceName = functionReference(functionDescriptor); + + final String functionArgsClassname = StringUtils.capitalize(funcReferenceName + "$args"); + final TypeSpec.Builder directFunctionArgs = TypeSpec.classBuilder(functionArgsClassname) + .addModifiers(Modifier.PRIVATE) + .superclass(ClassName.get(FunctionArgs.class)); + final MethodSpec.Builder constructorBuilder = MethodSpec.constructorBuilder() + .addStatement("super($L, $T.emptyMap())", funcReferenceName, ClassName.get(Collections.class)); + + final CodeBlock.Builder parameterValues = CodeBlock.builder(); + functionDescriptor.params().forEach(pd -> { + directFunctionArgs.addMethod(MethodSpec.methodBuilder("setAndTransform$" + pd.name()) + .returns(TypeName.VOID) + .addParameter(ClassName.get(pd.type()), "arg$" + pd.name()) + .addStatement("transformed$$$L = transformer$$$L.apply(arg$$$L)", + pd.name(), pd.name(), pd.name()) + .build() + ); + final ParameterizedTypeName transformerType = ParameterizedTypeName.get(Function.class, pd.type(), pd.transformedType()); + directFunctionArgs.addField( + transformerType, + "transformer$" + pd.name(), + Modifier.PRIVATE, Modifier.FINAL); + directFunctionArgs.addField(ClassName.get(pd.transformedType()), "transformed$" + pd.name()); + constructorBuilder.addStatement("transformer$$$L = ($T) $L.descriptor().param($S).transform()", + pd.name(), + transformerType, + funcReferenceName, + pd.name()); + + parameterValues.add(CodeBlock.builder() + .beginControlFlow("case $S:", pd.name()) + .addStatement("return transformed$$$L", pd.name()) + .endControlFlow().build()); + }); + + directFunctionArgs.addMethod(MethodSpec.methodBuilder("getPreComputedValue") + .returns(TypeName.OBJECT) + .addParameter(String.class, "name") + .addModifiers(Modifier.PUBLIC) + .addAnnotation(Override.class) + .addCode(CodeBlock.builder() + .beginControlFlow("switch (name)") + .add(parameterValues.build()) + .endControlFlow() + .addStatement("return null") + .build()) + .build()); + + directFunctionArgs.addMethod(constructorBuilder.build()); + + final TypeSpec holder = directFunctionArgs.build(); + if (!functionArgsHolderTypes.contains(holder)) { + functionArgsHolderTypes.add(holder); + } + return functionArgsClassname; + } + + @Nonnull + private String functionReference(FunctionDescriptor function) { + return "func$" + function.name(); + } + + @Override + public void exitEquality(EqualityExpression expr) { + final String intermediateName = subExpressionName(); + + // equality is one of the points we generate intermediate values at + final CodeBlock leftBlock = codeSnippet.get(expr.left()); + final CodeBlock rightBlock = codeSnippet.get(expr.right()); + + final Class leftType = expr.left().getType(); + final Class rightType = expr.right().getType(); + boolean useOperator = false; + if (OPERATOR_SAFE_TYPES.contains(leftType) && OPERATOR_SAFE_TYPES.contains(rightType)) { + useOperator = true; + } + String statement = "boolean $L = "; + final boolean checkEquality = expr.isCheckEquality(); + if (useOperator) { + statement += "$L " + (checkEquality ? "==" : "!=") + " $L"; + currentMethod.addStatement(statement, + intermediateName, + blockOrMissing(leftBlock, expr.left()), + blockOrMissing(rightBlock, expr.right())); + } else { + // Dates + if (DateTime.class.equals(leftType)) { + if (DateTime.class.equals(rightType)) { + codeSnippet.putIfAbsent(expr, CodeBlock.of("$L.isEqual($L)", leftBlock, rightBlock)); + return; + } + } else if (Period.class.equals(leftType)) { + if (Period.class.equals(rightType)) { + codeSnippet.putIfAbsent(expr, + CodeBlock.of("$L.toDuration().equals($L.toDuration())", leftBlock, rightBlock)); + return; + } + } + + statement += (checkEquality ? "" : "!") + "$T.equals($L, $L)"; + currentMethod.addStatement(statement, + intermediateName, + ClassName.get(Objects.class), + blockOrMissing(leftBlock, expr.left()), + blockOrMissing(rightBlock, expr.right())); + } + + codeSnippet.put(expr, CodeBlock.of("$L", intermediateName)); + } + + @Override + public void exitComparison(ComparisonExpression expr) { + final CodeBlock left = codeSnippet.get(expr.left()); + final CodeBlock right = codeSnippet.get(expr.right()); + + final Class leftType = expr.left().getType(); + final Class rightType = expr.right().getType(); + + if (DateTime.class.equals(leftType)) { + if (DateTime.class.equals(rightType)) { + CodeBlock block; + switch (expr.getOperator()) { + case ">": + block = CodeBlock.of("$L.isAfter($L)", left, right); + break; + case ">=": + block = CodeBlock.of("!$L.isBefore($L)", left, right); + break; + case "<": + block = CodeBlock.of("$L.isBefore($L)", left, right); + break; + case "<=": + block = CodeBlock.of("!$L.isAfter($L)", left, right); + break; + default: + block = null; + } + if (block != null) { + codeSnippet.putIfAbsent(expr, block); + return; + } + } + } else if (Period.class.equals(leftType)) { + if (Period.class.equals(rightType)) { + codeSnippet.putIfAbsent(expr, + CodeBlock.of("($L.toDuration().getMillis() " + expr.getOperator() + " $L.toDuration().getMillis())", + blockOrMissing(left, expr.left()), + blockOrMissing(right, expr.right()))); + return; + + } + } + + codeSnippet.putIfAbsent(expr, CodeBlock.of("($L " + expr.getOperator() + " $L)", + blockOrMissing(left, expr.left()), + blockOrMissing(right, expr.right()))); + } + + @Override + public void exitBooleanFuncWrapper(BooleanValuedFunctionWrapper expr) { + final CodeBlock embeddedExpr = codeSnippet.get(expr.expression()); + + // simply forward the other expression's code + codeSnippet.put(expr, CodeBlock.of("$L", blockOrMissing(embeddedExpr, expr.expression()))); + } + + @Override + public void exitConstant(ConstantExpression expr) { + codeSnippet.putIfAbsent(expr, CodeBlock.of("$L", expr.evaluateUnsafe())); + } + + @Override + public void exitString(StringExpression expr) { + // this overrides what exitConstant would do for strings… + codeSnippet.putIfAbsent(expr, CodeBlock.of("$S", expr.evaluateUnsafe())); + } + + @Override + public void exitLong(LongExpression expr) { + // long needs a suffix + codeSnippet.putIfAbsent(expr, CodeBlock.of("$LL", expr.evaluateUnsafe())); + } + + @Override + public void exitDouble(DoubleExpression expr) { + // double should have a suffix + codeSnippet.putIfAbsent(expr, CodeBlock.of("$Ld", expr.evaluateUnsafe())); + } + + @Override + public void exitMessageRef(MessageRefExpression expr) { + final Object field = blockOrMissing(codeSnippet.get(expr.getFieldExpr()), expr.getFieldExpr()); + + codeSnippet.putIfAbsent(expr, CodeBlock.of("context.currentMessage().getField($S)", field)); + } + + @Override + public void exitVariableAssignStatement(VarAssignStatement assign) { + final Object value = blockOrMissing(codeSnippet.get(assign.getValueExpression()), assign.getValueExpression()); + final Class type = assign.getValueExpression().getType(); + + // always hoist declaration + hoistedExpressionMembers.add(FieldSpec.builder(type, "var$" + assign.getName(), Modifier.PRIVATE).build()); + if (assign.getValueExpression().isConstant()) { + // also hoist the assignment + hoistedConstantExpressions.addStatement("var$$$L = $L", assign.getName(), value); + } else { + currentMethod.addStatement("var$$$L = $L", assign.getName(), value); + } + } + + @Override + public void exitVariableReference(VarRefExpression expr) { + codeSnippet.putIfAbsent(expr, CodeBlock.of("var$$$L", expr.varName())); + } + + @Override + public void exitMapLiteral(MapLiteralExpression expr) { + // we need an intermediate value for creating the map + final String mapName = "mapLiteral$" + subExpressionName(); + final boolean constantMap = expr.isConstant(); + if (constantMap) { + // we can hoist both the declaration, as well as the definition of the map + hoistedExpressionMembers.add(FieldSpec.builder(Map.class, mapName, Modifier.PRIVATE, Modifier.FINAL).build()); + hoistedConstantExpressions.addStatement("$L = $T.newHashMap()", mapName, Maps.class); + } else { + currentMethod.addStatement("$T $L = $T.newHashMap()", Map.class, mapName, Maps.class); + } + + expr.entries().forEach(entry -> { + final String code = "$L.put($S, $L)"; + final Object[] args = {mapName, entry.getKey(), blockOrMissing(codeSnippet.get(entry.getValue()), entry.getValue())}; + // TODO convert to code block + // hoist only completely constant maps (otherwise we would need to regenerate the non-constant ones per evaluation) + if (constantMap) { + hoistedConstantExpressions.addStatement(code, args); + } else { + currentMethod.addStatement(code, args); + } + }); + // add the reference to the map we created + codeSnippet.putIfAbsent(expr, CodeBlock.of("$L", mapName)); + } + + @Override + public void exitArrayLiteral(ArrayLiteralExpression expr) { + final String listName = "arrayLiteral$" + subExpressionName(); + final boolean constantList = expr.isConstant(); + + final ImmutableList.Builder elementsBuilder = ImmutableList.builder(); + expr.children().forEach(expression -> elementsBuilder.add(blockOrMissing(codeSnippet.get(expression), expression))); + final ImmutableList elements = elementsBuilder.build(); + + // if possible hoist decl to constructor + if (constantList) { + hoistedExpressionMembers.add(FieldSpec.builder(List.class, listName, Modifier.PRIVATE, Modifier.FINAL).build()); + } + final String assignmentFormat = "$L = $T.newArrayList(" + + Stream.generate(() -> "$L").limit(elements.size()).reduce(Joiner.on(", ")::join).orElseGet(() -> "$") + + ")"; + // sigh java varargs + List args = Lists.newArrayList(ArrayList.class, listName, Lists.class); + args.addAll(elements); + // if constant, initialize completely in constructor + if (constantList) { + hoistedConstantExpressions.addStatement(assignmentFormat, args.subList(1, args.size()).toArray()); + } else { + currentMethod.addStatement("$T " + assignmentFormat, args.toArray()); + } + codeSnippet.putIfAbsent(expr, CodeBlock.of("$L", listName)); + } + + @Override + public void exitAddition(AdditionExpression expr) { + final Object leftBlock = blockOrMissing(codeSnippet.get(expr.left()), expr.left()); + final Object rightBlock = blockOrMissing(codeSnippet.get(expr.right()), expr.right()); + + Class leftType = expr.left().getType(); + Class rightType = expr.right().getType(); + + if (DateTime.class.equals(leftType)) { + if (DateTime.class.equals(rightType)) { + // calculate duration between two dates (adding two dates is invalid) + if (expr.isPlus()) { + throw new IllegalStateException("Cannot add two dates, this is a parser bug"); + } + codeSnippet.putIfAbsent(expr, CodeBlock.of( + "new $T($L, $L)", Duration.class, leftBlock, rightBlock)); + } else if (Period.class.equals(rightType)) { + // new datetime + codeSnippet.putIfAbsent(expr, + CodeBlock.of("$L." + (expr.isPlus() ? "plus" : "minus") + "($L)", leftBlock, rightBlock)); + } + return; + } else if (Period.class.equals(leftType)) { + if (DateTime.class.equals(rightType)) { + // invert the arguments, adding the period to the date, yielding a new DateTime + codeSnippet.putIfAbsent(expr, + CodeBlock.of("$L." + (expr.isPlus() ? "plus" : "minus") + "($L)", rightBlock, leftBlock)); + } else if (Period.class.equals(rightType)) { + // adding two periods yields a new period + codeSnippet.putIfAbsent(expr, + CodeBlock.of("$L." + (expr.isPlus() ? "plus" : "minus") + "($L)", leftBlock, rightBlock)); + } + return; + } + + codeSnippet.putIfAbsent(expr, + CodeBlock.of("$L " + (expr.isPlus() ? "+" : "-") + " $L", leftBlock, rightBlock)); + } + + @Override + public void exitMultiplication(MultiplicationExpression expr) { + final Object leftBlock = blockOrMissing(codeSnippet.get(expr.left()), expr.left()); + final Object rightBlock = blockOrMissing(codeSnippet.get(expr.right()), expr.right()); + + codeSnippet.putIfAbsent(expr, + CodeBlock.of("$L " + expr.getOperator() + " $L", leftBlock, rightBlock)); + } + + @Override + public void exitSigned(SignedExpression expr) { + final Object rightBlock = blockOrMissing(codeSnippet.get(expr.right()), expr.right()); + codeSnippet.putIfAbsent(expr, CodeBlock.of((expr.isPlus() ? "+" : "-") + "$L", rightBlock)); + } + + @Override + public void exitIndexedAccess(IndexedAccessExpression expr) { + final Expression indexableObject = expr.getIndexableObject(); + final Expression index = expr.getIndex(); + + final Object objectBlock = blockOrMissing(codeSnippet.get(indexableObject), indexableObject); + final Object indexBlock = blockOrMissing(codeSnippet.get(index), index); + + final Class indexType = index.getType(); + final Class indexableObjectType = indexableObject.getType(); + CodeBlock block; + if (Long.class.equals(indexType)) { + // array indexing + if (indexableObjectType.isArray()) { + block = CodeBlock.of("Arrays.get($L, $L)", objectBlock, indexBlock); + } else if (List.class.isAssignableFrom(indexableObjectType)) { + block = CodeBlock.of("$L.get($T.saturatedCast($L))", objectBlock, ClassName.get(Ints.class), indexBlock); + } else if (Iterable.class.isAssignableFrom(indexableObjectType)) { + block = CodeBlock.of("$T.get($L, $L)", ClassName.get(Iterables.class), objectBlock, indexBlock); + } else { + log.error("Unhandled indexable object type: {}", indexableObject); + block = null; + } + } else if (String.class.equals(indexType) && Map.class.isAssignableFrom(indexableObjectType)) { + // map indexing + block = CodeBlock.of("$L.get($L)", objectBlock, indexBlock); + } else { + // illegal + log.error("Invalid index type: {}", index); + block = null; + } + codeSnippet.putIfAbsent(expr, block); + } + + @Nonnull + private String subExpressionName() { + return "im$" + counter++; + } + + private Object blockOrMissing(Object block, Expression fallBackExpression) { + if (block == null) { + log.warn("Missing code snippet for {}: ", fallBackExpression.nodeType(), fallBackExpression); + } + return firstNonNull(block, fallBackExpression); + } + + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/GeneratedRule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/GeneratedRule.java new file mode 100644 index 000000000000..df8bce64cf79 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/GeneratedRule.java @@ -0,0 +1,29 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; + +public interface GeneratedRule { + + String name(); + + boolean when(EvaluationContext context); + + void then(EvaluationContext context); + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/PipelineClassloader.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/PipelineClassloader.java new file mode 100644 index 000000000000..dee93ef21f7d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/PipelineClassloader.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen; + +import java.util.concurrent.atomic.AtomicLong; + +public class PipelineClassloader extends ClassLoader { + + public static AtomicLong loadedClasses = new AtomicLong(); + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + loadedClasses.incrementAndGet(); + return super.loadClass(name); + } + + public void defineClass(String className, byte[] bytes) { + super.defineClass(className, bytes, 0, bytes.length); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/InMemoryFileManager.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/InMemoryFileManager.java new file mode 100644 index 000000000000..9640b9ce70bd --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/InMemoryFileManager.java @@ -0,0 +1,93 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +/* + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen.compiler; + +import com.google.common.collect.Maps; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.tools.FileObject; +import javax.tools.ForwardingJavaFileManager; +import javax.tools.JavaFileObject; +import javax.tools.JavaFileObject.Kind; +import javax.tools.SimpleJavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.StandardLocation; +import javax.validation.constraints.NotNull; + +public class InMemoryFileManager extends ForwardingJavaFileManager { + private final Map classBytes = Maps.newLinkedHashMap(); + + public InMemoryFileManager(StandardJavaFileManager fileManager) { + super(fileManager); + } + + public JavaFileObject getJavaFileForInput(Location location, String className, Kind kind) throws IOException { + if (location == StandardLocation.CLASS_OUTPUT && classBytes.containsKey(className) && kind == Kind.CLASS) { + final byte[] bytes = classBytes.get(className).toByteArray(); + return new SimpleJavaFileObject(URI.create(className), kind) { + @NotNull + public InputStream openInputStream() { + return new ByteArrayInputStream(bytes); + } + }; + } + return fileManager.getJavaFileForInput(location, className, kind); + } + + @NotNull + public JavaFileObject getJavaFileForOutput(Location location, final String className, Kind kind, FileObject sibling) throws IOException { + return new SimpleJavaFileObject(URI.create(className), kind) { + @NotNull + public OutputStream openOutputStream() { + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + classBytes.put(className, stream); + return stream; + } + }; + } + + @NotNull + public Map getAllClassBytes() { + return classBytes.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toByteArray())); + } +} + diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaCompiler.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaCompiler.java new file mode 100644 index 000000000000..ce9d5b0c8eea --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaCompiler.java @@ -0,0 +1,86 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +/* + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen.compiler; + +import com.google.common.collect.Lists; + +import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import javax.tools.Diagnostic; +import javax.tools.ToolProvider; +import javax.validation.constraints.NotNull; + +public class JavaCompiler { + private static final Logger log = LoggerFactory.getLogger(JavaCompiler.class); + + private static final javax.tools.JavaCompiler JAVA_COMPILER = ToolProvider.getSystemJavaCompiler(); + + public JavaCompiler() { + } + + @NotNull + private Map compileFromSource(@NotNull String className, @NotNull String javaCode) { + if (JAVA_COMPILER == null) { + log.error("No compiler present, unable to compile {}", className); + return Collections.emptyMap(); + } + + final InMemoryFileManager memoryFileManager = new InMemoryFileManager(JAVA_COMPILER.getStandardFileManager(null, null, null)); + List errors = Lists.newArrayList(); + JAVA_COMPILER.getTask(null, memoryFileManager, diagnostic -> { + if (diagnostic.getKind() == Diagnostic.Kind.ERROR) { + errors.add(diagnostic); + } + }, null, null, Collections.singleton(new JavaSourceFromString(className, javaCode))).call(); + Map result = memoryFileManager.getAllClassBytes(); + if (!errors.isEmpty()) { + throw new PipelineCompilationException(errors); + } + return result; + } + + public Class loadFromString(@NotNull PipelineClassloader classLoader, @NotNull String className, @NotNull String source) throws ClassNotFoundException { + for (Map.Entry entry : compileFromSource(className, source).entrySet()) { + String name = entry.getKey(); + byte[] bytes = entry.getValue(); + classLoader.defineClass(name, bytes); + } + return classLoader.loadClass(className); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaSourceFromString.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaSourceFromString.java new file mode 100644 index 000000000000..2543f8e0ce02 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaSourceFromString.java @@ -0,0 +1,55 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +/* + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen.compiler; + +import java.net.URI; + +import javax.tools.SimpleJavaFileObject; +import javax.validation.constraints.NotNull; + +import static javax.tools.JavaFileObject.Kind.SOURCE; + +public class JavaSourceFromString extends SimpleJavaFileObject { + + private final String sourceCode; + + public JavaSourceFromString(@NotNull String name, String sourceCode) { + super(URI.create("string:///" + name.replace('.', '/') + SOURCE.extension), SOURCE); + this.sourceCode = sourceCode; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return sourceCode; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/PipelineCompilationException.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/PipelineCompilationException.java new file mode 100644 index 000000000000..f5611645336f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/PipelineCompilationException.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +/* + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.codegen.compiler; + +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import javax.tools.Diagnostic; + +public class PipelineCompilationException extends RuntimeException { + private final List errors; + + public PipelineCompilationException(List errors) { + this.errors = errors; + } + + @Override + public String getMessage() { + return errors.stream() + .map(diagnostic -> diagnostic.getMessage(Locale.ENGLISH)) + .collect(Collectors.joining("\n")); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java new file mode 100644 index 000000000000..760fa8fee1cf --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineDao.java @@ -0,0 +1,93 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.joda.time.DateTime; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; + +@AutoValue +public abstract class PipelineDao { + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String title(); + + @JsonProperty + @Nullable + public abstract String description(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + @Nullable + public abstract DateTime createdAt(); + + @JsonProperty + @Nullable + public abstract DateTime modifiedAt(); + + public static Builder builder() { + return new AutoValue_PipelineDao.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static PipelineDao create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @Nullable @JsonProperty("created_at") DateTime createdAt, + @Nullable @JsonProperty("modified_at") DateTime modifiedAt) { + return builder() + .id(id) + .title(title) + .description(description) + .source(source) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract PipelineDao build(); + + public abstract Builder id(String id); + + public abstract Builder title(String title); + + public abstract Builder description(String description); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java new file mode 100644 index 000000000000..e632691bf46f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java @@ -0,0 +1,31 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import org.graylog2.database.NotFoundException; + +import java.util.Collection; + +public interface PipelineService { + PipelineDao save(PipelineDao pipeline); + + PipelineDao load(String id) throws NotFoundException; + + Collection loadAll(); + + void delete(String id); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java new file mode 100644 index 000000000000..b65502434b75 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java @@ -0,0 +1,32 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.database.NotFoundException; + +import java.util.Set; + +public interface PipelineStreamConnectionsService { + PipelineConnections save(PipelineConnections connections); + + PipelineConnections load(String streamId) throws NotFoundException; + + Set loadAll(); + + void delete(String streamId); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java new file mode 100644 index 000000000000..bda5da2ff68a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleDao.java @@ -0,0 +1,94 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.joda.time.DateTime; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; + +@AutoValue +public abstract class RuleDao { + + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String title(); + + @JsonProperty + @Nullable + public abstract String description(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + @Nullable + public abstract DateTime createdAt(); + + @JsonProperty + @Nullable + public abstract DateTime modifiedAt(); + + public static Builder builder() { + return new AutoValue_RuleDao.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static RuleDao create(@Id @ObjectId @JsonProperty("_id") @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @JsonProperty("created_at") @Nullable DateTime createdAt, + @JsonProperty("modified_at") @Nullable DateTime modifiedAt) { + return builder() + .id(id) + .source(source) + .title(title) + .description(description) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract RuleDao build(); + + public abstract Builder id(String id); + + public abstract Builder title(String title); + + public abstract Builder description(String description); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java new file mode 100644 index 000000000000..168286de229a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java @@ -0,0 +1,33 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db; + +import org.graylog2.database.NotFoundException; + +import java.util.Collection; + +public interface RuleService { + RuleDao save(RuleDao rule); + + RuleDao load(String id) throws NotFoundException; + + Collection loadAll(); + + void delete(String id); + + Collection loadNamed(Collection ruleNames); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java new file mode 100644 index 000000000000..9b2c9b26c25f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java @@ -0,0 +1,87 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.memory; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MapMaker; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog2.database.NotFoundException; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A PipelineService that does not persist any data, but simply keeps it in memory. + */ +public class InMemoryPipelineService implements PipelineService { + + // poor man's id generator + private AtomicLong idGen = new AtomicLong(0); + + private Map store = new MapMaker().makeMap(); + private Map titleToId = new MapMaker().makeMap(); + + @Override + public PipelineDao save(PipelineDao pipeline) { + PipelineDao toSave = pipeline.id() != null + ? pipeline + : pipeline.toBuilder().id(createId()).build(); + // enforce the title unique constraint + if (titleToId.containsKey(toSave.title())) { + // if this is an update and the title belongs to the passed pipeline, then it's fine + if (!titleToId.get(toSave.title()).equals(toSave.id())) { + throw new IllegalArgumentException("Duplicate pipeline titles are not allowed: " + toSave.title()); + } + } + titleToId.put(toSave.title(), toSave.id()); + store.put(toSave.id(), toSave); + + return toSave; + } + + @Override + public PipelineDao load(String id) throws NotFoundException { + final PipelineDao pipeline = store.get(id); + if (pipeline == null) { + throw new NotFoundException("No such pipeline with id " + id); + } + return pipeline; + } + + @Override + public Collection loadAll() { + return ImmutableSet.copyOf(store.values()); + } + + @Override + public void delete(String id) { + if (id == null) { + return; + } + final PipelineDao removed = store.remove(id); + // clean up title index if the pipeline existed + if (removed != null) { + titleToId.remove(removed.title()); + } + } + + private String createId() { + return String.valueOf(idGen.incrementAndGet()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java new file mode 100644 index 000000000000..524d023e2927 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java @@ -0,0 +1,73 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.memory; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MapMaker; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.database.NotFoundException; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryPipelineStreamConnectionsService implements PipelineStreamConnectionsService { + + // poor man's id generator + private AtomicLong idGen = new AtomicLong(0); + + private Map store = new MapMaker().makeMap(); + + @Override + public PipelineConnections save(PipelineConnections connections) { + PipelineConnections toSave = connections.id() != null + ? connections + : connections.toBuilder().id(createId()).build(); + store.put(toSave.id(), toSave); + + return toSave; + } + + @Override + public PipelineConnections load(String streamId) throws NotFoundException { + final PipelineConnections connections = store.get(streamId); + if (connections == null) { + throw new NotFoundException("No such pipeline connections for stream " + streamId); + } + return connections; + } + + @Override + public Set loadAll() { + return ImmutableSet.copyOf(store.values()); + } + + @Override + public void delete(String streamId) { + try { + final PipelineConnections connections = load(streamId); + store.remove(connections.id()); + } catch (NotFoundException e) { + // Do nothing + } + } + + private String createId() { + return String.valueOf(idGen.incrementAndGet()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java new file mode 100644 index 000000000000..4ca7b41436d8 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java @@ -0,0 +1,98 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.memory; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Sets; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.database.NotFoundException; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +/** + * A RuleService that does not persist any data, but simply keeps it in memory. + */ +public class InMemoryRuleService implements RuleService { + + // poor man's id generator + private AtomicLong idGen = new AtomicLong(0); + + private Map store = new MapMaker().makeMap(); + private Map titleToId = new MapMaker().makeMap(); + + @Override + public RuleDao save(RuleDao rule) { + RuleDao toSave = rule.id() != null + ? rule + : rule.toBuilder().id(createId()).build(); + // enforce the title unique constraint + if (titleToId.containsKey(toSave.title())) { + // if this is an update and the title belongs to the passed rule, then it's fine + if (!titleToId.get(toSave.title()).equals(toSave.id())) { + throw new IllegalArgumentException("Duplicate rule titles are not allowed: " + toSave.title()); + } + } + titleToId.put(toSave.title(), toSave.id()); + store.put(toSave.id(), toSave); + + return toSave; + } + + @Override + public RuleDao load(String id) throws NotFoundException { + final RuleDao rule = store.get(id); + if (rule == null) { + throw new NotFoundException("No such rule with id " + id); + } + return rule; + } + + @Override + public Collection loadAll() { + return ImmutableSet.copyOf(store.values()); + } + + @Override + public void delete(String id) { + if (id == null) { + return; + } + final RuleDao removed = store.remove(id); + // clean up title index if the rule existed + if (removed != null) { + titleToId.remove(removed.title()); + } + } + + @Override + public Collection loadNamed(Collection ruleNames) { + final Set needles = Sets.newHashSet(ruleNames); + return store.values().stream() + .filter(ruleDao -> needles.contains(ruleDao.title())) + .collect(Collectors.toList()); + } + + private String createId() { + return String.valueOf(idGen.incrementAndGet()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryServicesModule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryServicesModule.java new file mode 100644 index 000000000000..537b569eb822 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryServicesModule.java @@ -0,0 +1,31 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.memory; + +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.plugin.PluginModule; + +public class InMemoryServicesModule extends PluginModule { + @Override + protected void configure() { + bind(RuleService.class).to(InMemoryRuleService.class).asEagerSingleton(); + bind(PipelineService.class).to(InMemoryPipelineService.class).asEagerSingleton(); + bind(PipelineStreamConnectionsService.class).to(InMemoryPipelineStreamConnectionsService.class).asEagerSingleton(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java new file mode 100644 index 000000000000..0544a18a2668 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java @@ -0,0 +1,85 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.mongodb; + +import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoException; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoConnection; +import org.graylog2.database.NotFoundException; +import org.mongojack.DBCursor; +import org.mongojack.DBSort; +import org.mongojack.JacksonDBCollection; +import org.mongojack.WriteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.Collections; + +public class MongoDbPipelineService implements PipelineService { + private static final Logger log = LoggerFactory.getLogger(MongoDbPipelineService.class); + + private static final String COLLECTION = "pipeline_processor_pipelines"; + + private final JacksonDBCollection dbCollection; + + @Inject + public MongoDbPipelineService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + dbCollection = JacksonDBCollection.wrap( + mongoConnection.getDatabase().getCollection(COLLECTION), + PipelineDao.class, + String.class, + mapper.get()); + dbCollection.createIndex(DBSort.asc("title"), new BasicDBObject("unique", true)); + } + + @Override + public PipelineDao save(PipelineDao pipeline) { + final WriteResult save = dbCollection.save(pipeline); + return save.getSavedObject(); + } + + @Override + public PipelineDao load(String id) throws NotFoundException { + final PipelineDao pipeline = dbCollection.findOneById(id); + if (pipeline == null) { + throw new NotFoundException("No pipeline with id " + id); + } + return pipeline; + } + + @Override + public Collection loadAll() { + try { + final DBCursor daos = dbCollection.find(); + return Sets.newHashSet(daos.iterator()); + } catch (MongoException e) { + log.error("Unable to load pipelines", e); + return Collections.emptySet(); + } + } + + @Override + public void delete(String id) { + dbCollection.removeById(id); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java new file mode 100644 index 000000000000..fb5e3d4450a2 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java @@ -0,0 +1,99 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.mongodb; + +import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoException; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoConnection; +import org.graylog2.database.NotFoundException; +import org.mongojack.DBCursor; +import org.mongojack.DBQuery; +import org.mongojack.DBSort; +import org.mongojack.JacksonDBCollection; +import org.mongojack.WriteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collections; +import java.util.Set; + +public class MongoDbPipelineStreamConnectionsService implements PipelineStreamConnectionsService { + private static final Logger log = LoggerFactory.getLogger(MongoDbPipelineStreamConnectionsService.class); + + private static final String COLLECTION = "pipeline_processor_pipelines_streams"; + + private final JacksonDBCollection dbCollection; + + @Inject + public MongoDbPipelineStreamConnectionsService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + dbCollection = JacksonDBCollection.wrap( + mongoConnection.getDatabase().getCollection(COLLECTION), + PipelineConnections.class, + String.class, + mapper.get()); + dbCollection.createIndex(DBSort.asc("stream_id"), new BasicDBObject("unique", true)); + } + + + @Override + public PipelineConnections save(PipelineConnections connections) { + PipelineConnections existingConnections = dbCollection.findOne(DBQuery.is("stream_id", connections.streamId())); + if (existingConnections == null) { + existingConnections = PipelineConnections.create(null, connections.streamId(), Collections.emptySet()); + } + + final PipelineConnections toSave = existingConnections.toBuilder() + .pipelineIds(connections.pipelineIds()).build(); + final WriteResult save = dbCollection.save(toSave); + return save.getSavedObject(); + } + + @Override + public PipelineConnections load(String streamId) throws NotFoundException { + final PipelineConnections oneById = dbCollection.findOne(DBQuery.is("stream_id", streamId)); + if (oneById == null) { + throw new NotFoundException("No pipeline connections with for stream " + streamId); + } + return oneById; + } + + @Override + public Set loadAll() { + try { + final DBCursor connections = dbCollection.find(); + return Sets.newHashSet(connections.iterator()); + } catch (MongoException e) { + log.error("Unable to load pipeline connections", e); + return Collections.emptySet(); + } + } + + @Override + public void delete(String streamId) { + try { + final PipelineConnections connections = load(streamId); + dbCollection.removeById(connections.id()); + } catch (NotFoundException e) { + log.debug("No connections found for stream " + streamId); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java new file mode 100644 index 000000000000..25b45e040c28 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java @@ -0,0 +1,104 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoException; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.database.MongoConnection; +import org.graylog2.database.NotFoundException; +import org.mongojack.DBCursor; +import org.mongojack.DBQuery; +import org.mongojack.DBSort; +import org.mongojack.JacksonDBCollection; +import org.mongojack.WriteResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.Collections; + +/** + * A RuleService backed by a MongoDB collection. + */ +public class MongoDbRuleService implements RuleService { + private static final Logger log = LoggerFactory.getLogger(MongoDbRuleService.class); + + private static final String COLLECTION = "pipeline_processor_rules"; + + private final JacksonDBCollection dbCollection; + + @Inject + public MongoDbRuleService(MongoConnection mongoConnection, MongoJackObjectMapperProvider mapper) { + dbCollection = JacksonDBCollection.wrap( + mongoConnection.getDatabase().getCollection(COLLECTION), + RuleDao.class, + String.class, + mapper.get()); + dbCollection.createIndex(DBSort.asc("title"), new BasicDBObject("unique", true)); + } + + @Override + public RuleDao save(RuleDao rule) { + final WriteResult save = dbCollection.save(rule); + return save.getSavedObject(); + } + + @Override + public RuleDao load(String id) throws NotFoundException { + final RuleDao rule = dbCollection.findOneById(id); + if (rule == null) { + throw new NotFoundException("No rule with id " + id); + } + return rule; + } + + @Override + public Collection loadAll() { + try { + final DBCursor ruleDaos = dbCollection.find().sort(DBSort.asc("title")); + return ruleDaos.toArray(); + } catch (MongoException e) { + log.error("Unable to load processing rules", e); + return Collections.emptySet(); + } + } + + @Override + public void delete(String id) { + final WriteResult result = dbCollection.removeById(id); + if (result.getN() != 1) { + log.error("Unable to delete rule {}", id); + } + } + + @Override + public Collection loadNamed(Collection ruleNames) { + try { + final DBCursor ruleDaos = dbCollection.find(DBQuery.in("title", ruleNames)); + return Sets.newHashSet(ruleDaos.iterator()); + } catch (MongoException e) { + log.error("Unable to bulk load rules", e); + return Collections.emptySet(); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbServicesModule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbServicesModule.java new file mode 100644 index 000000000000..3d70d78d460f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbServicesModule.java @@ -0,0 +1,31 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.mongodb; + +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.plugin.PluginModule; + +public class MongoDbServicesModule extends PluginModule { + @Override + protected void configure() { + bind(PipelineService.class).to(MongoDbPipelineService.class); + bind(RuleService.class).to(MongoDbRuleService.class); + bind(PipelineStreamConnectionsService.class).to(MongoDbPipelineStreamConnectionsService.class); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/LegacyDefaultStreamMigrated.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/LegacyDefaultStreamMigrated.java new file mode 100644 index 000000000000..bb2b17629918 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/LegacyDefaultStreamMigrated.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +@AutoValue +@JsonAutoDetect +public abstract class LegacyDefaultStreamMigrated { + @JsonProperty + public abstract boolean migrationDone(); + + @JsonCreator + public static LegacyDefaultStreamMigrated create(@JsonProperty("migration_done") boolean migrationDone) { + return new AutoValue_LegacyDefaultStreamMigrated(migrationDone); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java new file mode 100644 index 000000000000..8d4c5eb13c22 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java @@ -0,0 +1,40 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import java.util.Set; + +@JsonAutoDetect +@AutoValue +public abstract class PipelineConnectionsChangedEvent { + @JsonProperty("stream_id") + public abstract String streamId(); + + @JsonProperty("pipeline_ids") + public abstract Set pipelineIds(); + + @JsonCreator + public static PipelineConnectionsChangedEvent create(@JsonProperty("stream_id") String streamId, + @JsonProperty("pipeline_ids") Set pipelineIds) { + return new AutoValue_PipelineConnectionsChangedEvent(streamId, pipelineIds); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java new file mode 100644 index 000000000000..ef695da56caf --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelinesChangedEvent.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import com.google.common.collect.Sets; + +import java.util.Set; + +import static java.util.Collections.emptySet; + +@AutoValue +public abstract class PipelinesChangedEvent { + + @JsonProperty + public abstract Set deletedPipelineIds(); + + @JsonProperty + public abstract Set updatedPipelineIds(); + + public static Builder builder() { + return new AutoValue_PipelinesChangedEvent.Builder().deletedPipelineIds(emptySet()).updatedPipelineIds(emptySet()); + } + + public static PipelinesChangedEvent updatedPipelineId(String id) { + return builder().updatedPipelineId(id).build(); + } + + public static PipelinesChangedEvent deletedPipelineId(String id) { + return builder().deletedPipelineId(id).build(); + } + + @JsonCreator + public static PipelinesChangedEvent create(@JsonProperty("deleted_pipeline_ids") Set deletedIds, @JsonProperty("updated_pipeline_ids") Set updatedIds) { + return builder().deletedPipelineIds(deletedIds).updatedPipelineIds(updatedIds).build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder deletedPipelineIds(Set ids); + public Builder deletedPipelineId(String id) { + return deletedPipelineIds(Sets.newHashSet(id)); + } + public abstract Builder updatedPipelineIds(Set ids); + public Builder updatedPipelineId(String id) { + return updatedPipelineIds(Sets.newHashSet(id)); + } + public abstract PipelinesChangedEvent build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/RulesChangedEvent.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/RulesChangedEvent.java new file mode 100644 index 000000000000..7ba7832306ff --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/events/RulesChangedEvent.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.events; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import com.google.common.collect.Sets; + +import java.util.Set; + +import static java.util.Collections.emptySet; + +@AutoValue +public abstract class RulesChangedEvent { + + @JsonProperty + public abstract Set deletedRuleIds(); + + @JsonProperty + public abstract Set updatedRuleIds(); + + public static Builder builder() { + return new AutoValue_RulesChangedEvent.Builder().deletedRuleIds(emptySet()).updatedRuleIds(emptySet()); + } + + public static RulesChangedEvent updatedRuleId(String id) { + return builder().updatedRuleId(id).build(); + } + + public static RulesChangedEvent deletedRuleId(String id) { + return builder().deletedRuleId(id).build(); + } + + @JsonCreator + public static RulesChangedEvent create(@JsonProperty("deleted_rule_ids") Set deletedIds, @JsonProperty("updated_rule_ids") Set updatedIds) { + return builder().deletedRuleIds(deletedIds).updatedRuleIds(updatedIds).build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder deletedRuleIds(Set ids); + public Builder deletedRuleId(String id) { + return deletedRuleIds(Sets.newHashSet(id)); + } + public abstract Builder updatedRuleIds(Set ids); + public Builder updatedRuleId(String id) { + return updatedRuleIds(Sets.newHashSet(id)); + } + public abstract RulesChangedEvent build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java new file mode 100644 index 000000000000..d29fa4ebf905 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/FromInput.java @@ -0,0 +1,89 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.IOState; +import org.graylog2.plugin.inputs.MessageInput; +import org.graylog2.shared.inputs.InputRegistry; + +import javax.inject.Inject; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class FromInput extends AbstractFunction { + + public static final String NAME = "from_input"; + public static final String ID_ARG = "id"; + public static final String NAME_ARG = "name"; + + private final InputRegistry inputRegistry; + private final ParameterDescriptor idParam; + private final ParameterDescriptor nameParam; + + @Inject + public FromInput(InputRegistry inputRegistry) { + this.inputRegistry = inputRegistry; + idParam = string(ID_ARG).optional().description("The input's ID, this is much faster than 'name'").build(); + nameParam = string(NAME_ARG).optional().description("The input's name").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + String id = idParam.optional(args, context).orElse(""); + + MessageInput input = null; + if ("".equals(id)) { + final String name = nameParam.optional(args, context).orElse(""); + for (IOState messageInputIOState : inputRegistry.getInputStates()) { + final MessageInput messageInput = messageInputIOState.getStoppable(); + if (messageInput.getTitle().equalsIgnoreCase(name)) { + input = messageInput; + break; + } + } + if ("".equals(name)) { + return null; + } + } else { + final IOState inputState = inputRegistry.getInputState(id); + if (inputState != null) { + input = inputState.getStoppable(); + } + + } + return input != null + && input.getId().equals(context.currentMessage().getSourceInputId()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + idParam, + nameParam)) + .description("Checks if a message arrived on a given input") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java new file mode 100644 index 000000000000..7c41654c816c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNotNull.java @@ -0,0 +1,55 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class IsNotNull extends AbstractFunction { + + public static final String NAME = "is_not_null"; + private final ParameterDescriptor valueParam; + + public IsNotNull() { + valueParam = ParameterDescriptor.type("value", Object.class).description("The value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + try { + final Object value = valueParam.required(args, context); + return value != null; + } catch (Exception e) { + return false; + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(valueParam)) + .description("Checks whether a value is not 'null'") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java new file mode 100644 index 000000000000..e41d42762ee5 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/IsNull.java @@ -0,0 +1,55 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class IsNull extends AbstractFunction { + + public static final String NAME = "is_null"; + private final ParameterDescriptor valueParam; + + public IsNull() { + valueParam = ParameterDescriptor.type("value", Object.class).description("The value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + try { + final Object value = valueParam.required(args, context); + return value == null; + } catch (Exception e) { + return true; + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(valueParam)) + .description("Checks whether a value is 'null'") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java new file mode 100644 index 000000000000..ad54967c7c38 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java @@ -0,0 +1,257 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.google.inject.Binder; +import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.conversion.BooleanConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsBoolean; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsCollection; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsDouble; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsList; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsLong; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsMap; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsNumber; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsString; +import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.MapConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.DateConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.IsDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.Now; +import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.ParseUnixMilliseconds; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Days; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Hours; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.IsPeriod; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Millis; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Minutes; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Months; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.PeriodParseFunction; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Seconds; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Weeks; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Years; +import org.graylog.plugins.pipelineprocessor.functions.debug.Debug; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base16Decode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base16Encode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32Decode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32Encode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32HumanDecode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32HumanEncode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64Decode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64Encode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64UrlDecode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64UrlEncode; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32C; +import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_128; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_32; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; +import org.graylog.plugins.pipelineprocessor.functions.ips.CidrMatch; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddressConversion; +import org.graylog.plugins.pipelineprocessor.functions.ips.IsIp; +import org.graylog.plugins.pipelineprocessor.functions.json.IsJson; +import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; +import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; +import org.graylog.plugins.pipelineprocessor.functions.lookup.Lookup; +import org.graylog.plugins.pipelineprocessor.functions.lookup.LookupValue; +import org.graylog.plugins.pipelineprocessor.functions.messages.CloneMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveFromStream; +import org.graylog.plugins.pipelineprocessor.functions.messages.RenameField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; +import org.graylog.plugins.pipelineprocessor.functions.messages.StreamCacheService; +import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; +import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Concat; +import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; +import org.graylog.plugins.pipelineprocessor.functions.strings.EndsWith; +import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.KeyValue; +import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.Split; +import org.graylog.plugins.pipelineprocessor.functions.strings.StartsWith; +import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; +import org.graylog.plugins.pipelineprocessor.functions.strings.Swapcase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogFacilityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogLevelConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityToStringConversion; +import org.graylog.plugins.pipelineprocessor.functions.urls.IsUrl; +import org.graylog.plugins.pipelineprocessor.functions.urls.UrlConversion; +import org.graylog2.plugin.PluginModule; + +public class ProcessorFunctionsModule extends PluginModule { + @Override + protected void configure() { + // built-in functions + addMessageProcessorFunction(BooleanConversion.NAME, BooleanConversion.class); + addMessageProcessorFunction(DoubleConversion.NAME, DoubleConversion.class); + addMessageProcessorFunction(LongConversion.NAME, LongConversion.class); + addMessageProcessorFunction(StringConversion.NAME, StringConversion.class); + addMessageProcessorFunction(MapConversion.NAME, MapConversion.class); + + // Comparison functions + addMessageProcessorFunction(IsBoolean.NAME, IsBoolean.class); + addMessageProcessorFunction(IsNumber.NAME, IsNumber.class); + addMessageProcessorFunction(IsDouble.NAME, IsDouble.class); + addMessageProcessorFunction(IsLong.NAME, IsLong.class); + addMessageProcessorFunction(IsString.NAME, IsString.class); + addMessageProcessorFunction(IsCollection.NAME, IsCollection.class); + addMessageProcessorFunction(IsList.NAME, IsList.class); + addMessageProcessorFunction(IsMap.NAME, IsMap.class); + addMessageProcessorFunction(IsDate.NAME, IsDate.class); + addMessageProcessorFunction(IsPeriod.NAME, IsPeriod.class); + addMessageProcessorFunction(IsIp.NAME, IsIp.class); + addMessageProcessorFunction(IsJson.NAME, IsJson.class); + addMessageProcessorFunction(IsUrl.NAME, IsUrl.class); + + // message related functions + addMessageProcessorFunction(HasField.NAME, HasField.class); + addMessageProcessorFunction(SetField.NAME, SetField.class); + addMessageProcessorFunction(SetFields.NAME, SetFields.class); + addMessageProcessorFunction(RenameField.NAME, RenameField.class); + addMessageProcessorFunction(RemoveField.NAME, RemoveField.class); + + addMessageProcessorFunction(DropMessage.NAME, DropMessage.class); + addMessageProcessorFunction(CreateMessage.NAME, CreateMessage.class); + addMessageProcessorFunction(CloneMessage.NAME, CloneMessage.class); + addMessageProcessorFunction(RemoveFromStream.NAME, RemoveFromStream.class); + addMessageProcessorFunction(RouteToStream.NAME, RouteToStream.class); + // helper service for route_to_stream + serviceBinder().addBinding().to(StreamCacheService.class).in(Scopes.SINGLETON); + + // input related functions + addMessageProcessorFunction(FromInput.NAME, FromInput.class); + + // generic functions + addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class); + addMessageProcessorFunction(GrokMatch.NAME, GrokMatch.class); + + // string functions + addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class); + addMessageProcessorFunction(Capitalize.NAME, Capitalize.class); + addMessageProcessorFunction(Contains.NAME, Contains.class); + addMessageProcessorFunction(EndsWith.NAME, EndsWith.class); + addMessageProcessorFunction(Lowercase.NAME, Lowercase.class); + addMessageProcessorFunction(Substring.NAME, Substring.class); + addMessageProcessorFunction(Swapcase.NAME, Swapcase.class); + addMessageProcessorFunction(Uncapitalize.NAME, Uncapitalize.class); + addMessageProcessorFunction(Uppercase.NAME, Uppercase.class); + addMessageProcessorFunction(Concat.NAME, Concat.class); + addMessageProcessorFunction(KeyValue.NAME, KeyValue.class); + addMessageProcessorFunction(Split.NAME, Split.class); + addMessageProcessorFunction(StartsWith.NAME, StartsWith.class); + + // json + addMessageProcessorFunction(JsonParse.NAME, JsonParse.class); + addMessageProcessorFunction(SelectJsonPath.NAME, SelectJsonPath.class); + + // dates + addMessageProcessorFunction(DateConversion.NAME, DateConversion.class); + addMessageProcessorFunction(Now.NAME, Now.class); + addMessageProcessorFunction(ParseDate.NAME, ParseDate.class); + addMessageProcessorFunction(ParseUnixMilliseconds.NAME, ParseUnixMilliseconds.class); + addMessageProcessorFunction(FlexParseDate.NAME, FlexParseDate.class); + addMessageProcessorFunction(FormatDate.NAME, FormatDate.class); + addMessageProcessorFunction(Years.NAME, Years.class); + addMessageProcessorFunction(Months.NAME, Months.class); + addMessageProcessorFunction(Weeks.NAME, Weeks.class); + addMessageProcessorFunction(Days.NAME, Days.class); + addMessageProcessorFunction(Hours.NAME, Hours.class); + addMessageProcessorFunction(Minutes.NAME, Minutes.class); + addMessageProcessorFunction(Seconds.NAME, Seconds.class); + addMessageProcessorFunction(Millis.NAME, Millis.class); + addMessageProcessorFunction(PeriodParseFunction.NAME, PeriodParseFunction.class); + + // hash digest + addMessageProcessorFunction(CRC32.NAME, CRC32.class); + addMessageProcessorFunction(CRC32C.NAME, CRC32C.class); + addMessageProcessorFunction(MD5.NAME, MD5.class); + addMessageProcessorFunction(Murmur3_32.NAME, Murmur3_32.class); + addMessageProcessorFunction(Murmur3_128.NAME, Murmur3_128.class); + addMessageProcessorFunction(SHA1.NAME, SHA1.class); + addMessageProcessorFunction(SHA256.NAME, SHA256.class); + addMessageProcessorFunction(SHA512.NAME, SHA512.class); + + // encoding + addMessageProcessorFunction(Base16Encode.NAME, Base16Encode.class); + addMessageProcessorFunction(Base16Decode.NAME, Base16Decode.class); + addMessageProcessorFunction(Base32Encode.NAME, Base32Encode.class); + addMessageProcessorFunction(Base32Decode.NAME, Base32Decode.class); + addMessageProcessorFunction(Base32HumanEncode.NAME, Base32HumanEncode.class); + addMessageProcessorFunction(Base32HumanDecode.NAME, Base32HumanDecode.class); + addMessageProcessorFunction(Base64Encode.NAME, Base64Encode.class); + addMessageProcessorFunction(Base64Decode.NAME, Base64Decode.class); + addMessageProcessorFunction(Base64UrlEncode.NAME, Base64UrlEncode.class); + addMessageProcessorFunction(Base64UrlDecode.NAME, Base64UrlDecode.class); + + // ip handling + addMessageProcessorFunction(CidrMatch.NAME, CidrMatch.class); + addMessageProcessorFunction(IpAddressConversion.NAME, IpAddressConversion.class); + + // null support + addMessageProcessorFunction(IsNull.NAME, IsNull.class); + addMessageProcessorFunction(IsNotNull.NAME, IsNotNull.class); + + // URL parsing + addMessageProcessorFunction(UrlConversion.NAME, UrlConversion.class); + + // Syslog support + addMessageProcessorFunction(SyslogFacilityConversion.NAME, SyslogFacilityConversion.class); + addMessageProcessorFunction(SyslogLevelConversion.NAME, SyslogLevelConversion.class); + addMessageProcessorFunction(SyslogPriorityConversion.NAME, SyslogPriorityConversion.class); + addMessageProcessorFunction(SyslogPriorityToStringConversion.NAME, SyslogPriorityToStringConversion.class); + + // Lookup tables + addMessageProcessorFunction(Lookup.NAME, Lookup.class); + addMessageProcessorFunction(LookupValue.NAME, LookupValue.class); + + // Debug + addMessageProcessorFunction(Debug.NAME, Debug.class); + } + + protected void addMessageProcessorFunction(String name, Class> functionClass) { + addMessageProcessorFunction(binder(), name, functionClass); + } + + public static MapBinder> processorFunctionBinder(Binder binder) { + return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral>() {}); + } + + public static void addMessageProcessorFunction(Binder binder, String name, Class> functionClass) { + processorFunctionBinder(binder).addBinding(name).to(functionClass); + + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java new file mode 100644 index 000000000000..ddfbca7275c7 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/BooleanConversion.java @@ -0,0 +1,59 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.bool; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class BooleanConversion extends AbstractFunction { + public static final String NAME = "to_bool"; + + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + + public BooleanConversion() { + valueParam = object("value").description("Value to convert").build(); + defaultParam = bool("default").optional().description("Used when 'value' is null, defaults to false").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + if (value == null) { + return defaultParam.optional(args, context).orElse(false); + } + return Boolean.parseBoolean(String.valueOf(value)); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of(valueParam, defaultParam)) + .description("Converts a value to a boolean value using its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java new file mode 100644 index 000000000000..e8942cfdcaeb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/DoubleConversion.java @@ -0,0 +1,72 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import com.google.common.primitives.Doubles; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.floating; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class DoubleConversion extends AbstractFunction { + + public static final String NAME = "to_double"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public DoubleConversion() { + valueParam = object(VALUE).description("Value to convert").build(); + defaultParam = floating(DEFAULT).optional().description("Used when 'value' is null, defaults to 0").build(); + } + + @Override + public Double evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = valueParam.required(args, context); + final Double defaultValue = defaultParam.optional(args, context).orElse(0d); + + if (evaluated == null) { + return defaultValue; + } else if (evaluated instanceof Number) { + return ((Number) evaluated).doubleValue(); + } else { + final String s = String.valueOf(evaluated); + return firstNonNull(Doubles.tryParse(s), defaultValue); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Double.class) + .params(of( + valueParam, + defaultParam + )) + .description("Converts a value to a double value using its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsBoolean.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsBoolean.java new file mode 100644 index 000000000000..0930a8594bd5 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsBoolean.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsBoolean extends AbstractFunction { + public static final String NAME = "is_bool"; + + private final ParameterDescriptor valueParam; + + public IsBoolean() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Boolean; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a boolean") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsCollection.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsCollection.java new file mode 100644 index 000000000000..c5e2fffe8a04 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsCollection.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Collection; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsCollection extends AbstractFunction { + public static final String NAME = "is_collection"; + + private final ParameterDescriptor valueParam; + + public IsCollection() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Collection; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a collection") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsDouble.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsDouble.java new file mode 100644 index 000000000000..f4aacff40c88 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsDouble.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsDouble extends AbstractFunction { + public static final String NAME = "is_double"; + + private final ParameterDescriptor valueParam; + + public IsDouble() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Double; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a double") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsList.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsList.java new file mode 100644 index 000000000000..05543edd3020 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsList.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.List; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsList extends AbstractFunction { + public static final String NAME = "is_list"; + + private final ParameterDescriptor valueParam; + + public IsList() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof List; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a list") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsLong.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsLong.java new file mode 100644 index 000000000000..fea93da9ca44 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsLong.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsLong extends AbstractFunction { + public static final String NAME = "is_long"; + + private final ParameterDescriptor valueParam; + + public IsLong() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Long; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a long integer") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsMap.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsMap.java new file mode 100644 index 000000000000..c65ca2ebdb75 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsMap.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Map; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsMap extends AbstractFunction { + public static final String NAME = "is_map"; + + private final ParameterDescriptor valueParam; + + public IsMap() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Map; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a map") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsNumber.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsNumber.java new file mode 100644 index 000000000000..a4926a6d855b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsNumber.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsNumber extends AbstractFunction { + public static final String NAME = "is_number"; + + private final ParameterDescriptor valueParam; + + public IsNumber() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Number; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a number") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsString.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsString.java new file mode 100644 index 000000000000..813e5683a47d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/IsString.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsString extends AbstractFunction { + public static final String NAME = "is_string"; + + private final ParameterDescriptor valueParam; + + public IsString() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof String; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a string") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java new file mode 100644 index 000000000000..e210238c3e4b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/LongConversion.java @@ -0,0 +1,73 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.collect.ImmutableList.of; +import static com.google.common.primitives.Longs.tryParse; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.integer; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class LongConversion extends AbstractFunction { + + public static final String NAME = "to_long"; + + private static final String VALUE = "value"; + private static final String DEFAULT = "default"; + + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public LongConversion() { + valueParam = object(VALUE).description("Value to convert").build(); + defaultParam = integer(DEFAULT).optional().description("Used when 'value' is null, defaults to 0").build(); + } + + @Override + public Long evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = valueParam.required(args, context); + final Long defaultValue = defaultParam.optional(args, context).orElse(0L); + + if (evaluated == null) { + return defaultValue; + } else if (evaluated instanceof Number) { + return ((Number) evaluated).longValue(); + } else { + final String s = String.valueOf(evaluated); + return firstNonNull(tryParse(s), defaultValue); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Long.class) + .params(of( + valueParam, + defaultParam + )) + .description("Converts a value to a long value using its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/MapConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/MapConversion.java new file mode 100644 index 000000000000..d288b25ef759 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/MapConversion.java @@ -0,0 +1,71 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Collections; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class MapConversion extends AbstractFunction { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static final String NAME = "to_map"; + private static final String VALUE = "value"; + + private final ParameterDescriptor valueParam; + + + public MapConversion() { + this.valueParam = object(VALUE).description("Map-like value to convert").build(); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + + if (value == null) { + return Collections.emptyMap(); + } else if (value instanceof Map) { + return (Map) value; + } else if (value instanceof JsonNode) { + final JsonNode jsonNode = (JsonNode) value; + return MAPPER.convertValue(jsonNode, Map.class); + } else { + return Collections.emptyMap(); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Map.class) + .params(of(valueParam)) + .description("Converts a map-like value into a map usable by set_fields()") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java new file mode 100644 index 000000000000..0af8332dbfba --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/conversion/StringConversion.java @@ -0,0 +1,108 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.conversion; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddress; +import org.joda.time.DateTime; + +import java.util.LinkedHashMap; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class StringConversion extends AbstractFunction { + + public static final String NAME = "to_string"; + + // this is per-thread to save an expensive concurrent hashmap access + private final ThreadLocal, Class>> declaringClassCache; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public StringConversion() { + declaringClassCache = new ThreadLocal, Class>>() { + @Override + protected LinkedHashMap, Class> initialValue() { + return new LinkedHashMap, Class>() { + @Override + protected boolean removeEldestEntry(Map.Entry, Class> eldest) { + return size() > 1024; + } + }; + } + }; + valueParam = object("value").description("Value to convert").build(); + defaultParam = string("default").optional().description("Used when 'value' is null, defaults to \"\"").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object evaluated = valueParam.required(args, context); + if (evaluated == null) { + return defaultParam.optional(args, context).orElse(""); + } + // fast path for the most common targets + if (evaluated instanceof String + || evaluated instanceof Number + || evaluated instanceof Boolean + || evaluated instanceof DateTime + || evaluated instanceof IpAddress) { + return evaluated.toString(); + } else { + //noinspection Duplicates + try { + // slow path, we aren't sure that the object's class actually overrides toString() so we'll look it up. + final Class klass = evaluated.getClass(); + final LinkedHashMap, Class> classCache = declaringClassCache.get(); + + Class declaringClass = classCache.get(klass); + if (declaringClass == null) { + declaringClass = klass.getMethod("toString").getDeclaringClass(); + classCache.put(klass, declaringClass); + } + if ((declaringClass != Object.class)) { + return evaluated.toString(); + } else { + return defaultParam.optional(args, context).orElse(""); + } + } catch (NoSuchMethodException ignored) { + // should never happen because toString is always there + return defaultParam.optional(args, context).orElse(""); + } + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + valueParam, + defaultParam + )) + .description("Converts a value to its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateConversion.java new file mode 100644 index 000000000000..c2d418ab4e84 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateConversion.java @@ -0,0 +1,70 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.time.ZonedDateTime; +import java.util.Date; + +public class DateConversion extends TimezoneAwareFunction { + + public static final String NAME = "to_date"; + private final ParameterDescriptor value; + + public DateConversion() { + value = ParameterDescriptor.object("value").description("The value to convert to a date").build(); + } + + @Override + protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + final Object datish = value.required(args, context); + if (datish instanceof DateTime) { + return (DateTime) datish; + } + if (datish instanceof Date) { + return new DateTime(datish); + } + if (datish instanceof ZonedDateTime) { + final ZonedDateTime zonedDateTime = (ZonedDateTime) datish; + final DateTimeZone timeZone = DateTimeZone.forID(zonedDateTime.getZone().getId()); + return new DateTime(zonedDateTime.toInstant().toEpochMilli(), timeZone); + } + return null; + } + + @Override + protected String description() { + return "Converts a type to a date, useful for $message.timestamp or related message fields."; + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of(value); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java new file mode 100644 index 000000000000..b45e27441364 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FlexParseDate.java @@ -0,0 +1,77 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import com.joestelmach.natty.DateGroup; +import com.joestelmach.natty.Parser; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.util.List; +import java.util.Optional; + +public class FlexParseDate extends TimezoneAwareFunction { + + public static final String VALUE = "value"; + public static final String NAME = "flex_parse_date"; + public static final String DEFAULT = "default"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor defaultParam; + + public FlexParseDate() { + valueParam = ParameterDescriptor.string(VALUE).description("Date string to parse").build(); + defaultParam = ParameterDescriptor.type(DEFAULT, DateTime.class).optional().description("Used when 'value' could not be parsed, 'null' otherwise").build(); + } + + @Override + protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + final String time = valueParam.required(args, context); + + final List dates = new Parser(timezone.toTimeZone()).parse(time); + if (dates.size() == 0) { + final Optional defaultTime = defaultParam.optional(args, context); + if (defaultTime.isPresent()) { + return defaultTime.get(); + } + // TODO really? this should probably throw an exception of some sort to be handled in the interpreter + return null; + } + return new DateTime(dates.get(0).getDates().get(0), timezone); + } + + @Override + protected String description() { + return "Parses a date string using natural language (see http://natty.joestelmach.com/)"; + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of( + valueParam, + defaultParam + ); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java new file mode 100644 index 000000000000..e9e0574b9ee5 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/FormatDate.java @@ -0,0 +1,74 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import static com.google.common.collect.ImmutableList.of; + +public class FormatDate extends AbstractFunction { + + public static final String NAME = "format_date"; + + private final ParameterDescriptor value; + private final ParameterDescriptor format; + private final ParameterDescriptor timeZoneParam; + + public FormatDate() { + value = ParameterDescriptor.type("value", DateTime.class).description("The date to format").build(); + format = ParameterDescriptor.string("format", DateTimeFormatter.class) + .transform(DateTimeFormat::forPattern) + .description("The format string to use, see http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html") + .build(); + timeZoneParam = ParameterDescriptor.string("timezone", DateTimeZone.class) + .transform(DateTimeZone::forID) + .optional() + .description("The timezone to apply to the date, defaults to UTC") + .build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final DateTime dateTime = value.required(args, context); + final DateTimeFormatter formatter = format.required(args, context); + if (dateTime == null || formatter == null) { + return null; + } + final DateTimeZone timeZone = timeZoneParam.optional(args, context).orElse(DateTimeZone.UTC); + + return formatter.withZone(timeZone).print(dateTime); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of(value, format, timeZoneParam)) + .description("Formats a date using the given format string") + .build(); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/IsDate.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/IsDate.java new file mode 100644 index 000000000000..e341a75d7287 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/IsDate.java @@ -0,0 +1,54 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; + +import java.util.Date; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsDate extends AbstractFunction { + public static final String NAME = "is_date"; + + private final ParameterDescriptor valueParam; + + public IsDate() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Date || value instanceof DateTime; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a date") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java new file mode 100644 index 000000000000..c0c16ac4f18e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java @@ -0,0 +1,49 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +public class Now extends TimezoneAwareFunction { + + public static final String NAME = "now"; + + @Override + protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + return DateTime.now(timezone); + } + + @Override + protected String description() { + return "Returns the current time"; + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java new file mode 100644 index 000000000000..2454435a89e2 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseDate.java @@ -0,0 +1,86 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.Locale; +import java.util.Optional; + +public class ParseDate extends TimezoneAwareFunction { + public static final String NAME = "parse_date"; + + private static final String VALUE = "value"; + private static final String PATTERN = "pattern"; + private static final String LOCALE = "locale"; + + private final ParameterDescriptor valueParam; + private final ParameterDescriptor patternParam; + private final ParameterDescriptor localeParam; + + public ParseDate() { + valueParam = ParameterDescriptor.string(VALUE).description("Date string to parse").build(); + patternParam = ParameterDescriptor.string(PATTERN).description("The pattern to parse the date with, see http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html").build(); + localeParam = ParameterDescriptor.string(LOCALE).optional().description("The locale to parse the date with, see https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html").build(); + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of( + valueParam, + patternParam, + localeParam + ); + } + + @Override + public DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + final String dateString = valueParam.required(args, context); + final String pattern = patternParam.required(args, context); + final Optional localeString = localeParam.optional(args, context); + + if (dateString == null || pattern == null) { + return null; + } + + final Locale locale = localeString.map(Locale::forLanguageTag).orElse(Locale.getDefault()); + + final DateTimeFormatter formatter = DateTimeFormat + .forPattern(pattern) + .withLocale(locale) + .withZone(timezone); + + return formatter.parseDateTime(dateString); + } + + @Override + protected String description() { + return "Parses a date string using the given date format"; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseUnixMilliseconds.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseUnixMilliseconds.java new file mode 100644 index 000000000000..0b9cd12d48ff --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/ParseUnixMilliseconds.java @@ -0,0 +1,57 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +public class ParseUnixMilliseconds extends TimezoneAwareFunction { + public static final String NAME = "parse_unix_milliseconds"; + + private static final String VALUE = "value"; + + private final ParameterDescriptor valueParam; + + public ParseUnixMilliseconds() { + valueParam = ParameterDescriptor.integer(VALUE).description("UNIX millisecond timestamp to parse").build(); + } + + @Override + protected String getName() { + return NAME; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of(valueParam); + } + + @Override + public DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + final Long unixMillis = valueParam.required(args, context); + return unixMillis == null ? null : new DateTime(unixMillis, timezone); + } + + @Override + protected String description() { + return "Converts a UNIX millisecond timestamp into a date"; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java new file mode 100644 index 000000000000..ef06f09e7436 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/TimezoneAwareFunction.java @@ -0,0 +1,76 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.util.Locale; + +public abstract class TimezoneAwareFunction extends AbstractFunction { + + private static final String TIMEZONE = "timezone"; + private static final ImmutableMap UPPER_ZONE_MAP = Maps.uniqueIndex( + DateTimeZone.getAvailableIDs(), + input -> input != null ? input.toUpperCase(Locale.ENGLISH) : "UTC"); + private final ParameterDescriptor timeZoneParam; + + protected TimezoneAwareFunction() { + timeZoneParam = ParameterDescriptor + .string(TIMEZONE, DateTimeZone.class) + .transform(id -> DateTimeZone.forID(UPPER_ZONE_MAP.getOrDefault(id.toUpperCase(Locale.ENGLISH), "UTC"))) + .optional() + .description("The timezone to apply to the date, defaults to UTC") + .build(); + } + + @Override + public DateTime evaluate(FunctionArgs args, EvaluationContext context) { + final DateTimeZone timezone = timeZoneParam.optional(args, context).orElse(DateTimeZone.UTC); + + return evaluate(args, context, timezone); + } + + protected abstract DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone); + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(getName()) + .returnType(DateTime.class) + .params(ImmutableList.builder() + .addAll(params()) + .add(timeZoneParam) + .build()) + .description(description()) + .build(); + } + + protected abstract String description(); + + protected abstract String getName(); + + protected abstract ImmutableList params(); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/AbstractPeriodComponentFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/AbstractPeriodComponentFunction.java new file mode 100644 index 000000000000..c7ebf3985e17 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/AbstractPeriodComponentFunction.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import com.google.common.primitives.Ints; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public abstract class AbstractPeriodComponentFunction extends AbstractFunction { + + private final ParameterDescriptor value = + ParameterDescriptor + .integer("value", Period.class) + .transform(this::getPeriodOfInt) + .build(); + + private Period getPeriodOfInt(long period) { + return getPeriod(Ints.saturatedCast(period)); + } + + @Nonnull + protected abstract Period getPeriod(int period); + + @Override + public Period evaluate(FunctionArgs args, EvaluationContext context) { + return value.required(args, context); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(getName()) + .description(getDescription()) + .pure(true) + .returnType(Period.class) + .params(value) + .build(); + } + + @Nonnull + protected abstract String getName(); + + @Nonnull + protected abstract String getDescription(); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Days.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Days.java new file mode 100644 index 000000000000..21b107731b77 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Days.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Days extends AbstractPeriodComponentFunction { + + public static final String NAME = "days"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.days(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of days."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Hours.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Hours.java new file mode 100644 index 000000000000..7a7582bc371c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Hours.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Hours extends AbstractPeriodComponentFunction { + + public static final String NAME = "hours"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.hours(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of hours."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/IsPeriod.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/IsPeriod.java new file mode 100644 index 000000000000..a755191a2f76 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/IsPeriod.java @@ -0,0 +1,52 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.Period; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsPeriod extends AbstractFunction { + public static final String NAME = "is_period"; + + private final ParameterDescriptor valueParam; + + public IsPeriod() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof Period; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a time period") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Millis.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Millis.java new file mode 100644 index 000000000000..f9108b0872a1 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Millis.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Millis extends AbstractPeriodComponentFunction { + + public static final String NAME = "millis"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.millis(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of millis."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Minutes.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Minutes.java new file mode 100644 index 000000000000..8594670e3cc7 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Minutes.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Minutes extends AbstractPeriodComponentFunction { + + public static final String NAME = "minutes"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.minutes(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of minutes."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Months.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Months.java new file mode 100644 index 000000000000..1832415db40b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Months.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Months extends AbstractPeriodComponentFunction { + + public static final String NAME = "months"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.months(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of months."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/PeriodParseFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/PeriodParseFunction.java new file mode 100644 index 000000000000..36cc33e1e658 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/PeriodParseFunction.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.joda.time.Period; + +public class PeriodParseFunction extends AbstractFunction { + + public static final String NAME = "period"; + private final ParameterDescriptor value = + ParameterDescriptor + .string("value", Period.class) + .transform(Period::parse) + .build(); + + + @Override + public Period evaluate(FunctionArgs args, EvaluationContext context) { + return value.required(args, context); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .description("Parses a ISO 8601 period from the specified string.") + .pure(true) + .returnType(Period.class) + .params(value) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Seconds.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Seconds.java new file mode 100644 index 000000000000..a6971687fa76 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Seconds.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Seconds extends AbstractPeriodComponentFunction { + + public static final String NAME = "seconds"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.seconds(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of seconds."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Weeks.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Weeks.java new file mode 100644 index 000000000000..7c3519af7cb5 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Weeks.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +public class Weeks extends AbstractPeriodComponentFunction { + + public static final String NAME = "weeks"; + + @Nonnull + @Override + protected Period getPeriod(int period) { + return Period.weeks(period); + } + + @Nonnull + @Override + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of weeks."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Years.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Years.java new file mode 100644 index 000000000000..083107d5726e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Years.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.dates.periods; + +import org.joda.time.Period; + +import javax.annotation.Nonnull; + +import static com.google.common.primitives.Ints.saturatedCast; + +public class Years extends AbstractPeriodComponentFunction { + + public static final String NAME = "years"; + + @Override + @Nonnull + protected Period getPeriod(int period) { + return Period.years(saturatedCast(period)); + } + + @Override + @Nonnull + protected String getName() { + return NAME; + } + + @Nonnull + @Override + protected String getDescription() { + return "Create a period with a specified number of years."; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/debug/Debug.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/debug/Debug.java new file mode 100644 index 000000000000..b7a44a5e3694 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/debug/Debug.java @@ -0,0 +1,61 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.debug; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Debug extends AbstractFunction { + + private final ParameterDescriptor valueParam; + + public static final String NAME = "debug"; + + Debug() { + valueParam = ParameterDescriptor.object("value").description("The value to print in the graylog-server log.").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + + if(value == null) { + log.info("PIPELINE DEBUG: Passed value is NULL."); + } else { + log.info("PIPELINE DEBUG: {}", value.toString()); + } + + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of(valueParam) ) + .description("Print any passed value as string in the graylog-server log. Note that this will only appear in the " + + "log of the graylog-server node that is processing the message you are trying to debug.") + .build(); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Decode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Decode.java new file mode 100644 index 000000000000..5bf2a62b857c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Decode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base16Decode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base16_decode"; + private static final String ENCODING_NAME = "base16"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base16(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return new String(encoding.decode(value), StandardCharsets.UTF_8); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Encode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Encode.java new file mode 100644 index 000000000000..31260366bfdd --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Encode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base16Encode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base16_encode"; + private static final String ENCODING_NAME = "base16"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base16(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Decode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Decode.java new file mode 100644 index 000000000000..b0c95bd87f71 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Decode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base32Decode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base32_decode"; + private static final String ENCODING_NAME = "base32"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base32Hex(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return new String(encoding.decode(value), StandardCharsets.UTF_8); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Encode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Encode.java new file mode 100644 index 000000000000..09b669493b86 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Encode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base32Encode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base32_encode"; + private static final String ENCODING_NAME = "base32"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base32Hex(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanDecode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanDecode.java new file mode 100644 index 000000000000..acc7c8e6f51e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanDecode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base32HumanDecode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base32human_decode"; + private static final String ENCODING_NAME = "base32 (human-friendly)"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base32(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return new String(encoding.decode(value), StandardCharsets.UTF_8); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanEncode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanEncode.java new file mode 100644 index 000000000000..cf3fd74cf185 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanEncode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base32HumanEncode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base32human_encode"; + private static final String ENCODING_NAME = "base32 (human-friendly)"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base32(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Decode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Decode.java new file mode 100644 index 000000000000..64cab87eb535 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Decode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base64Decode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base64_decode"; + private static final String ENCODING_NAME = "base64"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base64(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return new String(encoding.decode(value), StandardCharsets.UTF_8); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Encode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Encode.java new file mode 100644 index 000000000000..3535da24fec7 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Encode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base64Encode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base64_encode"; + private static final String ENCODING_NAME = "base64"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base64(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlDecode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlDecode.java new file mode 100644 index 000000000000..ce4b0064e16c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlDecode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base64UrlDecode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base64url_decode"; + private static final String ENCODING_NAME = "base64 (URL-safe)"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base64Url(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return new String(encoding.decode(value), StandardCharsets.UTF_8); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlEncode.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlEncode.java new file mode 100644 index 000000000000..066304dbd62e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlEncode.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import com.google.common.io.BaseEncoding; + +import java.nio.charset.StandardCharsets; + +public class Base64UrlEncode extends BaseEncodingSingleArgStringFunction { + public static final String NAME = "base64url_encode"; + private static final String ENCODING_NAME = "base64 (URL-safe)"; + + @Override + protected String getEncodedValue(String value, boolean omitPadding) { + BaseEncoding encoding = BaseEncoding.base64Url(); + encoding = omitPadding ? encoding.omitPadding() : encoding; + + return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); + } + + @Override + protected String getEncodingName() { + return ENCODING_NAME; + } + + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/BaseEncodingSingleArgStringFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/BaseEncodingSingleArgStringFunction.java new file mode 100644 index 000000000000..e30e40183478 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/BaseEncodingSingleArgStringFunction.java @@ -0,0 +1,65 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.encoding; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Locale; + +import static com.google.common.collect.ImmutableList.of; + +abstract class BaseEncodingSingleArgStringFunction extends AbstractFunction { + + private final ParameterDescriptor valueParam; + private final ParameterDescriptor omitPaddingParam; + + BaseEncodingSingleArgStringFunction() { + valueParam = ParameterDescriptor.string("value").description("The value to encode with " + getEncodingName()).build(); + omitPaddingParam = ParameterDescriptor.bool("omit_padding").optional().description("Omit any padding characters as specified by RFC 4648 section 3.2").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final boolean omitPadding = omitPaddingParam.optional(args, context).orElse(false); + return getEncodedValue(value, omitPadding); + } + + protected abstract String getEncodedValue(String value, boolean omitPadding); + + protected abstract String getName(); + + protected abstract String getEncodingName(); + + protected String description() { + return getEncodingName() + " encoding/decoding of the string"; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(getName()) + .returnType(String.class) + .params(of(valueParam, omitPaddingParam)) + .description(description()) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java new file mode 100644 index 000000000000..74a75543e804 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class CRC32 extends SingleArgStringFunction { + + public static final String NAME = "crc32"; + + @Override + protected String getDigest(String value) { + return Hashing.crc32().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java new file mode 100644 index 000000000000..99e4fa8a90d9 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class CRC32C extends SingleArgStringFunction { + + public static final String NAME = "crc32c"; + + @Override + protected String getDigest(String value) { + return Hashing.crc32c().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java new file mode 100644 index 000000000000..513581d8fa68 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class MD5 extends SingleArgStringFunction { + + public static final String NAME = "md5"; + + @Override + protected String getDigest(String value) { + return DigestUtils.md5Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java new file mode 100644 index 000000000000..ef6b1ce50e73 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class Murmur3_128 extends SingleArgStringFunction { + + public static final String NAME = "murmur3_128"; + + @Override + protected String getDigest(String value) { + return Hashing.murmur3_128().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java new file mode 100644 index 000000000000..8774823d1789 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import com.google.common.hash.Hashing; + +import java.nio.charset.StandardCharsets; + +public class Murmur3_32 extends SingleArgStringFunction { + + public static final String NAME = "murmur3_32"; + + @Override + protected String getDigest(String value) { + return Hashing.murmur3_32().hashString(value, StandardCharsets.UTF_8).toString(); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java new file mode 100644 index 000000000000..9bfd820f34a6 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class SHA1 extends SingleArgStringFunction { + + public static final String NAME = "sha1"; + + @Override + protected String getDigest(String value) { + return DigestUtils.sha1Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java new file mode 100644 index 000000000000..c843c6fa7459 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class SHA256 extends SingleArgStringFunction { + + public static final String NAME = "sha256"; + + @Override + protected String getDigest(String value) { + return DigestUtils.sha256Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java new file mode 100644 index 000000000000..eee726822d8b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java @@ -0,0 +1,34 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.apache.commons.codec.digest.DigestUtils; + +public class SHA512 extends SingleArgStringFunction { + + public static final String NAME = "sha512"; + + @Override + protected String getDigest(String value) { + return DigestUtils.sha512Hex(value); + } + + @Override + protected String getName() { + return NAME; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java new file mode 100644 index 000000000000..ed835f5d1395 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SingleArgStringFunction.java @@ -0,0 +1,62 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.hashing; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Locale; + +import static com.google.common.collect.ImmutableList.of; + +abstract class SingleArgStringFunction extends AbstractFunction { + + private final ParameterDescriptor valueParam; + + SingleArgStringFunction() { + valueParam = ParameterDescriptor.string("value").description("The value to hash").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + return getDigest(value); + } + + protected abstract String getDigest(String value); + + protected abstract String getName(); + + protected String description() { + return getName().toUpperCase(Locale.ENGLISH) + " hash of the string"; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(getName()) + .returnType(String.class) + .params(of( + valueParam) + ) + .description(description()) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java new file mode 100644 index 000000000000..daa11be42bad --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/CidrMatch.java @@ -0,0 +1,71 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.utilities.IpSubnet; + +import java.net.UnknownHostException; + +import static com.google.common.collect.ImmutableList.of; + +public class CidrMatch extends AbstractFunction { + + public static final String NAME = "cidr_match"; + public static final String IP = "ip"; + + private final ParameterDescriptor cidrParam; + private final ParameterDescriptor ipParam; + + public CidrMatch() { + // a little ugly because newCIDR throws a checked exception :( + cidrParam = ParameterDescriptor.string("cidr", IpSubnet.class).transform(cidrString -> { + try { + return new IpSubnet(cidrString); + } catch (UnknownHostException e) { + throw new IllegalArgumentException(e); + } + }).description("The CIDR subnet mask").build(); + ipParam = ParameterDescriptor.type(IP, IpAddress.class).description("The parsed IP address to match against the CIDR mask").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final IpSubnet cidr = cidrParam.required(args, context); + final IpAddress ipAddress = ipParam.required(args, context); + if (cidr == null || ipAddress == null) { + return null; + } + return cidr.contains(ipAddress.inetAddress()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + cidrParam, + ipParam)) + .description("Checks if an IP address matches a CIDR subnet mask") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java new file mode 100644 index 000000000000..82b402e10e84 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddress.java @@ -0,0 +1,74 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import com.google.common.net.InetAddresses; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Objects; + +/** + * Graylog's rule language wrapper for {@link InetAddress}. + *
+ * The purpose of this class is to guard against accidentally accessing properties which can trigger name resolutions + * and to provide a known interface to deal with IP addresses. + *
+ * Almost all of the logic is in the actual {@link InetAddress} delegate object. + */ +public class IpAddress { + + private InetAddress address; + + public IpAddress(InetAddress address) { + this.address = address; + } + + public InetAddress inetAddress() { + return address; + } + + @Override + public String toString() { + return InetAddresses.toAddrString(address); + } + + @SuppressWarnings("unused") + public IpAddress getAnonymized() { + final byte[] address = this.address.getAddress(); + address[address.length-1] = 0x00; + try { + return new IpAddress(InetAddress.getByAddress(address)); + } catch (UnknownHostException e) { + // cannot happen, it's created from a valid InetAddress to begin with + throw new IllegalStateException(e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof IpAddress)) return false; + IpAddress ipAddress = (IpAddress) o; + return Objects.equals(address, ipAddress.address); + } + + @Override + public int hashCode() { + return Objects.hash(address); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java new file mode 100644 index 000000000000..4ce4794750db --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IpAddressConversion.java @@ -0,0 +1,78 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import com.google.common.net.InetAddresses; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.net.InetAddress; +import java.util.IllegalFormatException; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; + +public class IpAddressConversion extends AbstractFunction { + + public static final String NAME = "to_ip"; + private static final InetAddress ANYV4 = InetAddresses.forString("0.0.0.0"); + + private final ParameterDescriptor ipParam; + private final ParameterDescriptor defaultParam; + + public IpAddressConversion() { + ipParam = ParameterDescriptor.object("ip").description("Value to convert").build(); + defaultParam = ParameterDescriptor.string("default").optional().description("Used when 'ip' is null or malformed, defaults to '0.0.0.0'").build(); + } + + @Override + public IpAddress evaluate(FunctionArgs args, EvaluationContext context) { + final String ipString = String.valueOf(ipParam.required(args, context)); + + try { + final InetAddress inetAddress = InetAddresses.forString(ipString); + return new IpAddress(inetAddress); + } catch (IllegalArgumentException e) { + final Optional defaultValue = defaultParam.optional(args, context); + if (!defaultValue.isPresent()) { + return new IpAddress(ANYV4); + } + try { + return new IpAddress(InetAddresses.forString(defaultValue.get())); + } catch (IllegalFormatException e1) { + log.warn("Parameter `default` for to_ip() is not a valid IP address: {}", defaultValue.get()); + throw e1; + } + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(IpAddress.class) + .params(of( + ipParam, + defaultParam + )) + .description("Converts a value to an IPAddress using its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IsIp.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IsIp.java new file mode 100644 index 000000000000..006d6f32802b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ips/IsIp.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.ips; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsIp extends AbstractFunction { + public static final String NAME = "is_ip"; + + private final ParameterDescriptor valueParam; + + public IsIp() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof IpAddress; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is an IP address") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/IsJson.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/IsJson.java new file mode 100644 index 000000000000..fc0c6cc1841d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/IsJson.java @@ -0,0 +1,52 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.json; + +import com.fasterxml.jackson.databind.JsonNode; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsJson extends AbstractFunction { + public static final String NAME = "is_json"; + + private final ParameterDescriptor valueParam; + + public IsJson() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof JsonNode; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a JSON value") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java new file mode 100644 index 000000000000..83e4e1bef202 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/JsonParse.java @@ -0,0 +1,71 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.MissingNode; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +import java.io.IOException; + +import static com.google.common.collect.ImmutableList.of; + +public class JsonParse extends AbstractFunction { + private static final Logger log = LoggerFactory.getLogger(JsonParse.class); + public static final String NAME = "parse_json"; + + private final ObjectMapper objectMapper; + private final ParameterDescriptor valueParam; + + @Inject + public JsonParse(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + valueParam = ParameterDescriptor.string("value").description("The string to parse as a JSON tree").build(); + } + + @Override + public JsonNode evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + try { + return objectMapper.readTree(value); + } catch (IOException e) { + log.warn("Unable to parse JSON", e); + } + return MissingNode.getInstance(); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(JsonNode.class) + .params(of( + valueParam + )) + .description("Parses a string as a JSON tree") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java new file mode 100644 index 000000000000..18c962ae09ce --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/json/SelectJsonPath.java @@ -0,0 +1,133 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.inject.TypeLiteral; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; +import static java.util.stream.Collectors.toMap; + +public class SelectJsonPath extends AbstractFunction> { + + public static final String NAME = "select_jsonpath"; + + private final Configuration configuration; + private final ParameterDescriptor jsonParam; + private final ParameterDescriptor, Map> pathsParam; + + @Inject + public SelectJsonPath(ObjectMapper objectMapper) { + configuration = Configuration.builder() + .options(Option.SUPPRESS_EXCEPTIONS) + .jsonProvider(new JacksonJsonNodeJsonProvider(objectMapper)) + .build(); + + jsonParam = ParameterDescriptor.type("json", JsonNode.class).description("A parsed JSON tree").build(); + // sigh generics and type erasure + //noinspection unchecked + pathsParam = ParameterDescriptor.type("paths", + (Class>) new TypeLiteral>() {}.getRawType(), + (Class>) new TypeLiteral>() {}.getRawType()) + .transform(inputMap -> inputMap + .entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> JsonPath.compile(e.getValue())))) + .description("A map of names to a JsonPath expression, see http://jsonpath.com") + .build(); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + final JsonNode json = jsonParam.required(args, context); + final Map paths = pathsParam.required(args, context); + if (json == null || paths == null) { + return Collections.emptyMap(); + } + // a plain Stream.collect(toMap(...)) will fail on null values, because of the HashMap#merge method in its implementation + // since json nodes at certain paths might be missing, the value could be null, so we use HashMap#put directly + final Map map = new HashMap<>(); + for (Map.Entry entry : paths.entrySet()) { + map.put(entry.getKey(), unwrapJsonNode(entry.getValue().read(json, configuration))); + } + return map; + } + + @Nullable + private Object unwrapJsonNode(Object value) { + if (!(value instanceof JsonNode)) { + return value; + } + JsonNode read = ((JsonNode) value); + switch (read.getNodeType()) { + case ARRAY: + return ImmutableList.copyOf(read.elements()); + case BINARY: + try { + return read.binaryValue(); + } catch (IOException e) { + return null; + } + case BOOLEAN: + return read.booleanValue(); + case MISSING: + case NULL: + return null; + case NUMBER: + return read.numberValue(); + case OBJECT: + return read; + case POJO: + return read; + case STRING: + return read.textValue(); + } + return read; + } + + @Override + public FunctionDescriptor> descriptor() { + //noinspection unchecked + return FunctionDescriptor.>builder() + .name(NAME) + .returnType((Class>) new TypeLiteral>() {}.getRawType()) + .params(of( + jsonParam, + pathsParam + )) + .description("Selects a map of fields containing the result of their JsonPath expressions") + .build(); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/lookup/Lookup.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/lookup/Lookup.java new file mode 100644 index 000000000000..78a5b8ccac55 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/lookup/Lookup.java @@ -0,0 +1,86 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.lookup; + +import com.google.inject.Inject; +import com.google.inject.TypeLiteral; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.lookup.LookupTableService; +import org.graylog2.plugin.lookup.LookupResult; + +import java.util.Collections; +import java.util.Map; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog2.plugin.lookup.LookupResult.SINGLE_VALUE_KEY; + +public class Lookup extends AbstractFunction> { + + public static final String NAME = "lookup"; + + private final ParameterDescriptor lookupTableParam; + private final ParameterDescriptor keyParam; + private final ParameterDescriptor defaultParam; + + @Inject + public Lookup(LookupTableService lookupTableService) { + lookupTableParam = string("lookup_table", LookupTableService.Function.class) + .description("The existing lookup table to use to lookup the given key") + .transform(tableName -> lookupTableService.newBuilder().lookupTable(tableName).build()) + .build(); + keyParam = object("key") + .description("The key to lookup in the table") + .build(); + defaultParam = object("default") + .description("The default multi value that should be used if there is no lookup result") + .optional() + .build(); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + Object key = keyParam.required(args, context); + if (key == null) { + return Collections.singletonMap(SINGLE_VALUE_KEY, defaultParam.optional(args, context).orElse(null)); + } + LookupTableService.Function table = lookupTableParam.required(args, context); + if (table == null) { + return Collections.singletonMap(SINGLE_VALUE_KEY, defaultParam.optional(args, context).orElse(null)); + } + LookupResult result = table.lookup(key); + if (result == null || result.isEmpty()) { + return Collections.singletonMap(SINGLE_VALUE_KEY, defaultParam.optional(args, context).orElse(null)); + } + return result.multiValue(); + } + + @Override + public FunctionDescriptor> descriptor() { + //noinspection unchecked + return FunctionDescriptor.>builder() + .name(NAME) + .description("Looks up a multi value in the named lookup table.") + .params(lookupTableParam, keyParam, defaultParam) + .returnType((Class>) new TypeLiteral>() {}.getRawType()) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/lookup/LookupValue.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/lookup/LookupValue.java new file mode 100644 index 000000000000..43a2862215e4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/lookup/LookupValue.java @@ -0,0 +1,81 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.lookup; + +import com.google.inject.Inject; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.lookup.LookupTableService; +import org.graylog2.plugin.lookup.LookupResult; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class LookupValue extends AbstractFunction { + + public static final String NAME = "lookup_value"; + + private final ParameterDescriptor lookupTableParam; + private final ParameterDescriptor keyParam; + private final ParameterDescriptor defaultParam; + + @Inject + public LookupValue(LookupTableService lookupTableService) { + lookupTableParam = string("lookup_table", LookupTableService.Function.class) + .description("The existing lookup table to use to lookup the given key") + .transform(tableName -> lookupTableService.newBuilder().lookupTable(tableName).build()) + .build(); + keyParam = object("key") + .description("The key to lookup in the table") + .build(); + defaultParam = object("default") + .description("The default single value that should be used if there is no lookup result") + .optional() + .build(); + } + + @Override + public Object evaluate(FunctionArgs args, EvaluationContext context) { + Object key = keyParam.required(args, context); + if (key == null) { + return defaultParam.optional(args, context).orElse(null); + } + LookupTableService.Function table = lookupTableParam.required(args, context); + if (table == null) { + return defaultParam.optional(args, context).orElse(null); + } + LookupResult result = table.lookup(key); + if (result == null || result.isEmpty()) { + return defaultParam.optional(args, context).orElse(null); + } + return result.singleValue(); + } + + @Override + public FunctionDescriptor descriptor() { + //noinspection unchecked + return FunctionDescriptor.builder() + .name(NAME) + .description("Looks up a single value in the named lookup table.") + .params(lookupTableParam, keyParam, defaultParam) + .returnType(Object.class) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CloneMessage.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CloneMessage.java new file mode 100644 index 000000000000..e7e327fc7228 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CloneMessage.java @@ -0,0 +1,82 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class CloneMessage extends AbstractFunction { + private static final Logger LOG = LoggerFactory.getLogger(CloneMessage.class); + + public static final String NAME = "clone_message"; + + private final ParameterDescriptor messageParam; + + public CloneMessage() { + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + } + + @Override + public Message evaluate(FunctionArgs args, EvaluationContext context) { + final Message currentMessage = messageParam.optional(args, context).orElse(context.currentMessage()); + + final Object tsField = currentMessage.getField(Message.FIELD_TIMESTAMP); + final Message clonedMessage; + if (tsField instanceof DateTime) { + clonedMessage = new Message(currentMessage.getMessage(), currentMessage.getSource(), currentMessage.getTimestamp()); + clonedMessage.addFields(currentMessage.getFields()); + } else { + LOG.warn("Invalid timestamp <{}> (type: {}) in message <{}>. Using current time instead.", + tsField, tsField.getClass().getCanonicalName(), currentMessage.getId()); + + final DateTime now = DateTime.now(DateTimeZone.UTC); + clonedMessage = new Message(currentMessage.getMessage(), currentMessage.getSource(), now); + clonedMessage.addFields(currentMessage.getFields()); + + // Message#addFields() overwrites the "timestamp" field. + clonedMessage.addField("timestamp", now); + clonedMessage.addField("gl2_original_timestamp", String.valueOf(tsField)); + } + + clonedMessage.addStreams(currentMessage.getStreams()); + + // register in context so the processor can inject it later on + context.addCreatedMessage(clonedMessage); + return clonedMessage; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .params(ImmutableList.of(messageParam)) + .returnType(Message.class) + .description("Clones a message") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java new file mode 100644 index 000000000000..26fff265ed6a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/CreateMessage.java @@ -0,0 +1,82 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Tools; +import org.joda.time.DateTime; + +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class CreateMessage extends AbstractFunction { + + public static final String NAME = "create_message"; + + private static final String MESSAGE_ARG = "message"; + private static final String SOURCE_ARG = "source"; + private static final String TIMESTAMP_ARG = "timestamp"; + private final ParameterDescriptor messageParam; + private final ParameterDescriptor sourceParam; + private final ParameterDescriptor timestampParam; + + public CreateMessage() { + messageParam = string(MESSAGE_ARG).optional().description("The 'message' field of the new message, defaults to '$message.message'").build(); + sourceParam = string(SOURCE_ARG).optional().description("The 'source' field of the new message, defaults to '$message.source'").build(); + timestampParam = type(TIMESTAMP_ARG, DateTime.class).optional().description("The 'timestamp' field of the message, defaults to 'now'").build(); + } + + @Override + public Message evaluate(FunctionArgs args, EvaluationContext context) { + final Optional optMessage = messageParam.optional(args, context); + final String message = optMessage.isPresent() ? optMessage.get() : context.currentMessage().getMessage(); + + final Optional optSource = sourceParam.optional(args, context); + final String source = optSource.isPresent() ? optSource.get() : context.currentMessage().getSource(); + + final Optional optTimestamp = timestampParam.optional(args, context); + final DateTime timestamp = optTimestamp.isPresent() ? optTimestamp.get() : Tools.nowUTC(); + + final Message newMessage = new Message(message, source, timestamp); + + // register in context so the processor can inject it later on + context.addCreatedMessage(newMessage); + return newMessage; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Message.class) + .params(of( + messageParam, + sourceParam, + timestampParam + )) + .description("Creates a new message") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java new file mode 100644 index 000000000000..a0cd197fa008 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/DropMessage.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class DropMessage extends AbstractFunction { + + public static final String NAME = "drop_message"; + public static final String MESSAGE_ARG = "message"; + private final ParameterDescriptor messageParam; + + public DropMessage() { + messageParam = type(MESSAGE_ARG, Message.class).optional().description("The message to drop, defaults to '$message'").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + message.setFilterOut(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .pure(true) + .returnType(Void.class) + .params(ImmutableList.of( + messageParam + )) + .description("Discards a message from further processing") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java new file mode 100644 index 000000000000..cebda2cb6192 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/HasField.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class HasField extends AbstractFunction { + + public static final String NAME = "has_field"; + public static final String FIELD = "field"; + private final ParameterDescriptor fieldParam; + private final ParameterDescriptor messageParam; + + public HasField() { + fieldParam = ParameterDescriptor.string(FIELD).description("The field to check").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final String field = fieldParam.required(args, context); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + + return message.hasField(field); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(ImmutableList.of(fieldParam, messageParam)) + .description("Checks whether a message contains a value for a field") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java new file mode 100644 index 000000000000..a38dc1ad4d12 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveField.java @@ -0,0 +1,59 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class RemoveField extends AbstractFunction { + + public static final String NAME = "remove_field"; + public static final String FIELD = "field"; + private final ParameterDescriptor fieldParam; + private final ParameterDescriptor messageParam; + + public RemoveField() { + fieldParam = ParameterDescriptor.string(FIELD).description("The field to remove").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final String field = fieldParam.required(args, context); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + + message.removeField(field); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(ImmutableList.of(fieldParam, messageParam)) + .description("Removes a field from a message") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveFromStream.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveFromStream.java new file mode 100644 index 000000000000..e632f59351e9 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RemoveFromStream.java @@ -0,0 +1,105 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.inject.Inject; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.streams.DefaultStream; +import org.graylog2.plugin.streams.Stream; + +import javax.inject.Provider; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class RemoveFromStream extends AbstractFunction { + + public static final String NAME = "remove_from_stream"; + private static final String ID_ARG = "id"; + private static final String NAME_ARG = "name"; + private final StreamCacheService streamCacheService; + private final Provider defaultStreamProvider; + private final ParameterDescriptor messageParam; + private final ParameterDescriptor nameParam; + private final ParameterDescriptor idParam; + + @Inject + public RemoveFromStream(StreamCacheService streamCacheService, @DefaultStream Provider defaultStreamProvider) { + this.streamCacheService = streamCacheService; + this.defaultStreamProvider = defaultStreamProvider; + + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + nameParam = string(NAME_ARG).optional().description("The name of the stream to remove the message from, must match exactly").build(); + idParam = string(ID_ARG).optional().description("The ID of the stream").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + Optional id = idParam.optional(args, context); + + Collection streams; + if (!id.isPresent()) { + final Optional> foundStreams = nameParam.optional(args, context).map(streamCacheService::getByName); + + if (!foundStreams.isPresent()) { + // TODO signal error somehow + return null; + } else { + streams = foundStreams.get(); + } + } else { + final Stream stream = streamCacheService.getById(id.get()); + if (stream == null) { + return null; + } + streams = Collections.singleton(stream); + } + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + streams.forEach(stream -> { + if (!stream.isPaused()) { + message.removeStream(stream); + } + }); + // always leave a message at least on the default stream if we removed the last stream it was on + if (message.getStreams().isEmpty()) { + message.addStream(defaultStreamProvider.get()); + } + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of( + nameParam, + idParam, + messageParam)) + .description("Removes a message from a stream. Removing the last stream will put the message back onto the default stream. To complete drop a message use the drop_message function.") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java new file mode 100644 index 000000000000..10291c6c9fc8 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RenameField.java @@ -0,0 +1,71 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class RenameField extends AbstractFunction { + + public static final String NAME = "rename_field"; + + private final ParameterDescriptor oldFieldParam; + private final ParameterDescriptor newFieldParam; + private final ParameterDescriptor messageParam; + + public RenameField() { + oldFieldParam = string("old_field").description("The old name of the field").build(); + newFieldParam = string("new_field").description("The new name of the field").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + final String oldName = oldFieldParam.required(args, context); + final String newName = newFieldParam.required(args, context); + + // exit early if the field names are the same (so we don't drop the field) + if (oldName != null && oldName.equals(newName)) { + return null; + } + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + + if (message.hasField(oldName)) { + message.addField(newName, message.getField(oldName)); + message.removeField(oldName); + } + + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(oldFieldParam, newFieldParam, messageParam) + .description("Rename a message field") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java new file mode 100644 index 000000000000..cf0bb085eccf --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/RouteToStream.java @@ -0,0 +1,109 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.inject.Inject; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.streams.DefaultStream; +import org.graylog2.plugin.streams.Stream; + +import javax.inject.Provider; +import java.util.Collection; +import java.util.Collections; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.bool; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class RouteToStream extends AbstractFunction { + + public static final String NAME = "route_to_stream"; + private static final String ID_ARG = "id"; + private static final String NAME_ARG = "name"; + private static final String REMOVE_FROM_DEFAULT = "remove_from_default"; + private final StreamCacheService streamCacheService; + private final Provider defaultStreamProvider; + private final ParameterDescriptor messageParam; + private final ParameterDescriptor nameParam; + private final ParameterDescriptor idParam; + private final ParameterDescriptor removeFromDefault; + + @Inject + public RouteToStream(StreamCacheService streamCacheService, @DefaultStream Provider defaultStreamProvider) { + this.streamCacheService = streamCacheService; + this.defaultStreamProvider = defaultStreamProvider; + + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + nameParam = string(NAME_ARG).optional().description("The name of the stream to route the message to, must match exactly").build(); + idParam = string(ID_ARG).optional().description("The ID of the stream").build(); + removeFromDefault = bool(REMOVE_FROM_DEFAULT).optional().description("After routing the message, remove it from the default stream").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + String id = idParam.optional(args, context).orElse(""); + + final Collection streams; + if ("".equals(id)) { + final String name = nameParam.optional(args, context).orElse(""); + if ("".equals(name)) { + return null; + } + streams = streamCacheService.getByName(name); + if (streams.isEmpty()) { + // TODO signal error somehow + return null; + } + } else { + final Stream stream = streamCacheService.getById(id); + if (stream == null) { + return null; + } + streams = Collections.singleton(stream); + } + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + streams.forEach(stream -> { + if (!stream.isPaused()) { + message.addStream(stream); + } + }); + if (removeFromDefault.optional(args, context).orElse(Boolean.FALSE)) { + message.removeStream(defaultStreamProvider.get()); + } + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of( + nameParam, + idParam, + messageParam, + removeFromDefault)) + .description("Routes a message to a stream") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java new file mode 100644 index 000000000000..71a5f24d7135 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetField.java @@ -0,0 +1,86 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.base.Strings; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class SetField extends AbstractFunction { + + public static final String NAME = "set_field"; + + private final ParameterDescriptor fieldParam; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor prefixParam; + private final ParameterDescriptor suffixParam; + private final ParameterDescriptor messageParam; + + public SetField() { + fieldParam = string("field").description("The new field name").build(); + valueParam = object("value").description("The new field value").build(); + prefixParam = string("prefix").optional().description("The prefix for the field name").build(); + suffixParam = string("suffix").optional().description("The suffix for the field name").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + String field = fieldParam.required(args, context); + final Object value = valueParam.required(args, context); + + if (!Strings.isNullOrEmpty(field)) { + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + final Optional prefix = prefixParam.optional(args, context); + final Optional suffix = suffixParam.optional(args, context); + + if (prefix.isPresent()) { + field = prefix.get() + field; + } + if (suffix.isPresent()) { + field = field + suffix.get(); + } + message.addField(field, value); + } + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of(fieldParam, + valueParam, + prefixParam, + suffixParam, + messageParam)) + .description("Sets a new field in a message") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java new file mode 100644 index 000000000000..d5cf7b3e99fc --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/SetFields.java @@ -0,0 +1,81 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.plugin.Message; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.type; + +public class SetFields extends AbstractFunction { + + public static final String NAME = "set_fields"; + + private final ParameterDescriptor fieldsParam; + private final ParameterDescriptor prefixParam; + private final ParameterDescriptor suffixParam; + private final ParameterDescriptor messageParam; + + public SetFields() { + fieldsParam = type("fields", Map.class).description("The map of new fields to set").build(); + prefixParam = string("prefix").optional().description("The prefix for the field names").build(); + suffixParam = string("suffix").optional().description("The suffix for the field names").build(); + messageParam = type("message", Message.class).optional().description("The message to use, defaults to '$message'").build(); + } + + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + //noinspection unchecked + final Map fields = fieldsParam.required(args, context); + final Message message = messageParam.optional(args, context).orElse(context.currentMessage()); + final Optional prefix = prefixParam.optional(args, context); + final Optional suffix = suffixParam.optional(args, context); + + if (fields != null) { + fields.forEach((field, value) -> { + if (prefix.isPresent()) { + field = prefix.get() + field; + } + if (suffix.isPresent()) { + field = field + suffix.get(); + } + message.addField(field, value); + }); + } + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Void.class) + .params(of(fieldsParam, prefixParam, suffixParam, messageParam)) + .description("Sets new fields in a message") + .build(); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheService.java new file mode 100644 index 000000000000..32c9368c40ae --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheService.java @@ -0,0 +1,126 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.Multimaps; +import com.google.common.collect.SortedSetMultimap; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.common.util.concurrent.AbstractIdleService; + +import org.graylog2.database.NotFoundException; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.streams.StreamService; +import org.graylog2.streams.events.StreamsChangedEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; + +@Singleton +public class StreamCacheService extends AbstractIdleService { + private static final Logger LOG = LoggerFactory.getLogger(StreamCacheService.class); + + private final EventBus eventBus; + private final StreamService streamService; + private final ScheduledExecutorService executorService; + + private final SortedSetMultimap nameToStream = Multimaps.synchronizedSortedSetMultimap( + MultimapBuilder.hashKeys() + .treeSetValues(Comparator.comparing(Stream::getId)) + .build()); + private final Map idToStream = Maps.newConcurrentMap(); + + @Inject + public StreamCacheService(EventBus eventBus, + StreamService streamService, + @Named("daemonScheduler") ScheduledExecutorService executorService) { + this.eventBus = eventBus; + this.streamService = streamService; + this.executorService = executorService; + } + + @Override + protected void startUp() throws Exception { + streamService.loadAllEnabled().forEach(this::updateCache); + eventBus.register(this); + } + + @Override + protected void shutDown() throws Exception { + eventBus.unregister(this); + } + + @Subscribe + public void handleStreamUpdate(StreamsChangedEvent event) { + executorService.schedule(() -> updateStreams(event.streamIds()), 0, TimeUnit.SECONDS); + } + + @VisibleForTesting + public void updateStreams(Collection ids) { + for (String id : ids) { + LOG.debug("Updating stream id/title cache for id {}", id); + try { + final Stream stream = streamService.load(id); + if (stream.getDisabled()) { + purgeCache(stream.getId()); + } else { + updateCache(stream); + } + } catch (NotFoundException e) { + // the stream was deleted, we only have to purge the existing entries + purgeCache(id); + } + } + } + + private void purgeCache(String id) { + final Stream stream = idToStream.remove(id); + LOG.debug("Purging stream id/title cache for id {}, stream {}", id, stream); + if (stream != null) { + nameToStream.remove(stream.getTitle(), stream); + } + } + + private void updateCache(Stream stream) { + LOG.debug("Updating stream id/title cache for {}/'{}'", stream.getId(), stream.getTitle()); + idToStream.put(stream.getId(), stream); + nameToStream.put(stream.getTitle(), stream); + } + + + public Collection getByName(String name) { + return nameToStream.get(name); + } + + @Nullable + public Stream getById(String id) { + return idToStream.get(id); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java new file mode 100644 index 000000000000..ffff5375e094 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Abbreviate.java @@ -0,0 +1,69 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.primitives.Ints.saturatedCast; + +public class Abbreviate extends AbstractFunction { + + public static final String NAME = "abbreviate"; + private static final String VALUE = "value"; + private static final String WIDTH = "width"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor widthParam; + + public Abbreviate() { + valueParam = ParameterDescriptor.string(VALUE).description("The string to abbreviate").build(); + widthParam = ParameterDescriptor.integer(WIDTH).description("The maximum number of characters including the '...' (at least 4)").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final Long required = widthParam.required(args, context); + if (required == null) { + return null; + } + final Long maxWidth = Math.max(required, 4L); + + return StringUtils.abbreviate(value, saturatedCast(maxWidth)); + } + + @Override + public FunctionDescriptor descriptor() { + ImmutableList.Builder params = ImmutableList.builder(); + params.add(); + + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(ImmutableList.of( + valueParam, + widthParam + )) + .description("Abbreviates a string by appending '...' to fit into a maximum amount of characters") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java new file mode 100644 index 000000000000..47f68cccc913 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Capitalize extends StringUtilsFunction { + + public static final String NAME = "capitalize"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected String description() { + return "Capitalizes a String changing the first letter to title case from lower case"; + } + + @Override + protected boolean isLocaleAware() { + return false; + } + + @Override + protected String apply(String value, Locale unused) { + return StringUtils.capitalize(value); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java new file mode 100644 index 000000000000..13c86210be2c --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Concat.java @@ -0,0 +1,55 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.base.Strings; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Concat extends AbstractFunction { + public static final String NAME = "concat"; + private final ParameterDescriptor firstParam; + private final ParameterDescriptor secondParam; + + public Concat() { + firstParam = ParameterDescriptor.string("first").description("First string").build(); + secondParam = ParameterDescriptor.string("second").description("Second string").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String first = Strings.nullToEmpty(firstParam.required(args, context)); + final String second = Strings.nullToEmpty(secondParam.required(args, context)); + + return first.concat(second); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of(firstParam, secondParam)) + .description("Concatenates two strings") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java new file mode 100644 index 000000000000..e554df2666cd --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Contains.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Contains extends AbstractFunction { + + public static final String NAME = "contains"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor searchParam; + private final ParameterDescriptor ignoreCaseParam; + + public Contains() { + valueParam = ParameterDescriptor.string("value").description("The string to check").build(); + searchParam = ParameterDescriptor.string("search").description("The substring to find").build(); + ignoreCaseParam = ParameterDescriptor.bool("ignore_case").optional().description("Whether to search case insensitive, defaults to false").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final String search = searchParam.required(args, context); + final boolean ignoreCase = ignoreCaseParam.optional(args, context).orElse(false); + if (ignoreCase) { + return StringUtils.containsIgnoreCase(value, search); + } else { + return StringUtils.contains(value, search); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + valueParam, + searchParam, + ignoreCaseParam + )) + .description("Checks if a string contains a substring") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/EndsWith.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/EndsWith.java new file mode 100644 index 000000000000..734d5652137e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/EndsWith.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class EndsWith extends AbstractFunction { + + public static final String NAME = "ends_with"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor suffixParam; + private final ParameterDescriptor ignoreCaseParam; + + public EndsWith() { + valueParam = ParameterDescriptor.string("value").description("The string to check").build(); + suffixParam = ParameterDescriptor.string("suffix").description("The suffix to check").build(); + ignoreCaseParam = ParameterDescriptor.bool("ignore_case").optional().description("Whether to search case insensitive, defaults to false").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final String suffix = suffixParam.required(args, context); + final boolean ignoreCase = ignoreCaseParam.optional(args, context).orElse(false); + if (ignoreCase) { + return StringUtils.endsWithIgnoreCase(value, suffix); + } else { + return StringUtils.endsWith(value, suffix); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + valueParam, + suffixParam, + ignoreCaseParam + )) + .description("Checks if a string ends with a suffix") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java new file mode 100644 index 000000000000..b9f90ac9efa6 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java @@ -0,0 +1,98 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ForwardingMap; +import oi.thekraken.grok.api.Grok; +import oi.thekraken.grok.api.Match; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog2.grok.GrokPatternRegistry; + +import javax.inject.Inject; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.of; + +public class GrokMatch extends AbstractFunction { + + public static final String NAME = "grok"; + + private final ParameterDescriptor valueParam; + private final ParameterDescriptor patternParam; + private final ParameterDescriptor namedOnly; + + private final GrokPatternRegistry grokPatternRegistry; + + @Inject + public GrokMatch(GrokPatternRegistry grokPatternRegistry) { + this.grokPatternRegistry = grokPatternRegistry; + + valueParam = ParameterDescriptor.string("value").description("The string to apply the Grok pattern against").build(); + patternParam = ParameterDescriptor.string("pattern").description("The Grok pattern").build(); + namedOnly = ParameterDescriptor.bool("only_named_captures").optional().description("Whether to only use explicitly named groups in the patterns").build(); + } + + @Override + public GrokResult evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final String pattern = patternParam.required(args, context); + final boolean onlyNamedCaptures = namedOnly.optional(args, context).orElse(false); + + if (value == null || pattern == null) { + return null; + } + + final Grok grok = grokPatternRegistry.cachedGrokForPattern(pattern, onlyNamedCaptures); + + final Match match = grok.match(value); + match.captures(); + return new GrokResult(match.toMap()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(GrokResult.class) + .params(of(patternParam, valueParam, namedOnly)) + .description("Applies a Grok pattern to a string") + .build(); + } + + public static class GrokResult extends ForwardingMap { + private final Map captures; + + public GrokResult(Map captures) { + this.captures = captures; + } + + @Override + protected Map delegate() { + return captures; + } + + public boolean isMatches() { + return captures.size() > 0; + } + } + + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java new file mode 100644 index 000000000000..c921d2cefcd0 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/KeyValue.java @@ -0,0 +1,187 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.inject.TypeLiteral; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.bool; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.string; + +public class KeyValue extends AbstractFunction> { + + public static final String NAME = "key_value"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor splitParam; + private final ParameterDescriptor valueSplitParam; + private final ParameterDescriptor ignoreEmptyValuesParam; + private final ParameterDescriptor allowDupeKeysParam; + private final ParameterDescriptor duplicateHandlingParam; + private final ParameterDescriptor trimCharactersParam; + private final ParameterDescriptor trimValueCharactersParam; + + public KeyValue() { + valueParam = string("value").description("The string to extract key/value pairs from").build(); + splitParam = string("delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().description("The characters used to separate pairs, defaults to whitespace").build(); + valueSplitParam = string("kv_delimiters", CharMatcher.class).transform(CharMatcher::anyOf).optional().description("The characters used to separate keys from values, defaults to '='").build(); + + ignoreEmptyValuesParam = bool("ignore_empty_values").optional().description("Whether to ignore keys with empty values, defaults to true").build(); + allowDupeKeysParam = bool("allow_dup_keys").optional().description("Whether to allow duplicate keys, defaults to true").build(); + duplicateHandlingParam = string("handle_dup_keys").optional().description("How to handle duplicate keys: 'take_first': only use first value, 'take_last': only take last value, default is to concatenate the values").build(); + trimCharactersParam = string("trim_key_chars", CharMatcher.class) + .transform(CharMatcher::anyOf) + .optional() + .description("The characters to trim from keys, default is not to trim") + .build(); + trimValueCharactersParam = string("trim_value_chars", CharMatcher.class) + .transform(CharMatcher::anyOf) + .optional() + .description("The characters to trim from values, default is not to trim") + .build(); + } + + @Override + public Map evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + if (Strings.isNullOrEmpty(value)) { + return null; + } + final CharMatcher kvPairsMatcher = splitParam.optional(args, context).orElse(CharMatcher.whitespace()); + final CharMatcher kvDelimMatcher = valueSplitParam.optional(args, context).orElse(CharMatcher.anyOf("=")); + + Splitter outerSplitter = Splitter.on(kvPairsMatcher) + .omitEmptyStrings() + .trimResults(); + + final Splitter entrySplitter = Splitter.on(kvDelimMatcher) + .omitEmptyStrings() + .trimResults(); + return new MapSplitter(outerSplitter, + entrySplitter, + ignoreEmptyValuesParam.optional(args, context).orElse(true), + trimCharactersParam.optional(args, context).orElse(CharMatcher.none()), + trimValueCharactersParam.optional(args, context).orElse(CharMatcher.none()), + allowDupeKeysParam.optional(args, context).orElse(true), + duplicateHandlingParam.optional(args, context).orElse("take_first")) + .split(value); + } + + @Override + public FunctionDescriptor> descriptor() { + //noinspection unchecked + return FunctionDescriptor.>builder() + .name(NAME) + .returnType((Class>) new TypeLiteral>() {}.getRawType()) + .params(valueParam, + splitParam, + valueSplitParam, + ignoreEmptyValuesParam, + allowDupeKeysParam, + duplicateHandlingParam, + trimCharactersParam, + trimValueCharactersParam + ) + .description("Extracts key/value pairs from a string") + .build(); + } + + + private static class MapSplitter { + + private final Splitter outerSplitter; + private final Splitter entrySplitter; + private final boolean ignoreEmptyValues; + private final CharMatcher keyTrimMatcher; + private final CharMatcher valueTrimMatcher; + private final Boolean allowDupeKeys; + private final String duplicateHandling; + + MapSplitter(Splitter outerSplitter, + Splitter entrySplitter, + boolean ignoreEmptyValues, + CharMatcher keyTrimMatcher, + CharMatcher valueTrimMatcher, + Boolean allowDupeKeys, + String duplicateHandling) { + this.outerSplitter = outerSplitter; + this.entrySplitter = entrySplitter; + this.ignoreEmptyValues = ignoreEmptyValues; + this.keyTrimMatcher = keyTrimMatcher; + this.valueTrimMatcher = valueTrimMatcher; + this.allowDupeKeys = allowDupeKeys; + this.duplicateHandling = duplicateHandling; + } + + + public Map split(CharSequence sequence) { + final Map map = new LinkedHashMap<>(); + + for (String entry : outerSplitter.split(sequence)) { + boolean concat = false; + Iterator entryFields = entrySplitter.split(entry).iterator(); + + if (!entryFields.hasNext()) { + continue; + } + String key = entryFields.next(); + key = keyTrimMatcher.trimFrom(key); + if (map.containsKey(key)) { + if (!allowDupeKeys) { + throw new IllegalArgumentException("Duplicate key " + key + " is not allowed in key_value function."); + } + switch (Strings.nullToEmpty(duplicateHandling).toLowerCase(Locale.ENGLISH)) { + case "take_first": + // ignore this value + continue; + case "take_last": + // simply reset the entry + break; + default: + concat = true; + } + } + + if (entryFields.hasNext()) { + String value = entryFields.next(); + value = valueTrimMatcher.trimFrom(value); + // already have a value, concating old+delim+new + if (concat) { + value = map.get(key) + duplicateHandling + value; + } + map.put(key, value); + } else if (!ignoreEmptyValues) { + throw new IllegalArgumentException("Missing value for key " + key); + } + + } + return Collections.unmodifiableMap(map); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java new file mode 100644 index 000000000000..fb9917478e50 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Lowercase extends StringUtilsFunction { + + public static final String NAME = "lowercase"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected String description() { + return "Lowercases a string"; + } + + @Override + protected boolean isLocaleAware() { + return true; + } + + @Override + protected String apply(String value, Locale locale) { + return StringUtils.lowerCase(value, locale); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java new file mode 100644 index 000000000000..d0011f353d64 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/RegexMatch.java @@ -0,0 +1,129 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ForwardingMap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.google.common.collect.ImmutableList.of; + +public class RegexMatch extends AbstractFunction { + + public static final String NAME = "regex"; + private final ParameterDescriptor pattern; + private final ParameterDescriptor value; + private final ParameterDescriptor optionalGroupNames; + + public RegexMatch() { + pattern = ParameterDescriptor.string("pattern", Pattern.class).transform(Pattern::compile).description("The regular expression to match against 'value', uses Java regex syntax").build(); + value = ParameterDescriptor.string("value").description("The string to match the pattern against").build(); + optionalGroupNames = ParameterDescriptor.type("group_names", List.class).optional().description("List of names to use for matcher groups").build(); + } + + @Override + public RegexMatchResult evaluate(FunctionArgs args, EvaluationContext context) { + final Pattern regex = pattern.required(args, context); + final String value = this.value.required(args, context); + if (regex == null || value == null) { + final String nullArgument = regex == null ? "pattern" : "value"; + throw new IllegalArgumentException("Argument '" + nullArgument + "' cannot be 'null'"); + } + //noinspection unchecked + final List groupNames = + (List) optionalGroupNames.optional(args, context).orElse(Collections.emptyList()); + + final Matcher matcher = regex.matcher(value); + final boolean matches = matcher.find(); + + return new RegexMatchResult(matches, matcher.toMatchResult(), groupNames); + + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .pure(true) + .returnType(RegexMatchResult.class) + .params(of( + pattern, + value, + optionalGroupNames + )) + .description("Match a string with a regular expression (Java syntax)") + .build(); + } + + /** + * The bean returned into the rule engine. It implements Map so rules can access it directly. + *
+ * At the same time there's an additional matches bean property to quickly check whether the regex has matched at all. + */ + public static class RegexMatchResult extends ForwardingMap { + private final boolean matches; + private final ImmutableMap groups; + + public RegexMatchResult(boolean matches, MatchResult matchResult, List groupNames) { + this.matches = matches; + ImmutableMap.Builder builder = ImmutableMap.builder(); + + if (matches) { + // arggggh! not 0 based. + final int groupCount = matchResult.groupCount(); + for (int i = 1; i <= groupCount; i++) { + final String groupValue = matchResult.group(i); + + if (groupValue == null) { + // You cannot add null values to an ImmutableMap but optional matcher groups may be null. + continue; + } + + // try to get a group name, if that fails use a 0-based index as the name + final String groupName = Iterables.get(groupNames, i - 1, null); + builder.put(groupName != null ? groupName : String.valueOf(i - 1), groupValue); + } + } + groups = builder.build(); + } + + public boolean isMatches() { + return matches; + } + + public Map getGroups() { + return groups; + } + + @Override + protected Map delegate() { + return getGroups(); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Split.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Split.java new file mode 100644 index 000000000000..10b3be167192 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Split.java @@ -0,0 +1,74 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Ints; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class Split extends AbstractFunction { + public static final String NAME = "split"; + + private final ParameterDescriptor pattern; + private final ParameterDescriptor value; + private final ParameterDescriptor limit; + + public Split() { + pattern = ParameterDescriptor.string("pattern", Pattern.class) + .transform(Pattern::compile) + .description("The regular expression to split by, uses Java regex syntax") + .build(); + value = ParameterDescriptor.string("value") + .description("The string to be split") + .build(); + limit = ParameterDescriptor.integer("limit", Integer.class) + .transform(Ints::saturatedCast) + .description("The number of times the pattern is applied") + .optional() + .build(); + } + + @Override + public String[] evaluate(FunctionArgs args, EvaluationContext context) { + final Pattern regex = requireNonNull(pattern.required(args, context), "Argument 'pattern' cannot be 'null'"); + final String value = requireNonNull(this.value.required(args, context), "Argument 'value' cannot be 'null'"); + + final int limit = this.limit.optional(args, context).orElse(0); + checkArgument(limit >= 0, "Argument 'limit' cannot be negative"); + return regex.split(value, limit); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .pure(true) + .returnType(String[].class) + .params(ImmutableList.of(pattern, value, limit)) + .description("Split a string around matches of this pattern (Java syntax)") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StartsWith.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StartsWith.java new file mode 100644 index 000000000000..8e5f6ea84d3b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StartsWith.java @@ -0,0 +1,66 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class StartsWith extends AbstractFunction { + + public static final String NAME = "starts_with"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor prefixParam; + private final ParameterDescriptor ignoreCaseParam; + + public StartsWith() { + valueParam = ParameterDescriptor.string("value").description("The string to check").build(); + prefixParam = ParameterDescriptor.string("prefix").description("The prefix to check").build(); + ignoreCaseParam = ParameterDescriptor.bool("ignore_case").optional().description("Whether to search case insensitive, defaults to false").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final String prefix = prefixParam.required(args, context); + final boolean ignoreCase = ignoreCaseParam.optional(args, context).orElse(false); + if (ignoreCase) { + return StringUtils.startsWithIgnoreCase(value, prefix); + } else { + return StringUtils.startsWith(value, prefix); + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(of( + valueParam, + prefixParam, + ignoreCaseParam + )) + .description("Checks if a string starts with a prefix") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java new file mode 100644 index 000000000000..f60a7a18ab6e --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/StringUtilsFunction.java @@ -0,0 +1,76 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.util.Locale; + +public abstract class StringUtilsFunction extends AbstractFunction { + + private static final String VALUE = "value"; + private static final String LOCALE = "locale"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor localeParam; + + public StringUtilsFunction() { + valueParam = ParameterDescriptor.string(VALUE).description("The input string").build(); + localeParam = ParameterDescriptor.string(LOCALE, Locale.class) + .optional() + .transform(Locale::forLanguageTag) + .description("The locale to use, defaults to English") + .build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + Locale locale = Locale.ENGLISH; + if (isLocaleAware()) { + locale = localeParam.optional(args, context).orElse(Locale.ENGLISH); + } + return apply(value, locale); + } + + @Override + public FunctionDescriptor descriptor() { + ImmutableList.Builder params = ImmutableList.builder(); + params.add(valueParam); + if (isLocaleAware()) { + params.add(localeParam); + } + return FunctionDescriptor.builder() + .name(getName()) + .returnType(String.class) + .params(params.build()) + .description(description()) + .build(); + } + + protected abstract String getName(); + + protected abstract String description(); + + protected abstract boolean isLocaleAware(); + + protected abstract String apply(String value, Locale locale); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java new file mode 100644 index 000000000000..941a8f972533 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Substring.java @@ -0,0 +1,68 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import com.google.common.primitives.Ints; +import org.apache.commons.lang3.StringUtils; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.collect.ImmutableList.of; + +public class Substring extends AbstractFunction { + + public static final String NAME = "substring"; + private final ParameterDescriptor valueParam; + private final ParameterDescriptor startParam; + private final ParameterDescriptor endParam; + + public Substring() { + valueParam = ParameterDescriptor.string("value").description("The string to extract from").build(); + startParam = ParameterDescriptor.integer("start").description("The position to start from, negative means count back from the end of the String by this many characters").build(); + endParam = ParameterDescriptor.integer("end").optional().description("The position to end at (exclusive), negative means count back from the end of the String by this many characters, defaults to length of the input string").build(); + } + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String value = valueParam.required(args, context); + final Long startValue = startParam.required(args, context); + if (value == null || startValue == null) { + return null; + } + final int start = Ints.saturatedCast(startValue); + final int end = Ints.saturatedCast(endParam.optional(args, context).orElse((long) value.length())); + + return StringUtils.substring(value, start, end); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(of( + valueParam, + startParam, + endParam + )) + .description("Extract a substring from a string") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java new file mode 100644 index 000000000000..3f2c339a6dd0 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Swapcase extends StringUtilsFunction { + + public static final String NAME = "swapcase"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected String description() { + return "Swaps the case of a String changing upper and title case to lower case, and lower case to upper case."; + } + + @Override + protected boolean isLocaleAware() { + return false; + } + + @Override + protected String apply(String value, Locale unused) { + return StringUtils.swapCase(value); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java new file mode 100644 index 000000000000..ce5dad116610 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Uncapitalize extends StringUtilsFunction { + + public static final String NAME = "uncapitalize"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected String description() { + return "Uncapitalizes a String changing the first letter to lower case from title case"; + } + + @Override + protected boolean isLocaleAware() { + return false; + } + + @Override + protected String apply(String value, Locale unused) { + return StringUtils.uncapitalize(value); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java new file mode 100644 index 000000000000..fcdd13817129 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java @@ -0,0 +1,46 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.strings; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Locale; + +public class Uppercase extends StringUtilsFunction { + + public static final String NAME = "uppercase"; + + @Override + protected String getName() { + return NAME; + } + + @Override + protected String description() { + return "Uppercases a string"; + } + + @Override + protected boolean isLocaleAware() { + return true; + } + + @Override + protected String apply(String value, Locale locale) { + return StringUtils.upperCase(value, locale); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java new file mode 100644 index 000000000000..a7771f4110e7 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogFacilityConversion.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.common.primitives.Ints; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogFacilityConversion extends AbstractFunction { + public static final String NAME = "syslog_facility"; + + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final Integer facility = firstNonNull(Ints.tryParse(s), -1); + + return SyslogUtils.facilityToString(facility); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(valueParam) + .description("Converts a syslog facility number to its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java new file mode 100644 index 000000000000..049d67d302df --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogLevelConversion.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.common.primitives.Ints; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogLevelConversion extends AbstractFunction { + public static final String NAME = "syslog_level"; + + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final Integer level = firstNonNull(Ints.tryParse(s), -1); + + return SyslogUtils.levelToString(level); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(String.class) + .params(valueParam) + .description("Converts a syslog level number to its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java new file mode 100644 index 000000000000..85a421d18805 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java @@ -0,0 +1,30 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class SyslogPriority { + public abstract int getLevel(); + + public abstract int getFacility(); + + public static SyslogPriority create(int level, int facility) { + return new AutoValue_SyslogPriority(level, facility); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java new file mode 100644 index 000000000000..db7db3061cba --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java @@ -0,0 +1,30 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class SyslogPriorityAsString { + public abstract String getLevel(); + + public abstract String getFacility(); + + public static SyslogPriorityAsString create(String level, String facility) { + return new AutoValue_SyslogPriorityAsString(level, facility); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java new file mode 100644 index 000000000000..f38ebe64a81d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityConversion.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogPriorityConversion extends AbstractFunction { + public static final String NAME = "expand_syslog_priority"; + + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); + + @Override + public SyslogPriority evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final int priority = Integer.parseInt(s); + final int facility = SyslogUtils.facilityFromPriority(priority); + final int level = SyslogUtils.levelFromPriority(priority); + + return SyslogPriority.create(level, facility); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(SyslogPriority.class) + .params(valueParam) + .description("Converts a syslog priority number to its level and facility") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java new file mode 100644 index 000000000000..cf99a758dad9 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityToStringConversion.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class SyslogPriorityToStringConversion extends AbstractFunction { + public static final String NAME = "expand_syslog_priority_as_string"; + + private final ParameterDescriptor valueParam = object("value").description("Value to convert").build(); + + @Override + public SyslogPriorityAsString evaluate(FunctionArgs args, EvaluationContext context) { + final String s = String.valueOf(valueParam.required(args, context)); + final int priority = Integer.parseInt(s); + final int facility = SyslogUtils.facilityFromPriority(priority); + final String facilityString = SyslogUtils.facilityToString(facility); + final int level = SyslogUtils.levelFromPriority(priority); + final String levelString = SyslogUtils.levelToString(level); + + return SyslogPriorityAsString.create(levelString, facilityString); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(SyslogPriorityAsString.class) + .params(valueParam) + .description("Converts a syslog priority number to its level and facility string representations") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java new file mode 100644 index 000000000000..14785da7f433 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogUtils.java @@ -0,0 +1,119 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.syslog; + +public final class SyslogUtils { + /** + * Converts integer syslog loglevel to human readable string + * + * @param level The level to convert + * @return The human readable level + * @see RFC 5424, Section 6.2.1 + */ + public static String levelToString(int level) { + switch (level) { + case 0: + return "Emergency"; + case 1: + return "Alert"; + case 2: + return "Critical"; + case 3: + return "Error"; + case 4: + return "Warning"; + case 5: + return "Notice"; + case 6: + return "Informational"; + case 7: + return "Debug"; + } + + return "Unknown"; + } + + /** + * Converts integer syslog facility to human readable string + * + * @param facility The facility to convert + * @return The human readable facility + * @see RFC 5424, Section 6.2.1 + */ + public static String facilityToString(int facility) { + switch (facility) { + case 0: + return "kern"; + case 1: + return "user"; + case 2: + return "mail"; + case 3: + return "daemon"; + case 4: + return "auth"; + case 5: + return "syslog"; + case 6: + return "lpr"; + case 7: + return "news"; + case 8: + return "uucp"; + case 9: + return "clock"; + case 10: + return "authpriv"; + case 11: + return "ftp"; + case 12: + return "ntp"; + case 13: + return "log audit"; + case 14: + return "log alert"; + case 15: + return "cron"; + case 16: + return "local0"; + case 17: + return "local1"; + case 18: + return "local2"; + case 19: + return "local3"; + case 20: + return "local4"; + case 21: + return "local5"; + case 22: + return "local6"; + case 23: + return "local7"; + default: + return "Unknown"; + } + } + + public static int levelFromPriority(int priority) { + return priority - (facilityFromPriority(priority) << 3); + } + + public static int facilityFromPriority(int priority) { + return priority >> 3; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/IsUrl.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/IsUrl.java new file mode 100644 index 000000000000..15eb2e5cc243 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/IsUrl.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.urls; + +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import static org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor.object; + +public class IsUrl extends AbstractFunction { + public static final String NAME = "is_url"; + + private final ParameterDescriptor valueParam; + + public IsUrl() { + valueParam = object("value").description("Value to check").build(); + } + + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + final Object value = valueParam.required(args, context); + return value instanceof URL; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(Boolean.class) + .params(valueParam) + .description("Checks whether a value is a URL") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/URL.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/URL.java new file mode 100644 index 000000000000..b1a3c3a66747 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/URL.java @@ -0,0 +1,119 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.urls; + +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import okhttp3.HttpUrl; + +import java.util.List; +import java.util.Map; + +/** + * This class simply delegates the safe methods to the {@link java.net.URL}. + * + * Specifically we want to disallow called {@link java.net.URL#getContent()} from a rule. + */ +public class URL { + private final HttpUrl url; + private Map queryMap; + + public URL(java.net.URL url) { + this.url = HttpUrl.get(url); + } + + public URL(java.net.URI uri) { + this.url = HttpUrl.get(uri); + } + + public URL(String urlString) { + url = HttpUrl.parse(urlString); + } + + public String getQuery() { + return url.encodedQuery(); + } + + public Map getQueryParams() { + if (queryMap == null) { + final Map queryMap = Maps.newHashMapWithExpectedSize(url.querySize()); + for(String name : url.queryParameterNames()) { + final List values = url.queryParameterValues(name); + final String valueString = Joiner.on(',').join(values); + queryMap.put(name, valueString); + } + this.queryMap = queryMap; + } + return queryMap; + } + + public String getUserInfo() { + final String username = url.encodedUsername(); + return username.isEmpty() ? "" : username + ':' + url.encodedPassword(); + } + + public String getHost() { + return url.host(); + } + + public String getPath() { + return url.encodedPath(); + } + + public String getFile() { + return url.querySize() == 0 ? url.encodedPath() : url.encodedPath() + '?' + url.encodedQuery(); + } + + public String getProtocol() { + return url.scheme(); + } + + public int getDefaultPort() { + return url.port(); + } + + /** + * alias for #getRef, fragment is more commonly used + */ + public String getFragment() { + return url.encodedFragment(); + } + + public String getRef() { + return getFragment(); + } + + public String getAuthority() { + final String userInfo = getUserInfo(); + return userInfo.isEmpty() ? getHost() + ':' + getPort() : getUserInfo() + '@' + getHost() + ':' + getPort(); + } + + public int getPort() { + return url.port(); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(Object obj) { + return url.equals(obj); + } + + @Override + public int hashCode() { + return url.hashCode(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java new file mode 100644 index 000000000000..44b06475ff9b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/urls/UrlConversion.java @@ -0,0 +1,67 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.urls; + +import com.google.common.base.Throwables; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; + +import java.net.MalformedURLException; +import java.util.Optional; + +public class UrlConversion extends AbstractFunction { + + public static final String NAME = "to_url"; + + private final ParameterDescriptor urlParam = ParameterDescriptor.object("url").description("Value to convert").build(); + private final ParameterDescriptor defaultParam = ParameterDescriptor.string("default").optional().description("Used when 'url' is null or malformed").build(); + + @Override + public URL evaluate(FunctionArgs args, EvaluationContext context) { + final String urlString = String.valueOf(urlParam.required(args, context)); + try { + return new URL(urlString); + } catch (IllegalArgumentException e) { + log.debug("Unable to parse URL for string {}", urlString, e); + + final Optional defaultUrl = defaultParam.optional(args, context); + if (!defaultUrl.isPresent()) { + return null; + } + try { + return new URL(defaultUrl.get()); + } catch (IllegalArgumentException e1) { + log.warn("Parameter `default` for to_url() is not a valid URL: {}", defaultUrl.get()); + throw Throwables.propagate(e1); + } + } + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name(NAME) + .returnType(URL.class) + .params(urlParam, + defaultParam) + .description("Converts a value to a valid URL using its string representation") + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java new file mode 100644 index 000000000000..6d73f0682362 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java @@ -0,0 +1,51 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; + +public class FunctionRegistry { + + private final Map> functions; + + @Inject + public FunctionRegistry(Map> functions) { + this.functions = functions; + } + + + public Function resolve(String name) { + return functions.get(name); + } + + public Function resolveOrError(String name) { + final Function function = resolve(name); + if (function == null) { + return Function.ERROR_FUNCTION; + } + return function; + } + + public Collection> all() { + return functions.values().stream().collect(Collectors.toList()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java new file mode 100644 index 000000000000..e8ba8ee9ecdb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java @@ -0,0 +1,42 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; + +import java.util.Set; + +public class ParseException extends RuntimeException { + private final Set errors; + + public ParseException(Set errors) { + this.errors = errors; + } + + public Set getErrors() { + return errors; + } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder("Errors:\n"); + for (ParseError parseError : getErrors()) { + sb.append(" ").append(parseError).append("\n"); + } + return sb.toString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java new file mode 100644 index 000000000000..61691c6e432d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParser.java @@ -0,0 +1,1032 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.tree.ParseTreeProperty; +import org.antlr.v4.runtime.tree.ParseTreeWalker; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.mina.util.IdentityHashSet; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.PrecomputeFailure; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AdditionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ArrayLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanValuedFunctionWrapper; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.DoubleExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FieldRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.IndexedAccessExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LongExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MapLiteralExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MessageRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.MultiplicationExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.SignedExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.StringExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.VarRefExpression; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.statements.FunctionStatement; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement; +import org.graylog.plugins.pipelineprocessor.codegen.CodeGenerator; +import org.graylog.plugins.pipelineprocessor.codegen.GeneratedRule; +import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleTypes; +import org.graylog.plugins.pipelineprocessor.parser.errors.InvalidFunctionArgument; +import org.graylog.plugins.pipelineprocessor.parser.errors.InvalidOperation; +import org.graylog.plugins.pipelineprocessor.parser.errors.MissingRequiredParam; +import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType; +import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; +import org.graylog.plugins.pipelineprocessor.parser.errors.SyntaxError; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; +import org.graylog.plugins.pipelineprocessor.parser.errors.WrongNumberOfArgs; +import org.graylog.plugins.pipelineprocessor.processors.ConfigurationStateUpdater; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.Period; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Stack; +import java.util.concurrent.atomic.AtomicLong; + +import javax.inject.Inject; + +import static com.google.common.collect.ImmutableSortedSet.orderedBy; +import static java.util.Comparator.comparingInt; +import static java.util.stream.Collectors.toList; + +public class PipelineRuleParser { + + private final FunctionRegistry functionRegistry; + private final CodeGenerator codeGenerator; + + private static AtomicLong uniqueId = new AtomicLong(0); + + @Inject + public PipelineRuleParser(FunctionRegistry functionRegistry, CodeGenerator codeGenerator) { + this.functionRegistry = functionRegistry; + this.codeGenerator = codeGenerator; + } + + private static final Logger log = LoggerFactory.getLogger(PipelineRuleParser.class); + public static final ParseTreeWalker WALKER = ParseTreeWalker.DEFAULT; + + public Rule parseRule(String rule, boolean silent) throws ParseException { + return parseRule(rule, silent, null); + } + + public Rule parseRule(String rule, boolean silent, PipelineClassloader classLoader) throws ParseException { + return parseRule("dummy" + uniqueId.getAndIncrement(), rule, silent, classLoader); + } + + public Rule parseRule(String id, String rule, boolean silent) throws ParseException { + return parseRule(id, rule, silent, null); + } + + /** + * Parses the given rule source and optionally generates a Java class for it if the classloader is not null. + * + * @param id the id of the rule, necessary to generate code + * @param rule rule source code + * @param silent don't emit status messages during parsing + * @param ruleClassLoader the classloader to load the generated code into (can be null) + * @return the parse rule + * @throws ParseException if a one or more parse errors occur + */ + public Rule parseRule(String id, String rule, boolean silent, PipelineClassloader ruleClassLoader) throws ParseException { + final ParseContext parseContext = new ParseContext(silent); + final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); + + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(rule)); + lexer.removeErrorListeners(); + lexer.addErrorListener(errorListener); + + final RuleLangParser parser = new RuleLangParser(new CommonTokenStream(lexer)); + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.removeErrorListeners(); + parser.addErrorListener(errorListener); + + final RuleLangParser.RuleDeclarationContext ruleDeclaration = parser.ruleDeclaration(); + + + // parsing stages: + // 1. build AST nodes, checks for invalid var, function refs + // 2. type annotator: infer type information from var refs, func refs + // 3. checker: static type check w/ coercion nodes + // 4. optimizer: TODO + + WALKER.walk(new RuleAstBuilder(parseContext), ruleDeclaration); + WALKER.walk(new RuleTypeAnnotator(parseContext), ruleDeclaration); + WALKER.walk(new RuleTypeChecker(parseContext), ruleDeclaration); + + if (parseContext.getErrors().isEmpty()) { + Rule parsedRule = parseContext.getRules().get(0).withId(id); + if (ruleClassLoader != null && ConfigurationStateUpdater.isAllowCodeGeneration()) { + try { + final Class generatedClass = codeGenerator.generateCompiledRule(parsedRule, ruleClassLoader); + if (generatedClass != null) { + parsedRule = parsedRule.toBuilder().generatedRuleClass(generatedClass).build(); + } + } catch (Exception e) { + log.warn("Unable to compile rule {} to native code, falling back to interpreting it: {}", parsedRule.name(), e.getMessage()); + } + } + return parsedRule; + } + throw new ParseException(parseContext.getErrors()); + } + + public List parsePipelines(String pipelines) throws ParseException { + final ParseContext parseContext = new ParseContext(false); + final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); + + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(pipelines)); + lexer.removeErrorListeners(); + lexer.addErrorListener(errorListener); + + final RuleLangParser parser = new RuleLangParser(new CommonTokenStream(lexer)); + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.removeErrorListeners(); + parser.addErrorListener(errorListener); + + final RuleLangParser.PipelineDeclsContext pipelineDeclsContext = parser.pipelineDecls(); + + WALKER.walk(new PipelineAstBuilder(parseContext), pipelineDeclsContext); + + if (parseContext.getErrors().isEmpty()) { + return parseContext.pipelines; + } + throw new ParseException(parseContext.getErrors()); + } + + public Pipeline parsePipeline(String id, String source) { + final ParseContext parseContext = new ParseContext(false); + final SyntaxErrorListener errorListener = new SyntaxErrorListener(parseContext); + + final RuleLangLexer lexer = new RuleLangLexer(new ANTLRInputStream(source)); + lexer.removeErrorListeners(); + lexer.addErrorListener(errorListener); + + final RuleLangParser parser = new RuleLangParser(new CommonTokenStream(lexer)); + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.removeErrorListeners(); + parser.addErrorListener(errorListener); + + final RuleLangParser.PipelineContext pipelineContext = parser.pipeline(); + + WALKER.walk(new PipelineAstBuilder(parseContext), pipelineContext); + + if (parseContext.getErrors().isEmpty()) { + final Pipeline pipeline = parseContext.pipelines.get(0); + return pipeline.withId(id); + } + throw new ParseException(parseContext.getErrors()); + } + + public static String unquote(String string, char quoteChar) { + if (string.length() >= 2 && + string.charAt(0) == quoteChar && string.charAt(string.length() - 1) == quoteChar) { + return string.substring(1, string.length() - 1); + } + return string; + } + + public static String unescape(String string) { + return StringEscapeUtils.unescapeJava(string); + } + + private static class SyntaxErrorListener extends BaseErrorListener { + private final ParseContext parseContext; + + public SyntaxErrorListener(ParseContext parseContext) { + this.parseContext = parseContext; + } + + @Override + public void syntaxError(Recognizer recognizer, + Object offendingSymbol, + int line, + int charPositionInLine, + String msg, + RecognitionException e) { + parseContext.addError(new SyntaxError(offendingSymbol, line, charPositionInLine, msg, e)); + } + } + + + private class RuleAstBuilder extends RuleLangBaseListener { + + private final ParseContext parseContext; + private final ParseTreeProperty> args; + private final ParseTreeProperty> argsList; + private final ParseTreeProperty exprs; + + private final Set definedVars = Sets.newHashSet(); + + // this is true for nested field accesses + private Stack isIdIsFieldAccess = new Stack<>(); + + public RuleAstBuilder(ParseContext parseContext) { + this.parseContext = parseContext; + args = parseContext.arguments(); + argsList = parseContext.argumentLists(); + exprs = parseContext.expressions(); + isIdIsFieldAccess.push(false); // top of stack + } + + @Override + public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { + final Rule.Builder ruleBuilder = Rule.builder(); + ruleBuilder.name(unquote(ctx.name == null ? "" : ctx.name.getText(), '"')); + final Expression expr = exprs.get(ctx.condition); + + LogicalExpression condition; + if (expr instanceof LogicalExpression) { + condition = (LogicalExpression) expr; + } else if (expr != null && expr.getType().equals(Boolean.class)) { + condition = new BooleanValuedFunctionWrapper(ctx.getStart(), expr); + } else { + condition = new BooleanExpression(ctx.getStart(), false); + log.debug("Unable to create condition, replacing with 'false'"); + } + ruleBuilder.when(condition); + ruleBuilder.then(parseContext.statements); + final Rule rule = ruleBuilder.build(); + parseContext.addRule(rule); + log.trace("Declaring rule {}", rule); + } + + @Override + public void exitFuncStmt(RuleLangParser.FuncStmtContext ctx) { + final Expression expr = exprs.get(ctx.functionCall()); + final FunctionStatement functionStatement = new FunctionStatement(expr); + parseContext.statements.add(functionStatement); + } + + @Override + public void exitVarAssignStmt(RuleLangParser.VarAssignStmtContext ctx) { + final String name = unquote(ctx.varName.getText(), '`'); + final Expression expr = exprs.get(ctx.expression()); + parseContext.defineVar(name, expr); + definedVars.add(name); + parseContext.statements.add(new VarAssignStatement(name, expr)); + } + + @Override + public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { + final String name = ctx.funcName.getText(); + Map argsMap = this.args.get(ctx.arguments()); + final List positionalArgs = this.argsList.get(ctx.arguments()); + + final Function function = functionRegistry.resolve(name); + if (function == null) { + parseContext.addError(new UndeclaredFunction(ctx)); + } else { + final ImmutableList params = function.descriptor().params(); + final boolean hasOptionalParams = params.stream().anyMatch(ParameterDescriptor::optional); + + if (argsMap != null) { + // check for the right number of arguments to the function if the function only has required params + if (!hasOptionalParams && params.size() != argsMap.size()) { + parseContext.addError(new WrongNumberOfArgs(ctx, function, argsMap.size())); + } else { + // there are optional parameters, check that all required ones are present + final Map givenArguments = argsMap; + final List missingParams = + params.stream() + .filter(p -> !p.optional()) + .map(p -> givenArguments.containsKey(p.name()) ? null : p) + .filter(p -> p != null) + .collect(toList()); + for (ParameterDescriptor param : missingParams) { + parseContext.addError(new MissingRequiredParam(ctx, function, param)); + } + } + } else if (positionalArgs != null) { + // use descriptor to turn positional arguments into a map + argsMap = Maps.newHashMap(); + // if we only have required parameters and the number doesn't match, complain + if (!hasOptionalParams && positionalArgs.size() != params.size()) { + parseContext.addError(new WrongNumberOfArgs(ctx, function, positionalArgs.size())); + } + // if optional parameters precede any required ones, the function must used named parameters + boolean hasError = false; + if (hasOptionalParams) { + // find the index of the first optional parameter + // then check if any non-optional come after it, if so, complain + int firstOptional = Integer.MAX_VALUE; + boolean requiredAfterOptional = false; + int i = 0; + for (ParameterDescriptor param : params) { + i++; + if (param.optional()) { + firstOptional = Math.min(firstOptional, i); + } else { + if (i > firstOptional) { + requiredAfterOptional = true; + } + } + } + if (requiredAfterOptional) { + parseContext.addError(new OptionalParametersMustBeNamed(ctx, function)); + hasError = true; + } + } + + if (!hasError) { + // only try to assign params if we didn't encounter a problem with position optional params above + int i = 0; + for (ParameterDescriptor p : params) { + if (i >= positionalArgs.size()) { + // avoid index out of bounds, we've added an error anyway + // the remaining parameters were optional, so we can safely skip them + break; + } + final Expression argExpr = positionalArgs.get(i); + argsMap.put(p.name(), argExpr); + i++; + } + } + } else if(! params.stream().allMatch(ParameterDescriptor::optional)) { + // no parameters given but some of them are required + parseContext.addError(new WrongNumberOfArgs(ctx, function, 0)); + } + } + + FunctionExpression expr; + try { + expr = new FunctionExpression( + ctx.getStart(), new FunctionArgs(functionRegistry.resolveOrError(name), argsMap) + ); + } catch (PrecomputeFailure precomputeFailure) { + parseContext.addError(new InvalidFunctionArgument(ctx, function, precomputeFailure)); + expr = new FunctionExpression(ctx.getStart(), new FunctionArgs(Function.ERROR_FUNCTION, argsMap)); + } + + log.trace("FUNC: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitNamedArgs(RuleLangParser.NamedArgsContext ctx) { + final Map argMap = Maps.newHashMap(); + for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { + final String argName = unquote(propAssignmentContext.Identifier().getText(), '`'); + final Expression argValue = exprs.get(propAssignmentContext.expression()); + argMap.put(argName, argValue); + } + args.put(ctx, argMap); + } + + @Override + public void exitPositionalArgs(RuleLangParser.PositionalArgsContext ctx) { + List expressions = Lists.newArrayListWithCapacity(ctx.expression().size()); + expressions.addAll(ctx.expression().stream().map(exprs::get).collect(toList())); + argsList.put(ctx, expressions); + } + + @Override + public void enterNested(RuleLangParser.NestedContext ctx) { + // nested field access is ok, these are not rule variables + isIdIsFieldAccess.push(true); + } + + @Override + public void exitNested(RuleLangParser.NestedContext ctx) { + isIdIsFieldAccess.pop(); // reset for error checks + final Expression object = exprs.get(ctx.fieldSet); + final Expression field = exprs.get(ctx.field); + final FieldAccessExpression expr = new FieldAccessExpression(ctx.getStart(), object, field); + log.trace("FIELDACCESS: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitNot(RuleLangParser.NotContext ctx) { + final Expression expression = upgradeBoolFunctionExpression(ctx.expression()); + final NotExpression expr = new NotExpression(ctx.getStart(), expression); + log.trace("NOT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitAnd(RuleLangParser.AndContext ctx) { + // if the expressions are function calls but boolean valued, upgrade them, + // we allow testing boolean valued functions without explicit comparison operator + final Expression left = upgradeBoolFunctionExpression(ctx.left); + final Expression right = upgradeBoolFunctionExpression(ctx.right); + + final AndExpression expr = new AndExpression(ctx.getStart(), left, right); + log.trace("AND: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + private Expression upgradeBoolFunctionExpression(RuleLangParser.ExpressionContext leftExprContext) { + Expression leftExpr = exprs.get(leftExprContext); + if (leftExpr instanceof FunctionExpression && leftExpr.getType().equals(Boolean.class)) { + leftExpr = new BooleanValuedFunctionWrapper(leftExprContext.getStart(), leftExpr); + } + return leftExpr; + } + + @Override + public void exitOr(RuleLangParser.OrContext ctx) { + final Expression left = upgradeBoolFunctionExpression(ctx.left); + final Expression right = upgradeBoolFunctionExpression(ctx.right); + final OrExpression expr = new OrExpression(ctx.getStart(), left, right); + log.trace("OR: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitEquality(RuleLangParser.EqualityContext ctx) { + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); + final boolean equals = ctx.equality.getText().equals("=="); + final EqualityExpression expr = new EqualityExpression(ctx.getStart(), left, right, equals); + log.trace("EQUAL: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitComparison(RuleLangParser.ComparisonContext ctx) { + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); + final String operator = ctx.comparison.getText(); + final ComparisonExpression expr = new ComparisonExpression(ctx.getStart(), left, right, operator); + log.trace("COMPARE: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitInteger(RuleLangParser.IntegerContext ctx) { + // TODO handle different radix and length + final LongExpression expr = new LongExpression(ctx.getStart(), Long.parseLong(ctx.getText())); + log.trace("INT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitFloat(RuleLangParser.FloatContext ctx) { + final DoubleExpression expr = new DoubleExpression(ctx.getStart(), Double.parseDouble(ctx.getText())); + log.trace("FLOAT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitChar(RuleLangParser.CharContext ctx) { + // TODO + super.exitChar(ctx); + } + + @Override + public void exitString(RuleLangParser.StringContext ctx) { + final String text = unescape(unquote(ctx.getText(), '\"')); + final StringExpression expr = new StringExpression(ctx.getStart(), text); + log.trace("STRING: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitBoolean(RuleLangParser.BooleanContext ctx) { + final BooleanExpression expr = new BooleanExpression(ctx.getStart(), Boolean.valueOf(ctx.getText())); + log.trace("BOOL: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitLiteralPrimary(RuleLangParser.LiteralPrimaryContext ctx) { + // nothing to do, just propagate the ConstantExpression + exprs.put(ctx, exprs.get(ctx.literal())); + parseContext.addInnerNode(ctx); + } + + @Override + public void exitArrayLiteralExpr(RuleLangParser.ArrayLiteralExprContext ctx) { + final List elements = ctx.expression().stream().map(exprs::get).collect(toList()); + exprs.put(ctx, new ArrayLiteralExpression(ctx.getStart(), elements)); + } + + @Override + public void exitMapLiteralExpr(RuleLangParser.MapLiteralExprContext ctx) { + final HashMap map = Maps.newHashMap(); + for (RuleLangParser.PropAssignmentContext propAssignmentContext : ctx.propAssignment()) { + final String key = unquote(propAssignmentContext.Identifier().getText(), '`'); + final Expression value = exprs.get(propAssignmentContext.expression()); + map.put(key, value); + } + exprs.put(ctx, new MapLiteralExpression(ctx.getStart(), map)); + } + + @Override + public void exitParenExpr(RuleLangParser.ParenExprContext ctx) { + // nothing to do, just propagate + exprs.put(ctx, exprs.get(ctx.expression())); + parseContext.addInnerNode(ctx); + } + + @Override + public void exitSignedExpression(RuleLangParser.SignedExpressionContext ctx) { + final Expression right = exprs.get(ctx.expr); + final boolean isPlus = ctx.sign.getText().equals("+"); + + final SignedExpression expr = new SignedExpression(ctx.getStart(), right, isPlus); + log.trace("SIGN: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitAddition(RuleLangParser.AdditionContext ctx) { + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); + final boolean isPlus = ctx.add.getText().equals("+"); + + final AdditionExpression expr = new AdditionExpression(ctx.getStart(), left, right, isPlus); + log.trace("ADD: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitMultiplication(RuleLangParser.MultiplicationContext ctx) { + final Expression left = exprs.get(ctx.left); + final Expression right = exprs.get(ctx.right); + final char operator = ctx.mult.getText().charAt(0); + + final MultiplicationExpression expr = new MultiplicationExpression(ctx.getStart(), left, right, operator); + log.trace("MULT: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void enterMessageRef(RuleLangParser.MessageRefContext ctx) { + // nested field access is ok, these are not rule variables + isIdIsFieldAccess.push(true); + } + + @Override + public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { + isIdIsFieldAccess.pop(); // reset for error checks + final Expression fieldExpr = exprs.get(ctx.field); + final MessageRefExpression expr = new MessageRefExpression(ctx.getStart(), fieldExpr); + log.trace("$MSG: ctx {} => {}", ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { + // unquote identifier if necessary + final String identifierName = unquote(ctx.Identifier().getText(), '`'); + + if (!isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { + parseContext.addError(new UndeclaredVariable(ctx)); + } + final Expression expr; + String type; + // if the identifier is also a declared variable name prefer the variable + if (isIdIsFieldAccess.peek() && !definedVars.contains(identifierName)) { + expr = new FieldRefExpression(ctx.getStart(), identifierName, parseContext.getDefinedVar(identifierName)); + type = "FIELDREF"; + } else { + expr = new VarRefExpression(ctx.getStart(), identifierName, parseContext.getDefinedVar(identifierName)); + type = "VARREF"; + } + log.trace("{}: ctx {} => {}", type, ctx, expr); + exprs.put(ctx, expr); + } + + @Override + public void exitFunc(RuleLangParser.FuncContext ctx) { + // nothing to do, just propagate + exprs.put(ctx, exprs.get(ctx.functionCall())); + parseContext.addInnerNode(ctx); + } + + @Override + public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { + final Expression array = exprs.get(ctx.array); + final Expression index = exprs.get(ctx.index); + + final IndexedAccessExpression expr = new IndexedAccessExpression(ctx.getStart(), array, index); + exprs.put(ctx, expr); + log.trace("IDXACCESS: ctx {} => {}", ctx, expr); + } + } + + private class RuleTypeAnnotator extends RuleLangBaseListener { + private final ParseContext parseContext; + + public RuleTypeAnnotator(ParseContext parseContext) { + this.parseContext = parseContext; + } + + @Override + public void exitIdentifier(RuleLangParser.IdentifierContext ctx) { + final Expression expr = parseContext.expressions().get(ctx); + if (expr instanceof VarRefExpression) { + final VarRefExpression varRefExpression = (VarRefExpression) expr; + final String name = varRefExpression.varName(); + final Expression expression = parseContext.getDefinedVar(name); + if (expression == null) { + if (parseContext.isSilent()) { + log.debug("Unable to retrieve expression for variable {}, this is a bug", name); + } else { + log.error("Unable to retrieve expression for variable {}, this is a bug", name); + } + return; + } + log.trace("Inferred type of variable {} to {}", name, expression.getType().getSimpleName()); + varRefExpression.setType(expression.getType()); + } + } + + @Override + public void exitAddition(RuleLangParser.AdditionContext ctx) { + final AdditionExpression expr = (AdditionExpression) parseContext.expressions().get(ctx); + final Class leftType = expr.left().getType(); + final Class rightType = expr.right().getType(); + + if (leftType.equals(rightType)) { + // propagate left type + expr.setType(leftType); + } else if (DateTime.class.equals(leftType) && DateTime.class.equals(rightType)) { + // fine to subtract two dates from each other, this results in a Duration + expr.setType(Duration.class); + } else if (DateTime.class.equals(leftType) && Period.class.equals(rightType) || Period.class.equals(leftType) && DateTime.class.equals(rightType)) { + expr.setType(DateTime.class); + } else { + // this will be detected as an error later + expr.setType(Void.class); + } + } + + @Override + public void exitMultiplication(RuleLangParser.MultiplicationContext ctx) { + final MultiplicationExpression expr = (MultiplicationExpression) parseContext.expressions().get(ctx); + final Class leftType = expr.left().getType(); + final Class rightType = expr.right().getType(); + + if (leftType.equals(rightType)) { + // propagate left type + expr.setType(leftType); + } else { + // this will be detected as an error later + expr.setType(Void.class); + } + } + } + + private class RuleTypeChecker extends RuleLangBaseListener { + private final ParseContext parseContext; + StringBuffer sb = new StringBuffer(); + + public RuleTypeChecker(ParseContext parseContext) { + this.parseContext = parseContext; + } + + @Override + public void exitRuleDeclaration(RuleLangParser.RuleDeclarationContext ctx) { + log.trace("Type tree {}", sb.toString()); + } + + @Override + public void exitAnd(RuleLangParser.AndContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitOr(RuleLangParser.OrContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitComparison(RuleLangParser.ComparisonContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitAddition(RuleLangParser.AdditionContext ctx) { + final AdditionExpression addExpression = (AdditionExpression) parseContext.expressions().get(ctx); + final Class leftType = addExpression.left().getType(); + final Class rightType = addExpression.right().getType(); + + // special case for DateTime/Period, which are all compatible + final boolean leftDate = DateTime.class.equals(leftType); + final boolean rightDate = DateTime.class.equals(rightType); + final boolean leftPeriod = Period.class.equals(leftType); + final boolean rightPeriod = Period.class.equals(rightType); + if (leftDate && rightDate) { + if (addExpression.isPlus()) { + parseContext.addError(new InvalidOperation(ctx, addExpression, "Unable to add two dates")); + } + return; + } else if (leftDate && rightPeriod || leftPeriod && rightDate || leftPeriod && rightPeriod) { + return; + } + // otherwise check generic binary expression + checkBinaryExpression(ctx); + } + + @Override + public void exitMultiplication(RuleLangParser.MultiplicationContext ctx) { + checkBinaryExpression(ctx); + } + + @Override + public void exitEquality(RuleLangParser.EqualityContext ctx) { + // TODO equality allows arbitrary types, in the future optimize by using specialized operators + } + + private void checkBinaryExpression(RuleLangParser.ExpressionContext ctx) { + final BinaryExpression binaryExpr = (BinaryExpression) parseContext.expressions().get(ctx); + final Class leftType = binaryExpr.left().getType(); + final Class rightType = binaryExpr.right().getType(); + + if (!leftType.equals(rightType) || Void.class.equals(leftType) || Void.class.equals(rightType)) { + parseContext.addError(new IncompatibleTypes(ctx, binaryExpr)); + } + } + + @Override + public void exitFunctionCall(RuleLangParser.FunctionCallContext ctx) { + final FunctionExpression expr = (FunctionExpression) parseContext.expressions().get(ctx); + final FunctionDescriptor descriptor = expr.getFunction().descriptor(); + final FunctionArgs args = expr.getArgs(); + for (ParameterDescriptor p : descriptor.params()) { + final Expression argExpr = args.expression(p.name()); + if (argExpr != null && !p.type().isAssignableFrom(argExpr.getType())) { + parseContext.addError(new IncompatibleArgumentType(ctx, expr, p, argExpr)); + } + } + } + + @Override + public void exitMessageRef(RuleLangParser.MessageRefContext ctx) { + final MessageRefExpression expr = (MessageRefExpression) parseContext.expressions().get(ctx); + if (!expr.getFieldExpr().getType().equals(String.class)) { + parseContext.addError(new IncompatibleType(ctx, String.class, expr.getFieldExpr().getType())); + } + } + + @Override + public void enterEveryRule(ParserRuleContext ctx) { + final Expression expression = parseContext.expressions().get(ctx); + if (expression != null && !parseContext.isInnerNode(ctx)) { + sb.append(" ( "); + sb.append(expression.getClass().getSimpleName()); + sb.append(":").append(ctx.getClass().getSimpleName()).append(" "); + sb.append(" <").append(expression.getType().getSimpleName()).append("> "); + sb.append(ctx.getText()); + } + } + + @Override + public void exitEveryRule(ParserRuleContext ctx) { + final Expression expression = parseContext.expressions().get(ctx); + if (expression != null && !parseContext.isInnerNode(ctx)) { + sb.append(" ) "); + } + } + + @Override + public void exitIndexedAccess(RuleLangParser.IndexedAccessContext ctx) { + final IndexedAccessExpression idxExpr = (IndexedAccessExpression) parseContext.expressions().get( + ctx); + + final Class indexableType = idxExpr.getIndexableObject().getType(); + final Class indexType = idxExpr.getIndex().getType(); + + final boolean isMap = Map.class.isAssignableFrom(indexableType); + if (indexableType.isArray() + || List.class.isAssignableFrom(indexableType) + || Iterable.class.isAssignableFrom(indexableType) + || isMap) { + // then check if the index type is compatible, must be long for array-like and string for map-like types + if (isMap) { + if (!String.class.equals(indexType)) { + // add type error + parseContext.addError(new IncompatibleIndexType(ctx, String.class, indexType)); + } + } else { + if (!Long.class.equals(indexType)) { + parseContext.addError(new IncompatibleIndexType(ctx, Long.class, indexType)); + } + } + } else { + // not an indexable type + parseContext.addError(new NonIndexableType(ctx, indexableType)); + } + + } + } + + /** + * Contains meta data about the parse tree, such as AST nodes, link to the function registry etc. + * + * Being used by tree walkers or visitors to perform AST construction, type checking and so on. + */ + private static class ParseContext { + private final ParseTreeProperty exprs = new ParseTreeProperty<>(); + private final ParseTreeProperty> args = new ParseTreeProperty<>(); + /** + * Should the parser be more silent about its error logging, useful for interactive parsing in the UI. + */ + private final boolean silent; + private ParseTreeProperty> argsLists = new ParseTreeProperty<>(); + private Set errors = Sets.newHashSet(); + // inner nodes in the parse tree will be ignored during type checker printing, they only transport type information + private Set innerNodes = new IdentityHashSet<>(); + public List statements = Lists.newArrayList(); + public List rules = Lists.newArrayList(); + private Map varDecls = Maps.newHashMap(); + public List pipelines = Lists.newArrayList(); + + public ParseContext(boolean silent) { + this.silent = silent; + } + + public ParseTreeProperty expressions() { + return exprs; + } + + public ParseTreeProperty> arguments() { + return args; + } + + public List getRules() { + return rules; + } + + public void setRules(List rules) { + this.rules = rules; + } + public void addRule(Rule rule) { + this.rules.add(rule); + } + + public List getPipelines() { + return pipelines; + } + + public Set getErrors() { + return errors; + } + + public void addError(ParseError error) { + errors.add(error); + } + + public void addInnerNode(RuleContext node) { + innerNodes.add(node); + } + + public boolean isInnerNode(RuleContext node) { + return innerNodes.contains(node); + } + + /** + * Links the declared var to its expression. + * + * @param name var name + * @param expr expression + * @return true if successful, false if previously declared + */ + public boolean defineVar(String name, Expression expr) { + return varDecls.put(name, expr) == null; + } + + public Expression getDefinedVar(String name) { + return varDecls.get(name); + } + + public ParseTreeProperty> argumentLists() { + return argsLists; + } + + public boolean isSilent() { + return silent; + } + } + + private class PipelineAstBuilder extends RuleLangBaseListener { + private final ParseContext parseContext; + + public PipelineAstBuilder(ParseContext parseContext) { + this.parseContext = parseContext; + } + + @Override + public void exitPipelineDeclaration(RuleLangParser.PipelineDeclarationContext ctx) { + final Pipeline.Builder builder = Pipeline.builder(); + + builder.name(unquote(ctx.name.getText(), '"')); + final ImmutableSortedSet.Builder stages = orderedBy(comparingInt(Stage::stage)); + + for (RuleLangParser.StageDeclarationContext stage : ctx.stageDeclaration()) { + final Stage.Builder stageBuilder = Stage.builder(); + + final Token stageToken = stage.stage; + if (stageToken == null) { + parseContext.addError(new SyntaxError(null, 0, 0, "", null)); + return; + } + final int stageNumber = Integer.parseInt(stageToken.getText()); + stageBuilder.stage(stageNumber); + + final Token modifier = stage.modifier; + if (modifier == null) { + parseContext.addError(new SyntaxError(null, stageToken.getLine(), stageToken.getCharPositionInLine(), "", null)); + return; + } + final boolean isAllModifier = modifier.getText().equalsIgnoreCase("all"); + stageBuilder.matchAll(isAllModifier); + + final List ruleRefs = stage.ruleRef().stream() + .map(ruleRefContext -> { + final Token name = ruleRefContext.name; + if (name == null) { + final Token symbol = ruleRefContext.Rule().getSymbol(); + parseContext.addError(new SyntaxError(symbol, symbol.getLine(), symbol.getCharPositionInLine(), "invalid rule reference", null)); + return "__illegal_reference"; + } + return unquote(name.getText(), '"'); + }) + .collect(toList()); + stageBuilder.ruleReferences(ruleRefs); + + stages.add(stageBuilder.build()); + } + + builder.stages(stages.build()); + parseContext.pipelines.add(builder.build()); + } + + @Override + public void exitInteger(RuleLangParser.IntegerContext ctx) { + // TODO handle different radix and length + final LongExpression expr = new LongExpression(ctx.getStart(), Long.parseLong(ctx.getText())); + log.trace("INT: ctx {} => {}", ctx, expr); + parseContext.exprs.put(ctx, expr); + } + + @Override + public void exitString(RuleLangParser.StringContext ctx) { + final String text = unescape(unquote(ctx.getText(), '\"')); + final StringExpression expr = new StringExpression(ctx.getStart(), text); + log.trace("STRING: ctx {} => {}", ctx, expr); + parseContext.exprs.put(ctx, expr); + } + + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java new file mode 100644 index 000000000000..a657ad7ac203 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleArgumentType.java @@ -0,0 +1,49 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleArgumentType extends ParseError { + private final FunctionExpression functionExpression; + private final ParameterDescriptor p; + private final Expression argExpr; + + public IncompatibleArgumentType(RuleLangParser.FunctionCallContext ctx, + FunctionExpression functionExpression, + ParameterDescriptor p, + Expression argExpr) { + super("incompatible_argument_type", ctx); + this.functionExpression = functionExpression; + this.p = p; + this.argExpr = argExpr; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + p.type().getSimpleName() + + " for argument " + p.name() + + " but found " + argExpr.getType().getSimpleName() + + " in call to function " + functionExpression.getFunction().descriptor().name() + + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java new file mode 100644 index 000000000000..204d11c328eb --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java @@ -0,0 +1,39 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleIndexType extends ParseError { + private final Class expected; + private final Class actual; + + public IncompatibleIndexType(RuleLangParser.IndexedAccessContext ctx, + Class expected, + Class actual) { + super("incompatible_index_type", ctx); + this.expected = expected; + this.actual = actual; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + " when indexing" + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java new file mode 100644 index 000000000000..502108e65929 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java @@ -0,0 +1,37 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleType extends ParseError { + private final Class expected; + private final Class actual; + + public IncompatibleType(RuleLangParser.MessageRefContext ctx, Class expected, Class actual) { + super("incompatible_type", ctx); + this.expected = expected; + this.actual = actual; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java new file mode 100644 index 000000000000..8f4f9a0e18f6 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java @@ -0,0 +1,45 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class IncompatibleTypes extends ParseError { + private final RuleLangParser.ExpressionContext ctx; + private final BinaryExpression binaryExpr; + + public IncompatibleTypes(RuleLangParser.ExpressionContext ctx, BinaryExpression binaryExpr) { + super("incompatible_types", ctx); + this.ctx = ctx; + this.binaryExpr = binaryExpr; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); + } + + private String exprString(Expression e) { + return "(" + e.toString() + ") : " + e.getType().getSimpleName(); + } + + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java new file mode 100644 index 000000000000..bb077192d6e5 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidFunctionArgument.java @@ -0,0 +1,63 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.exceptions.PrecomputeFailure; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class InvalidFunctionArgument extends ParseError { + private final Function function; + private final PrecomputeFailure failure; + + public InvalidFunctionArgument(RuleLangParser.FunctionCallContext ctx, + Function function, + PrecomputeFailure failure) { + super("invalid_function_argument", ctx); + this.function = function; + this.failure = failure; + } + + @JsonProperty("reason") + @Override + public String toString() { + int paramPosition = 1; + for (ParameterDescriptor descriptor : function.descriptor().params()) { + if (descriptor.name().equals(failure.getArgumentName())) { + break; + } + paramPosition++; + } + + return "Unable to pre-compute value for " + ordinal(paramPosition) + " argument " + failure.getArgumentName() + " in call to function " + function.descriptor().name() + ": " + failure.getCause().getMessage(); + } + + private static String ordinal(int i) { + String[] suffixes = new String[]{"th", "st", "nd", "rd", "th", "th", "th", "th", "th", "th"}; + switch (i % 100) { + case 11: + case 12: + case 13: + return i + "th"; + default: + return i + suffixes[i % 10]; + + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidOperation.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidOperation.java new file mode 100644 index 000000000000..968f8316b1b4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidOperation.java @@ -0,0 +1,48 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; + +public class InvalidOperation extends ParseError { + private final Expression expr; + + private final String message; + + public InvalidOperation(ParserRuleContext ctx, Expression expr, String message) { + super("invalid_operation", ctx); + this.expr = expr; + this.message = message; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Invalid operation: " + message; + } + + public Expression getExpression() { + return expr; + } + + public String getMessage() { + return message; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java new file mode 100644 index 000000000000..6813e3bc8715 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java @@ -0,0 +1,44 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class MissingRequiredParam extends ParseError { + private final Function function; + private final ParameterDescriptor param; + + public MissingRequiredParam(RuleLangParser.FunctionCallContext ctx, + Function function, + ParameterDescriptor param) { + super("missing_required_param", ctx); + this.function = function; + this.param = param; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Missing required parameter " + param.name() + + " of type " + param.type().getSimpleName() + + " in call to function " + function.descriptor().name() + + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java new file mode 100644 index 000000000000..6640ac6bfbe6 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java @@ -0,0 +1,35 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class NonIndexableType extends ParseError { + private final Class indexableType; + + public NonIndexableType(RuleLangParser.IndexedAccessContext ctx, Class indexableType) { + super("non_indexable", ctx); + this.indexableType = indexableType; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Cannot index value of type " + indexableType.getSimpleName() + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java new file mode 100644 index 000000000000..c9ace0a8b652 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class OptionalParametersMustBeNamed extends ParseError { + private final Function function; + + public OptionalParametersMustBeNamed(RuleLangParser.FunctionCallContext ctx, Function function) { + super("must_name_optional_params", ctx); + this.function = function; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Function " + function.descriptor().name() + " has optional parameters, must use named parameters to call" + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java new file mode 100644 index 000000000000..7e7d046f9979 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java @@ -0,0 +1,67 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.antlr.v4.runtime.ParserRuleContext; + +import java.util.Objects; + +public abstract class ParseError { + + @JsonProperty + private final String type; + + @JsonIgnore + private final ParserRuleContext ctx; + + protected ParseError(String type, ParserRuleContext ctx) { + this.type = type; + this.ctx = ctx; + } + + @JsonProperty + public int line() { + return ctx.getStart().getLine(); + } + + @JsonProperty + public int positionInLine() { + return ctx.getStart().getCharPositionInLine(); + } + + protected String positionString() { + return " in" + + " line " + line() + + " pos " + positionInLine(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ParseError)) return false; + ParseError that = (ParseError) o; + return Objects.equals(type, that.type) && + Objects.equals(ctx, that.ctx); + } + + @Override + public int hashCode() { + return Objects.hash(type, ctx); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java new file mode 100644 index 000000000000..8f8d106667c4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java @@ -0,0 +1,58 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; + +import javax.annotation.Nullable; + +public class SyntaxError extends ParseError { + + private final Object offendingSymbol; + private final int line; + private final int charPositionInLine; + private final String msg; + private final RecognitionException e; + + public SyntaxError(@Nullable Object offendingSymbol, int line, int charPositionInLine, String msg, @Nullable RecognitionException e) { + super("syntax_error", new ParserRuleContext()); + + this.offendingSymbol = offendingSymbol; + this.line = line; + this.charPositionInLine = charPositionInLine; + this.msg = msg; + this.e = e; + } + + @Override + public int line() { + return line; + } + + @Override + public int positionInLine() { + return charPositionInLine; + } + + @JsonProperty("reason") + @Override + public String toString() { + return msg; + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java new file mode 100644 index 000000000000..cf440615c486 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java @@ -0,0 +1,35 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class UndeclaredFunction extends ParseError { + private final RuleLangParser.FunctionCallContext ctx; + + public UndeclaredFunction(RuleLangParser.FunctionCallContext ctx) { + super("undeclared_function", ctx); + this.ctx = ctx; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Unknown function " + ctx.funcName.getText() + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java new file mode 100644 index 000000000000..14ccbce378d8 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java @@ -0,0 +1,39 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +public class UndeclaredVariable extends ParseError { + + @JsonIgnore + private final RuleLangParser.IdentifierContext ctx; + + public UndeclaredVariable(RuleLangParser.IdentifierContext ctx) { + super("undeclared_variable", ctx); + this.ctx = ctx; + } + + @JsonProperty("reason") + @Override + public String toString() { + return "Undeclared variable " + ctx.Identifier().getText() + positionString(); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java new file mode 100644 index 000000000000..a5f9afc96eff --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java @@ -0,0 +1,47 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser.errors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; + +import java.util.function.Predicate; + +public class WrongNumberOfArgs extends ParseError { + private final Function function; + private final int argCount; + + public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, + Function function, + int argCount) { + super("wrong_number_of_arguments", ctx); + this.function = function; + this.argCount = argCount; + } + + @JsonProperty("reason") + @Override + public String toString() { + final Predicate optional = ParameterDescriptor::optional; + return "Expected " + function.descriptor().params().stream().filter(optional.negate()).count() + + " arguments but found " + argCount + + " in call to function " + function.descriptor().name() + + positionString(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/periodical/LegacyDefaultStreamMigration.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/periodical/LegacyDefaultStreamMigration.java new file mode 100644 index 000000000000..f29999166ac7 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/periodical/LegacyDefaultStreamMigration.java @@ -0,0 +1,99 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.periodical; + +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.events.LegacyDefaultStreamMigrated; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.database.NotFoundException; +import org.graylog2.plugin.cluster.ClusterConfigService; +import org.graylog2.plugin.periodical.Periodical; +import org.graylog2.plugin.streams.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +public class LegacyDefaultStreamMigration extends Periodical { + private static final Logger LOG = LoggerFactory.getLogger(LegacyDefaultStreamMigration.class); + private final ClusterConfigService clusterConfigService; + private final PipelineStreamConnectionsService connectionsService; + + private static final String LEGACY_STREAM_ID = "default"; + + @Inject + public LegacyDefaultStreamMigration(ClusterConfigService clusterConfigService, PipelineStreamConnectionsService connectionsService) { + this.clusterConfigService = clusterConfigService; + this.connectionsService = connectionsService; + } + + @Override + public boolean runsForever() { + return true; + } + + @Override + public boolean stopOnGracefulShutdown() { + return false; + } + + @Override + public boolean masterOnly() { + return true; + } + + @Override + public boolean startOnThisNode() { + final LegacyDefaultStreamMigrated migrationState = + clusterConfigService.getOrDefault(LegacyDefaultStreamMigrated.class, + LegacyDefaultStreamMigrated.create(false)); + return !migrationState.migrationDone(); + } + + @Override + public boolean isDaemon() { + return false; + } + + @Override + public int getInitialDelaySeconds() { + return 0; + } + + @Override + public int getPeriodSeconds() { + return 0; + } + + @Override + protected Logger getLogger() { + return LOG; + } + + @Override + public void doRun() { + try { + final PipelineConnections defaultConnections = connectionsService.load(LEGACY_STREAM_ID); + connectionsService.save(defaultConnections.toBuilder().streamId(Stream.DEFAULT_STREAM_ID).build()); + connectionsService.delete(LEGACY_STREAM_ID); + clusterConfigService.write(LegacyDefaultStreamMigrated.create(true)); + LOG.info("Pipeline connections to legacy default streams migrated successfully."); + } catch (NotFoundException e) { + LOG.info("Legacy default stream has no connections, no migration needed."); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/ConfigurationStateUpdater.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/ConfigurationStateUpdater.java new file mode 100644 index 000000000000..27e8615c3c72 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/ConfigurationStateUpdater.java @@ -0,0 +1,243 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.codahale.metrics.MetricRegistry; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent; +import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; +import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; +import javax.tools.ToolProvider; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static com.codahale.metrics.MetricRegistry.name; + +@Singleton +public class ConfigurationStateUpdater { + private static final Logger log = LoggerFactory.getLogger(ConfigurationStateUpdater.class); + + private final RuleService ruleService; + private final PipelineService pipelineService; + private final PipelineStreamConnectionsService pipelineStreamConnectionsService; + private final PipelineRuleParser pipelineRuleParser; + private final MetricRegistry metricRegistry; + private final FunctionRegistry functionRegistry; + private final ScheduledExecutorService scheduler; + private final EventBus serverEventBus; + private final PipelineInterpreter.State.Factory stateFactory; + /** + * non-null if the update has successfully loaded a state + */ + private final AtomicReference latestState = new AtomicReference<>(); + private static boolean allowCodeGeneration = false; + + @Inject + public ConfigurationStateUpdater(RuleService ruleService, + PipelineService pipelineService, + PipelineStreamConnectionsService pipelineStreamConnectionsService, + PipelineRuleParser pipelineRuleParser, + MetricRegistry metricRegistry, + FunctionRegistry functionRegistry, + @Named("daemonScheduler") ScheduledExecutorService scheduler, + EventBus serverEventBus, + PipelineInterpreter.State.Factory stateFactory, + @Named("generate_native_code") boolean allowCodeGeneration) { + this.ruleService = ruleService; + this.pipelineService = pipelineService; + this.pipelineStreamConnectionsService = pipelineStreamConnectionsService; + this.pipelineRuleParser = pipelineRuleParser; + this.metricRegistry = metricRegistry; + this.functionRegistry = functionRegistry; + this.scheduler = scheduler; + this.serverEventBus = serverEventBus; + this.stateFactory = stateFactory; + // ignore global config, never allow generating code + setAllowCodeGeneration(false); + + // listens to cluster wide Rule, Pipeline and pipeline stream connection changes + serverEventBus.register(this); + + reloadAndSave(); + } + + private static void setAllowCodeGeneration(Boolean allowCodeGeneration) { + if (allowCodeGeneration && ToolProvider.getSystemJavaCompiler() == null) { + log.warn("Your Java runtime does not have a compiler available, turning off dynamic " + + "code generation. Please consider running Graylog in a JDK, not a JRE, to " + + "avoid a performance penalty in pipeline processing."); + allowCodeGeneration = false; + } + ConfigurationStateUpdater.allowCodeGeneration = allowCodeGeneration; + } + + public static boolean isAllowCodeGeneration() { + return allowCodeGeneration; + } + + // only the singleton instance should mutate itself, others are welcome to reload a new state, but we don't + // currently allow direct global state updates from external sources (if you need to, send an event on the bus instead) + private synchronized PipelineInterpreter.State reloadAndSave() { + // this classloader will hold all generated rule classes + PipelineClassloader commonClassLoader = allowCodeGeneration ? new PipelineClassloader() : null; + + // read all rules and parse them + Map ruleNameMap = Maps.newHashMap(); + ruleService.loadAll().forEach(ruleDao -> { + Rule rule; + try { + rule = pipelineRuleParser.parseRule(ruleDao.id(), ruleDao.source(), false, commonClassLoader); + } catch (ParseException e) { + rule = Rule.alwaysFalse("Failed to parse rule: " + ruleDao.id()); + } + ruleNameMap.put(rule.name(), rule); + }); + + // read all pipelines and parse them + ImmutableMap.Builder pipelineIdMap = ImmutableMap.builder(); + pipelineService.loadAll().forEach(pipelineDao -> { + Pipeline pipeline; + try { + pipeline = pipelineRuleParser.parsePipeline(pipelineDao.id(), pipelineDao.source()); + } catch (ParseException e) { + pipeline = Pipeline.empty("Failed to parse pipeline" + pipelineDao.id()); + } + //noinspection ConstantConditions + pipelineIdMap.put(pipelineDao.id(), resolvePipeline(pipeline, ruleNameMap)); + }); + + final ImmutableMap currentPipelines = pipelineIdMap.build(); + + // read all stream connections of those pipelines to allow processing messages through them + final HashMultimap connections = HashMultimap.create(); + for (PipelineConnections streamConnection : pipelineStreamConnectionsService.loadAll()) { + streamConnection.pipelineIds().stream() + .map(currentPipelines::get) + .filter(Objects::nonNull) + .forEach(pipeline -> connections.put(streamConnection.streamId(), pipeline)); + } + ImmutableSetMultimap streamPipelineConnections = ImmutableSetMultimap.copyOf(connections); + + final PipelineInterpreter.State newState = stateFactory.newState(currentPipelines, streamPipelineConnections); + latestState.set(newState); + return newState; + } + + + /** + * Can be used to inspect or use the current state of the pipeline system. + * For example, the interpreter + * @return the currently loaded state of the updater + */ + public PipelineInterpreter.State getLatestState() { + return latestState.get(); + } + + @Nonnull + private Pipeline resolvePipeline(Pipeline pipeline, Map ruleNameMap) { + log.debug("Resolving pipeline {}", pipeline.name()); + + pipeline.stages().forEach(stage -> { + final List resolvedRules = stage.ruleReferences().stream() + .map(ref -> { + Rule rule = ruleNameMap.get(ref); + if (rule == null) { + rule = Rule.alwaysFalse("Unresolved rule " + ref); + } + // make a copy so that the metrics match up (we don't share actual objects between stages) + // this also makes sure we don't accidentally share state of generated code between threads + rule = rule.invokableCopy(functionRegistry); + log.debug("Resolved rule `{}` to {}", ref, rule); + // include back reference to stage + rule.registerMetrics(metricRegistry, pipeline.id(), String.valueOf(stage.stage())); + return rule; + }) + .collect(Collectors.toList()); + stage.setRules(resolvedRules); + stage.setPipeline(pipeline); + stage.registerMetrics(metricRegistry, pipeline.id()); + }); + + pipeline.registerMetrics(metricRegistry); + return pipeline; + } + + // TODO avoid reloading everything on every change, certain changes can get away with doing less work + @Subscribe + public void handleRuleChanges(RulesChangedEvent event) { + event.deletedRuleIds().forEach(id -> { + log.debug("Invalidated rule {}", id); + metricRegistry.removeMatching((name, metric) -> name.startsWith(name(Rule.class, id))); + }); + event.updatedRuleIds().forEach(id -> log.debug("Refreshing rule {}", id)); + scheduler.schedule(() -> serverEventBus.post(reloadAndSave()), 0, TimeUnit.SECONDS); + } + + @Subscribe + public void handlePipelineChanges(PipelinesChangedEvent event) { + event.deletedPipelineIds().forEach(id -> { + log.debug("Invalidated pipeline {}", id); + metricRegistry.removeMatching((name, metric) -> name.startsWith(name(Pipeline.class, id))); + }); + event.updatedPipelineIds().forEach(id -> log.debug("Refreshing pipeline {}", id)); + scheduler.schedule(() -> serverEventBus.post(reloadAndSave()), 0, TimeUnit.SECONDS); + } + + @Subscribe + public void handlePipelineConnectionChanges(PipelineConnectionsChangedEvent event) { + log.debug("Pipeline stream connection changed: {}", event); + scheduler.schedule(() -> serverEventBus.post(reloadAndSave()), 0, TimeUnit.SECONDS); + } + + @Subscribe + public void handlePipelineStateChange(PipelineInterpreter.State event) { + log.debug("Pipeline interpreter state got updated"); + } + + @VisibleForTesting + PipelineInterpreter.State reload() { + return reloadAndSave(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java new file mode 100644 index 000000000000..4201990431d2 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreter.java @@ -0,0 +1,504 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.assistedinject.AssistedInject; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.codegen.GeneratedRule; +import org.graylog.plugins.pipelineprocessor.processors.listeners.InterpreterListener; +import org.graylog.plugins.pipelineprocessor.processors.listeners.NoopInterpreterListener; +import org.graylog2.metrics.CacheStatsSet; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.MessageCollection; +import org.graylog2.plugin.Messages; +import org.graylog2.plugin.messageprocessors.MessageProcessor; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.shared.buffers.processors.ProcessBufferProcessor; +import org.graylog2.shared.journal.Journal; +import org.graylog2.shared.metrics.MetricUtils; +import org.graylog2.shared.utilities.ExceptionUtils; +import org.jooq.lambda.tuple.Tuple2; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.inject.Named; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import static com.codahale.metrics.MetricRegistry.name; +import static org.jooq.lambda.tuple.Tuple.tuple; + +public class PipelineInterpreter implements MessageProcessor { + private static final Logger log = LoggerFactory.getLogger(PipelineInterpreter.class); + + public static final String GL2_PROCESSING_ERROR = "gl2_processing_error"; + + private final Journal journal; + private final Meter filteredOutMessages; + private final Timer executionTime; + private final ConfigurationStateUpdater stateUpdater; + + @Inject + public PipelineInterpreter(Journal journal, + MetricRegistry metricRegistry, + ConfigurationStateUpdater stateUpdater) { + + this.journal = journal; + this.filteredOutMessages = metricRegistry.meter(name(ProcessBufferProcessor.class, "filteredOutMessages")); + this.executionTime = metricRegistry.timer(name(PipelineInterpreter.class, "executionTime")); + this.stateUpdater = stateUpdater; + } + + /** + * @param messages messages to process + * @return messages to pass on to the next stage + */ + @Override + public Messages process(Messages messages) { + try (Timer.Context ignored = executionTime.time()) { + final State latestState = stateUpdater.getLatestState(); + return process(messages, new NoopInterpreterListener(), latestState); + } + } + + /** + * Evaluates all pipelines that apply to the given messages, based on the current stream routing + * of the messages. + * + * The processing loops on each single message (passed in or created by pipelines) until the set + * of streams does not change anymore. No cycle detection is performed. + * + * @param messages the messages to process through the pipelines + * @param interpreterListener a listener which gets called for each processing stage (e.g. to + * trace execution) + * @param state the pipeline/stage/rule/stream connection state to use during + * processing + * @return the processed messages + */ + public Messages process(Messages messages, InterpreterListener interpreterListener, State state) { + interpreterListener.startProcessing(); + // message id + stream id + final Set> processingBlacklist = Sets.newHashSet(); + + final List toProcess = Lists.newArrayList(messages); + final List fullyProcessed = Lists.newArrayListWithExpectedSize(toProcess.size()); + + while (!toProcess.isEmpty()) { + final MessageCollection currentSet = new MessageCollection(toProcess); + // we'll add them back below + toProcess.clear(); + + for (Message message : currentSet) { + final String msgId = message.getId(); + + // this makes a copy of the list, which is mutated later in updateStreamBlacklist + // it serves as a worklist, to keep track of which tuples need to be re-run again + final Set initialStreamIds = message.getStreams().stream().map(Stream::getId).collect(Collectors.toSet()); + + final ImmutableSet pipelinesToRun = selectPipelines(interpreterListener, + processingBlacklist, + message, + initialStreamIds, + state.getStreamPipelineConnections()); + + toProcess.addAll(processForResolvedPipelines(message, msgId, pipelinesToRun, interpreterListener, state)); + + // add each processed message-stream combination to the blacklist set and figure out if the processing + // has added a stream to the message, in which case we need to cycle and determine whether to process + // its pipeline connections, too + boolean addedStreams = updateStreamBlacklist(processingBlacklist, + message, + initialStreamIds); + potentiallyDropFilteredMessage(message); + + // go to 1 and iterate over all messages again until no more streams are being assigned + if (!addedStreams || message.getFilterOut()) { + log.debug("[{}] no new streams matches or dropped message, not running again", msgId); + fullyProcessed.add(message); + } else { + // process again, we've added a stream + log.debug("[{}] new streams assigned, running again for those streams", msgId); + toProcess.add(message); + } + } + } + + interpreterListener.finishProcessing(); + // 7. return the processed messages + return new MessageCollection(fullyProcessed); + } + + private void potentiallyDropFilteredMessage(Message message) { + if (message.getFilterOut()) { + log.debug("[{}] marked message to be discarded. Dropping message.", message.getId()); + filteredOutMessages.mark(); + journal.markJournalOffsetCommitted(message.getJournalOffset()); + } + } + + // given the initial streams the message was on before the processing and its current state, update the set of + // that should not be run again (which prevents re-running pipelines over and over again) + private boolean updateStreamBlacklist(Set> processingBlacklist, + Message message, + Set initialStreamIds) { + boolean addedStreams = false; + for (Stream stream : message.getStreams()) { + if (!initialStreamIds.remove(stream.getId())) { + addedStreams = true; + } else { + // only add pre-existing streams to blacklist, this has the effect of only adding already processed streams, + // not newly added ones. + processingBlacklist.add(tuple(message.getId(), stream.getId())); + } + } + return addedStreams; + } + + // determine which pipelines should be executed give the stream-pipeline connections and the current message + // the initialStreamIds are not mutated, but are begin passed for efficiency, as they are being used later in #process() + private ImmutableSet selectPipelines(InterpreterListener interpreterListener, + Set> processingBlacklist, + Message message, + Set initialStreamIds, + ImmutableSetMultimap streamConnection) { + final String msgId = message.getId(); + + // if a message-stream combination has already been processed (is in the set), skip that execution + final Set streamsIds = initialStreamIds.stream() + .filter(streamId -> !processingBlacklist.contains(tuple(msgId, streamId))) + .filter(streamConnection::containsKey) + .collect(Collectors.toSet()); + final ImmutableSet pipelinesToRun = streamsIds.stream() + .flatMap(streamId -> streamConnection.get(streamId).stream()) + .collect(ImmutableSet.toImmutableSet()); + interpreterListener.processStreams(message, pipelinesToRun, streamsIds); + log.debug("[{}] running pipelines {} for streams {}", msgId, pipelinesToRun, streamsIds); + return pipelinesToRun; + } + + /** + * Given a set of pipeline ids, process the given message according to the passed state. + * + * This method returns the list of messages produced by the configuration in state, it does not + * look at the database or any other external resource besides what is being passed as + * parameters. + * + * This can be used to simulate pipelines without having to store them in the database. + * + * @param message the message to process + * @param pipelineIds the ids of the pipelines to resolve and run the message through + * @param interpreterListener the listener tracing the execution + * @param state the pipeline/stage/rule state to interpret + * @return the list of messages created during the interpreter run + */ + public List processForPipelines(Message message, + Set pipelineIds, + InterpreterListener interpreterListener, + State state) { + final Map currentPipelines = state.getCurrentPipelines(); + final ImmutableSet pipelinesToRun = pipelineIds.stream() + .map(currentPipelines::get) + .filter(Objects::nonNull) + .collect(ImmutableSet.toImmutableSet()); + + return processForResolvedPipelines(message, message.getId(), pipelinesToRun, interpreterListener, state); + } + + private List processForResolvedPipelines(Message message, + String msgId, + Set pipelines, + InterpreterListener interpreterListener, + State state) { + final List result = new ArrayList<>(); + // record execution of pipeline in metrics + pipelines.forEach(Pipeline::markExecution); + + final StageIterator stages = state.getStageIterator(pipelines); + final Set pipelinesToSkip = Sets.newHashSet(); + + // iterate through all stages for all matching pipelines, per "stage slice" instead of per pipeline. + // pipeline execution ordering is not guaranteed + while (stages.hasNext()) { + final List stageSet = stages.next(); + for (final Stage stage : stageSet) + evaluateStage(stage, message, msgId, result, pipelinesToSkip, interpreterListener); + } + + // 7. return the processed messages + return result; + } + + private void evaluateStage(Stage stage, + Message message, + String msgId, + List result, + Set pipelinesToSkip, + InterpreterListener interpreterListener) { + final Pipeline pipeline = stage.getPipeline(); + if (pipelinesToSkip.contains(pipeline)) { + log.debug("[{}] previous stage result prevents further processing of pipeline `{}`", + msgId, + pipeline.name()); + return; + } + stage.markExecution(); + interpreterListener.enterStage(stage); + log.debug("[{}] evaluating rule conditions in stage {}: match {}", + msgId, + stage.stage(), + stage.matchAll() ? "all" : "either"); + + // TODO the message should be decorated to allow layering changes and isolate stages + final EvaluationContext context = new EvaluationContext(message); + + // 3. iterate over all the stages in these pipelines and execute them in order + final List stageRules = stage.getRules(); + final List rulesToRun = new ArrayList<>(stageRules.size()); + boolean anyRulesMatched = stageRules.isEmpty(); // If there are no rules, we can simply continue to the next stage + boolean allRulesMatched = true; + for (Rule rule : stageRules) { + final boolean ruleCondition = evaluateRuleCondition(rule, message, msgId, pipeline, context, rulesToRun, interpreterListener); + anyRulesMatched |= ruleCondition; + allRulesMatched &= ruleCondition; + } + + for (Rule rule : rulesToRun) { + if (!executeRuleActions(rule, message, msgId, pipeline, context, interpreterListener)) { + // if any of the rules raise an error, skip the rest of the rules + break; + } + } + // stage needed to match all rule conditions to enable the next stage, + // record that it is ok to proceed with this pipeline + // OR + // any rule could match, but at least one had to, + // record that it is ok to proceed with the pipeline + final boolean matchAllSuccess = stage.matchAll() && allRulesMatched; + final boolean matchEitherSuccess = !stage.matchAll() && anyRulesMatched; + if (matchAllSuccess || matchEitherSuccess) { + interpreterListener.continuePipelineExecution(pipeline, stage); + log.debug("[{}] stage {} for pipeline `{}` required match: {}, ok to proceed with next stage", + msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); + } else { + // no longer execute stages from this pipeline, the guard prevents it + interpreterListener.stopPipelineExecution(pipeline, stage); + log.debug("[{}] stage {} for pipeline `{}` required match: {}, NOT ok to proceed with next stage", + msgId, stage.stage(), pipeline.name(), stage.matchAll() ? "all" : "either"); + pipelinesToSkip.add(pipeline); + } + + // 4. after each complete stage run, merge the processing changes, stages are isolated from each other + // TODO message changes become visible immediately for now + + // 4a. also add all new messages from the context to the toProcess work list + Iterables.addAll(result, context.createdMessages()); + context.clearCreatedMessages(); + interpreterListener.exitStage(stage); + } + + private boolean executeRuleActions(Rule rule, + Message message, + String msgId, + Pipeline pipeline, + EvaluationContext context, + InterpreterListener interpreterListener) { + rule.markExecution(); + interpreterListener.executeRule(rule, pipeline); + log.debug("[{}] rule `{}` matched running actions", msgId, rule.name()); + final GeneratedRule generatedRule = rule.generatedRule(); + if (generatedRule != null) { + try { + generatedRule.then(context); + return true; + } catch (Exception ignored) { + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(rule, message, lastError.toString()); + log.debug("Encountered evaluation error, skipping rest of the rule: {}", lastError); + rule.markFailure(); + return false; + } + } else { + if (ConfigurationStateUpdater.isAllowCodeGeneration()) { + throw new IllegalStateException("Should have generated code and not interpreted the tree"); + } + for (Statement statement : rule.then()) { + if (!evaluateStatement(message, interpreterListener, pipeline, context, rule, statement)) { + // statement raised an error, skip the rest of the rule + return false; + } + } + } + return true; + } + + private boolean evaluateStatement(Message message, + InterpreterListener interpreterListener, + Pipeline pipeline, + EvaluationContext context, Rule rule, Statement statement) { + statement.evaluate(context); + if (context.hasEvaluationErrors()) { + // if the last statement resulted in an error, do not continue to execute this rules + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(rule, message, lastError.toString()); + interpreterListener.failExecuteRule(rule, pipeline); + log.debug("Encountered evaluation error, skipping rest of the rule: {}", + lastError); + rule.markFailure(); + return false; + } + return true; + } + + private boolean evaluateRuleCondition(Rule rule, + Message message, + String msgId, + Pipeline pipeline, + EvaluationContext context, + List rulesToRun, InterpreterListener interpreterListener) { + interpreterListener.evaluateRule(rule, pipeline); + final GeneratedRule generatedRule = rule.generatedRule(); + boolean matched = generatedRule != null ? generatedRule.when(context) : rule.when().evaluateBool(context); + if (matched) { + rule.markMatch(); + + if (context.hasEvaluationErrors()) { + final EvaluationContext.EvalError lastError = Iterables.getLast(context.evaluationErrors()); + appendProcessingError(rule, message, lastError.toString()); + interpreterListener.failEvaluateRule(rule, pipeline); + log.debug("Encountered evaluation error during condition, skipping rule actions: {}", + lastError); + return false; + } + interpreterListener.satisfyRule(rule, pipeline); + log.debug("[{}] rule `{}` matches, scheduling to run", msgId, rule.name()); + rulesToRun.add(rule); + return true; + } else { + rule.markNonMatch(); + interpreterListener.dissatisfyRule(rule, pipeline); + log.debug("[{}] rule `{}` does not match", msgId, rule.name()); + } + return false; + } + + private void appendProcessingError(Rule rule, Message message, String errorString) { + final String msg = "For rule '" + rule.name() + "': " + errorString; + if (message.hasField(GL2_PROCESSING_ERROR)) { + message.addField(GL2_PROCESSING_ERROR, message.getFieldAs(String.class, GL2_PROCESSING_ERROR) + "," + msg); + } else { + message.addField(GL2_PROCESSING_ERROR, msg); + } + } + + public static class Descriptor implements MessageProcessor.Descriptor { + @Override + public String name() { + return "Pipeline Processor"; + } + + @Override + public String className() { + return PipelineInterpreter.class.getCanonicalName(); + } + } + + public static class State { + private static final Logger LOG = LoggerFactory.getLogger(State.class); + + private final ImmutableMap currentPipelines; + private final ImmutableSetMultimap streamPipelineConnections; + private final LoadingCache, StageIterator.Configuration> cache; + private final boolean cachedIterators; + + @AssistedInject + public State(@Assisted ImmutableMap currentPipelines, + @Assisted ImmutableSetMultimap streamPipelineConnections, + MetricRegistry metricRegistry, + @Named("processbuffer_processors") int processorCount, + @Named("cached_stageiterators") boolean cachedIterators) { + this.currentPipelines = currentPipelines; + this.streamPipelineConnections = streamPipelineConnections; + this.cachedIterators = cachedIterators; + + cache = CacheBuilder.newBuilder() + .concurrencyLevel(processorCount) + .recordStats() + .build(new CacheLoader, StageIterator.Configuration>() { + @Override + public StageIterator.Configuration load(@Nonnull Set pipelines) throws Exception { + return new StageIterator.Configuration(pipelines); + } + }); + + // we have to remove the metrics, because otherwise we leak references to the cache (and the register call with throw) + metricRegistry.removeMatching((name, metric) -> name.startsWith(name(PipelineInterpreter.class, "stage-cache"))); + MetricUtils.safelyRegisterAll(metricRegistry, new CacheStatsSet(name(PipelineInterpreter.class, "stage-cache"), cache)); + } + + public ImmutableMap getCurrentPipelines() { + return currentPipelines; + } + + public ImmutableSetMultimap getStreamPipelineConnections() { + return streamPipelineConnections; + } + + public StageIterator getStageIterator(Set pipelines) { + try { + if (cachedIterators) { + return new StageIterator(cache.get(pipelines)); + } else { + return new StageIterator(pipelines); + } + } catch (ExecutionException e) { + LOG.error("Unable to get StageIterator from cache, this should not happen.", ExceptionUtils.getRootCause(e)); + return new StageIterator(pipelines); + } + } + + + public interface Factory { + State newState(ImmutableMap currentPipelines, + ImmutableSetMultimap streamPipelineConnections); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java new file mode 100644 index 000000000000..39fec2981d82 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/StageIterator.java @@ -0,0 +1,108 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ArrayListMultimap; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; + +import java.util.List; +import java.util.Set; +import java.util.SortedSet; + +public class StageIterator extends AbstractIterator> { + + private final Configuration config; + + // the currentStage is always one before the next one to be returned + private int currentStage; + + + public StageIterator(Configuration config) { + this.config = config; + currentStage = config.initialStage(); + } + + public StageIterator(Set pipelines) { + this.config = new Configuration(pipelines); + currentStage = config.initialStage(); + } + + @Override + protected List computeNext() { + if (currentStage == config.lastStage()) { + return endOfData(); + } + do { + currentStage++; + if (currentStage > config.lastStage()) { + return endOfData(); + } + } while (!config.hasStages(currentStage)); + return config.getStages(currentStage); + } + + public static class Configuration { + // first and last stage for the given pipelines + private final int[] extent = new int[]{Integer.MAX_VALUE, Integer.MIN_VALUE}; + + private final ArrayListMultimap stageMultimap = ArrayListMultimap.create(); + + private final int initialStage; + + public Configuration(Set pipelines) { + if (pipelines.isEmpty()) { + initialStage = extent[0] = extent[1] = 0; + return; + } + pipelines.forEach(pipeline -> { + // skip pipelines without any stages, they don't contribute any rules to run + final SortedSet stages = pipeline.stages(); + if (stages.isEmpty()) { + return; + } + extent[0] = Math.min(extent[0], stages.first().stage()); + extent[1] = Math.max(extent[1], stages.last().stage()); + stages.forEach(stage -> stageMultimap.put(stage.stage(), stage)); + }); + + if (extent[0] == Integer.MIN_VALUE) { + throw new IllegalArgumentException("First stage cannot be at " + Integer.MIN_VALUE); + } + // the stage before the first stage. + initialStage = extent[0] - 1; + } + + public int initialStage() { + return initialStage; + } + + public int lastStage() { + return extent[1]; + } + + public boolean hasStages(int stage) { + return stageMultimap.containsKey(stage); + } + + public List getStages(int stage) { + return stageMultimap.get(stage); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java new file mode 100644 index 000000000000..57969548d6ae --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java @@ -0,0 +1,40 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors.listeners; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog2.plugin.Message; + +import java.util.Set; + +public interface InterpreterListener { + void startProcessing(); + void finishProcessing(); + void processStreams(Message message, Set pipelines, Set streams); + void enterStage(Stage stage); + void exitStage(Stage stage); + void evaluateRule(Rule rule, Pipeline pipeline); + void failEvaluateRule(Rule rule, Pipeline pipeline); + void satisfyRule(Rule rule, Pipeline pipeline); + void dissatisfyRule(Rule rule, Pipeline pipeline); + void executeRule(Rule rule, Pipeline pipeline); + void failExecuteRule(Rule rule, Pipeline pipeline); + void continuePipelineExecution(Pipeline pipeline, Stage stage); + void stopPipelineExecution(Pipeline pipeline, Stage stage); +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java new file mode 100644 index 000000000000..a2b9eca6b40d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/NoopInterpreterListener.java @@ -0,0 +1,91 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors.listeners; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog2.plugin.Message; + +import java.util.Set; + +public class NoopInterpreterListener implements InterpreterListener { + @Override + public void startProcessing() { + + } + + @Override + public void finishProcessing() { + + } + + @Override + public void processStreams(Message messageId, Set pipelines, Set streams) { + + } + + @Override + public void enterStage(Stage stage) { + + } + + @Override + public void exitStage(Stage stage) { + + } + + @Override + public void evaluateRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void failEvaluateRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void satisfyRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void dissatisfyRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void executeRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void failExecuteRule(Rule rule, Pipeline pipeline) { + + } + + @Override + public void continuePipelineExecution(Pipeline pipeline, Stage stage) { + + } + + @Override + public void stopPipelineExecution(Pipeline pipeline, Stage stage) { + + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java new file mode 100644 index 000000000000..44165891047b --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java @@ -0,0 +1,36 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import java.util.List; + +@AutoValue +@JsonAutoDetect +public abstract class BulkRuleRequest { + @JsonProperty + public abstract List rules(); + + @JsonCreator + public static BulkRuleRequest create(@JsonProperty("rules") List rules) { + return new AutoValue_BulkRuleRequest(rules); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java new file mode 100644 index 000000000000..db11139259df --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnections.java @@ -0,0 +1,72 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; +import java.util.Set; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineConnections { + + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + public abstract String streamId(); + + @JsonProperty + public abstract Set pipelineIds(); + + @JsonCreator + public static PipelineConnections create(@JsonProperty("id") @Id @ObjectId @Nullable String id, + @JsonProperty("stream_id") String streamId, + @JsonProperty("pipeline_ids") Set pipelineIds) { + return builder() + .id(id) + .streamId(streamId) + .pipelineIds(pipelineIds) + .build(); + } + + public static Builder builder() { + return new AutoValue_PipelineConnections.Builder(); + } + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + public abstract PipelineConnections build(); + + public abstract Builder id(String id); + + public abstract Builder streamId(String streamId); + + public abstract Builder pipelineIds(Set pipelineIds); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java new file mode 100644 index 000000000000..bd9c78e231a9 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineConnectionsResource.java @@ -0,0 +1,202 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.events.PipelineConnectionsChangedEvent; +import org.graylog2.audit.jersey.AuditEvent; +import org.graylog2.database.NotFoundException; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.shared.security.RestPermissions; +import org.graylog2.streams.StreamService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.util.Set; +import java.util.stream.Collectors; + +@Api(value = "Pipelines/Connections", description = "Stream connections of processing pipelines") +@Path("/system/pipelines/connections") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication +public class PipelineConnectionsResource extends RestResource implements PluginRestResource { + private static final Logger LOG = LoggerFactory.getLogger(PipelineConnectionsResource.class); + + private final PipelineStreamConnectionsService connectionsService; + private final PipelineService pipelineService; + private final StreamService streamService; + private final EventBus clusterBus; + + @Inject + public PipelineConnectionsResource(PipelineStreamConnectionsService connectionsService, + PipelineService pipelineService, + StreamService streamService, + ClusterEventBus clusterBus) { + this.connectionsService = connectionsService; + this.pipelineService = pipelineService; + this.streamService = streamService; + this.clusterBus = clusterBus; + } + + @ApiOperation(value = "Connect processing pipelines to a stream", notes = "") + @POST + @Path("/to_stream") + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_EDIT) + @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CONNECTION_UPDATE) + public PipelineConnections connectPipelines(@ApiParam(name = "Json body", required = true) @NotNull PipelineConnections connection) throws NotFoundException { + final String streamId = connection.streamId(); + // verify the stream exists + checkPermission(RestPermissions.STREAMS_READ, streamId); + streamService.load(streamId); + + // verify the pipelines exist + for (String s : connection.pipelineIds()) { + checkPermission(PipelineRestPermissions.PIPELINE_READ, s); + pipelineService.load(s); + } + return savePipelineConnections(connection); + } + + @ApiOperation(value = "Connect streams to a processing pipeline", notes = "") + @POST + @Path("/to_pipeline") + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_EDIT) + @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CONNECTION_UPDATE) + public Set connectStreams(@ApiParam(name = "Json body", required = true) @NotNull PipelineReverseConnections connection) throws NotFoundException { + final String pipelineId = connection.pipelineId(); + final Set updatedConnections = Sets.newHashSet(); + + // verify the pipeline exists + checkPermission(PipelineRestPermissions.PIPELINE_READ, pipelineId); + pipelineService.load(pipelineId); + + // get all connections where the pipeline was present + final Set pipelineConnections = connectionsService.loadAll().stream() + .filter(p -> p.pipelineIds().contains(pipelineId)) + .collect(Collectors.toSet()); + + // remove deleted pipeline connections + for (PipelineConnections pipelineConnection : pipelineConnections) { + if (!connection.streamIds().contains(pipelineConnection.streamId())) { + final Set pipelines = pipelineConnection.pipelineIds(); + pipelines.remove(connection.pipelineId()); + pipelineConnection.toBuilder().pipelineIds(pipelines).build(); + + updatedConnections.add(pipelineConnection); + savePipelineConnections(pipelineConnection); + LOG.debug("Deleted stream {} connection with pipeline {}", pipelineConnection.streamId(), pipelineId); + } + } + + // update pipeline connections + for (String streamId : connection.streamIds()) { + // verify the stream exist + checkPermission(RestPermissions.STREAMS_READ, streamId); + streamService.load(streamId); + + PipelineConnections updatedConnection; + try { + updatedConnection = connectionsService.load(streamId); + } catch (NotFoundException e) { + updatedConnection = PipelineConnections.create(null, streamId, Sets.newHashSet()); + } + + final Set pipelines = updatedConnection.pipelineIds(); + pipelines.add(pipelineId); + updatedConnection.toBuilder().pipelineIds(pipelines).build(); + + updatedConnections.add(updatedConnection); + savePipelineConnections(updatedConnection); + LOG.debug("Added stream {} connection with pipeline {}", streamId, pipelineId); + } + + return updatedConnections; + } + + @ApiOperation("Get pipeline connections for the given stream") + @GET + @Path("/{streamId}") + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_READ) + public PipelineConnections getPipelinesForStream(@ApiParam(name = "streamId") @PathParam("streamId") String streamId) throws NotFoundException { + // the user needs to at least be able to read the stream + checkPermission(RestPermissions.STREAMS_READ, streamId); + + final PipelineConnections connections = connectionsService.load(streamId); + // filter out all pipelines the user does not have enough permissions to see + return PipelineConnections.create( + connections.id(), + connections.streamId(), + connections.pipelineIds() + .stream() + .filter(id -> isPermitted(PipelineRestPermissions.PIPELINE_READ, id)) + .collect(Collectors.toSet()) + ); + } + + @ApiOperation("Get all pipeline connections") + @GET + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CONNECTION_READ) + public Set getAll() throws NotFoundException { + final Set pipelineConnections = connectionsService.loadAll(); + + final Set filteredConnections = Sets.newHashSetWithExpectedSize(pipelineConnections.size()); + for (PipelineConnections pc : pipelineConnections) { + // only include the streams the user can see + if (isPermitted(RestPermissions.STREAMS_READ, pc.streamId())) { + // filter out all pipelines the user does not have enough permissions to see + filteredConnections.add(PipelineConnections.create( + pc.id(), + pc.streamId(), + pc.pipelineIds() + .stream() + .filter(id -> isPermitted(PipelineRestPermissions.PIPELINE_READ, id)) + .collect(Collectors.toSet())) + ); + } + } + + return filteredConnections; + } + + private PipelineConnections savePipelineConnections(PipelineConnections connection) { + final PipelineConnections save = connectionsService.save(connection); + clusterBus.post(PipelineConnectionsChangedEvent.create(save.streamId(), save.pipelineIds())); + return save; + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java new file mode 100644 index 000000000000..bf7069e49b6d --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineResource.java @@ -0,0 +1,190 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.collect.Lists; +import com.google.common.eventbus.EventBus; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.events.PipelinesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog2.audit.jersey.AuditEvent; +import org.graylog2.audit.jersey.NoAuditEvent; +import org.graylog2.database.NotFoundException; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.shared.rest.resources.RestResource; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.Collection; + +@Api(value = "Pipelines/Pipelines", description = "Pipelines for the pipeline message processor") +@Path("/system/pipelines/pipeline") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication +public class PipelineResource extends RestResource implements PluginRestResource { + + private static final Logger log = LoggerFactory.getLogger(PipelineResource.class); + + private final PipelineService pipelineService; + private final PipelineRuleParser pipelineRuleParser; + private final EventBus clusterBus; + + @Inject + public PipelineResource(PipelineService pipelineService, + PipelineRuleParser pipelineRuleParser, + ClusterEventBus clusterBus) { + this.pipelineService = pipelineService; + this.pipelineRuleParser = pipelineRuleParser; + this.clusterBus = clusterBus; + } + + @ApiOperation(value = "Create a processing pipeline from source", notes = "") + @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_CREATE) + @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_CREATE) + public PipelineSource createFromParser(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { + final Pipeline pipeline; + try { + pipeline = pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final DateTime now = DateTime.now(DateTimeZone.UTC); + final PipelineDao pipelineDao = PipelineDao.builder() + .title(pipeline.name()) + .description(pipelineSource.description()) + .source(pipelineSource.source()) + .createdAt(now) + .modifiedAt(now) + .build(); + final PipelineDao save = pipelineService.save(pipelineDao); + clusterBus.post(PipelinesChangedEvent.updatedPipelineId(save.id())); + log.debug("Created new pipeline {}", save); + return PipelineSource.fromDao(pipelineRuleParser, save); + } + + @ApiOperation(value = "Parse a processing pipeline without saving it", notes = "") + @POST + @Path("/parse") + @NoAuditEvent("only used to parse a pipeline, no changes made in the system") + public PipelineSource parse(@ApiParam(name = "pipeline", required = true) @NotNull PipelineSource pipelineSource) throws ParseException { + final Pipeline pipeline; + try { + pipeline = pipelineRuleParser.parsePipeline(pipelineSource.id(), pipelineSource.source()); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final DateTime now = DateTime.now(DateTimeZone.UTC); + return PipelineSource.builder() + .title(pipeline.name()) + .description(pipelineSource.description()) + .source(pipelineSource.source()) + .createdAt(now) + .modifiedAt(now) + .build(); + } + + @ApiOperation(value = "Get all processing pipelines") + @GET + public Collection getAll() { + final Collection daos = pipelineService.loadAll(); + final ArrayList results = Lists.newArrayList(); + for (PipelineDao dao : daos) { + if (isPermitted(PipelineRestPermissions.PIPELINE_READ, dao.id())) { + results.add(PipelineSource.fromDao(pipelineRuleParser, dao)); + } + } + + return results; + } + + @ApiOperation(value = "Get a processing pipeline", notes = "It can take up to a second until the change is applied") + @Path("/{id}") + @GET + public PipelineSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_READ, id); + final PipelineDao dao = pipelineService.load(id); + return PipelineSource.fromDao(pipelineRuleParser, dao); + } + + @ApiOperation(value = "Modify a processing pipeline", notes = "It can take up to a second until the change is applied") + @Path("/{id}") + @PUT + @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_UPDATE) + public PipelineSource update(@ApiParam(name = "id") @PathParam("id") String id, + @ApiParam(name = "pipeline", required = true) @NotNull PipelineSource update) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_EDIT, id); + + final PipelineDao dao = pipelineService.load(id); + final Pipeline pipeline; + try { + pipeline = pipelineRuleParser.parsePipeline(update.id(), update.source()); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final PipelineDao toSave = dao.toBuilder() + .title(pipeline.name()) + .description(update.description()) + .source(update.source()) + .modifiedAt(DateTime.now(DateTimeZone.UTC)) + .build(); + final PipelineDao savedPipeline = pipelineService.save(toSave); + clusterBus.post(PipelinesChangedEvent.updatedPipelineId(savedPipeline.id())); + + return PipelineSource.fromDao(pipelineRuleParser, savedPipeline); + } + + @ApiOperation(value = "Delete a processing pipeline", notes = "It can take up to a second until the change is applied") + @Path("/{id}") + @DELETE + @AuditEvent(type = PipelineProcessorAuditEventTypes.PIPELINE_DELETE) + public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_DELETE, id); + + pipelineService.load(id); + pipelineService.delete(id); + clusterBus.post(PipelinesChangedEvent.deletedPipelineId(id)); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java new file mode 100644 index 000000000000..f71503ef3156 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineRestPermissions.java @@ -0,0 +1,69 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.collect.ImmutableSet; +import org.graylog2.plugin.security.Permission; +import org.graylog2.plugin.security.PluginPermissions; + +import java.util.Collections; +import java.util.Set; + +import static org.graylog2.plugin.security.Permission.create; + +public class PipelineRestPermissions implements PluginPermissions { + + /* pipelines */ + public static final String PIPELINE_CREATE = "pipeline:create"; + public static final String PIPELINE_READ = "pipeline:read"; + public static final String PIPELINE_EDIT = "pipeline:edit"; + public static final String PIPELINE_DELETE = "pipeline:delete"; + + /* rules */ + public static final String PIPELINE_RULE_CREATE = "pipeline_rule:create"; + public static final String PIPELINE_RULE_READ = "pipeline_rule:read"; + public static final String PIPELINE_RULE_EDIT = "pipeline_rule:edit"; + public static final String PIPELINE_RULE_DELETE = "pipeline_rule:delete"; + + /* connections */ + public static final String PIPELINE_CONNECTION_READ = "pipeline_connection:read"; + public static final String PIPELINE_CONNECTION_EDIT = "pipeline_connection:edit"; + + + @Override + public Set permissions() { + return ImmutableSet.of( + create(PIPELINE_CREATE, "Create new processing pipeline"), + create(PIPELINE_READ, "Read a processing pipeline"), + create(PIPELINE_EDIT, "Update a processing pipeline"), + create(PIPELINE_DELETE, "Delete a processing pipeline"), + + create(PIPELINE_RULE_CREATE, "Create new processing rule"), + create(PIPELINE_RULE_READ, "Read a processing rule"), + create(PIPELINE_RULE_EDIT, "Update a processing rule"), + create(PIPELINE_RULE_DELETE, "Delete a processing rule"), + + create(PIPELINE_CONNECTION_READ, "Read a pipeline stream connection"), + create(PIPELINE_CONNECTION_EDIT, "Update a pipeline stream connections") + ); + } + + @Override + public Set readerBasePermissions() { + return Collections.emptySet(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java new file mode 100644 index 000000000000..178058989ddd --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java @@ -0,0 +1,40 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import java.util.Set; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineReverseConnections { + @JsonProperty + public abstract String pipelineId(); + + @JsonProperty + public abstract Set streamIds(); + + @JsonCreator + public static PipelineReverseConnections create(@JsonProperty("pipeline_id") String pipelineId, + @JsonProperty("stream_ids") Set streamIds) { + return new AutoValue_PipelineReverseConnections(pipelineId, streamIds); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java new file mode 100644 index 000000000000..61cfd35ee181 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSource.java @@ -0,0 +1,148 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; +import org.joda.time.DateTime; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineSource { + + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty("title") + @Nullable + public abstract String title(); + + @JsonProperty("description") + @Nullable + public abstract String description(); + + @JsonProperty("source") + public abstract String source(); + + @JsonProperty("created_at") + @Nullable + public abstract DateTime createdAt(); + + @JsonProperty("modified_at") + @Nullable + public abstract DateTime modifiedAt(); + + @JsonProperty("stages") + public abstract List stages(); + + @JsonProperty("errors") + @Nullable + public abstract Set errors(); + + public static Builder builder() { + return new AutoValue_PipelineSource.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static PipelineSource create(@JsonProperty("id") @Id @ObjectId @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @Nullable @JsonProperty("stages") List stages, + @Nullable @JsonProperty("created_at") DateTime createdAt, + @Nullable @JsonProperty("modified_at") DateTime modifiedAt) { + return builder() + .id(id) + .title(title) + .description(description) + .source(source) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .stages(stages == null ? Collections.emptyList() : stages) + .build(); + } + + public static PipelineSource fromDao(PipelineRuleParser parser, PipelineDao dao) { + Set errors = null; + Pipeline pipeline = null; + try { + pipeline = parser.parsePipeline(dao.id(), dao.source()); + } catch (ParseException e) { + errors = e.getErrors(); + } + final List stageSources = (pipeline == null) ? Collections.emptyList() : + pipeline.stages().stream() + .map(stage -> StageSource.builder() + .matchAll(stage.matchAll()) + .rules(stage.ruleReferences()) + .stage(stage.stage()) + .build()) + .collect(Collectors.toList()); + + return builder() + .id(dao.id()) + .title(dao.title()) + .description(dao.description()) + .source(dao.source()) + .createdAt(dao.createdAt()) + .modifiedAt(dao.modifiedAt()) + .stages(stageSources) + .errors(errors) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract PipelineSource build(); + + public abstract Builder id(String id); + + public abstract Builder title(String title); + + public abstract Builder description(String description); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + + public abstract Builder stages(List stages); + + public abstract Builder errors(Set errors); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java new file mode 100644 index 000000000000..b5388215ebb4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleResource.java @@ -0,0 +1,218 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.google.common.eventbus.EventBus; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog.plugins.pipelineprocessor.events.RulesChangedEvent; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog2.audit.jersey.AuditEvent; +import org.graylog2.audit.jersey.NoAuditEvent; +import org.graylog2.database.NotFoundException; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.shared.rest.resources.RestResource; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Collection; +import java.util.stream.Collectors; + +@Api(value = "Pipelines/Rules", description = "Rules for the pipeline message processor") +@Path("/system/pipelines/rule") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication +public class RuleResource extends RestResource implements PluginRestResource { + + private static final Logger log = LoggerFactory.getLogger(RuleResource.class); + + private final RuleService ruleService; + private final PipelineRuleParser pipelineRuleParser; + private final EventBus clusterBus; + private final FunctionRegistry functionRegistry; + + @Inject + public RuleResource(RuleService ruleService, + PipelineRuleParser pipelineRuleParser, + ClusterEventBus clusterBus, + FunctionRegistry functionRegistry) { + this.ruleService = ruleService; + this.pipelineRuleParser = pipelineRuleParser; + this.clusterBus = clusterBus; + this.functionRegistry = functionRegistry; + } + + + @ApiOperation(value = "Create a processing rule from source", notes = "") + @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_CREATE) + @AuditEvent(type = PipelineProcessorAuditEventTypes.RULE_CREATE) + public RuleSource createFromParser(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { + final Rule rule; + try { + rule = pipelineRuleParser.parseRule(ruleSource.id(), ruleSource.source(), false); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final DateTime now = DateTime.now(DateTimeZone.UTC); + final RuleDao newRuleSource = RuleDao.builder() + .title(rule.name()) // use the name from the parsed rule source. + .description(ruleSource.description()) + .source(ruleSource.source()) + .createdAt(now) + .modifiedAt(now) + .build(); + final RuleDao save = ruleService.save(newRuleSource); + // TODO determine which pipelines could change because of this new rule (there could be pipelines referring to a previously unresolved rule) + clusterBus.post(RulesChangedEvent.updatedRuleId(save.id())); + log.debug("Created new rule {}", save); + return RuleSource.fromDao(pipelineRuleParser, save); + } + + @ApiOperation(value = "Parse a processing rule without saving it", notes = "") + @POST + @Path("/parse") + @NoAuditEvent("only used to parse a rule, no changes made in the system") + public RuleSource parse(@ApiParam(name = "rule", required = true) @NotNull RuleSource ruleSource) throws ParseException { + final Rule rule; + try { + // be silent about parse errors here, many requests will result in invalid syntax + rule = pipelineRuleParser.parseRule(ruleSource.id(), ruleSource.source(), true); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final DateTime now = DateTime.now(DateTimeZone.UTC); + return RuleSource.builder() + .title(rule.name()) + .description(ruleSource.description()) + .source(ruleSource.source()) + .createdAt(now) + .modifiedAt(now) + .build(); + } + + @ApiOperation(value = "Get all processing rules") + @GET + @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_READ) + public Collection getAll() { + final Collection ruleDaos = ruleService.loadAll(); + return ruleDaos.stream() + .map(ruleDao -> RuleSource.fromDao(pipelineRuleParser, ruleDao)) + .collect(Collectors.toList()); + } + + @ApiOperation(value = "Get a processing rule", notes = "It can take up to a second until the change is applied") + @Path("/{id}") + @GET + public RuleSource get(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_RULE_READ, id); + return RuleSource.fromDao(pipelineRuleParser, ruleService.load(id)); + } + + @ApiOperation("Retrieve the named processing rules in bulk") + @Path("/multiple") + @POST + @NoAuditEvent("only used to get multiple pipeline rules") + public Collection getBulk(@ApiParam("rules") BulkRuleRequest rules) { + Collection ruleDaos = ruleService.loadNamed(rules.rules()); + + return ruleDaos.stream() + .map(ruleDao -> RuleSource.fromDao(pipelineRuleParser, ruleDao)) + .filter(rule -> isPermitted(PipelineRestPermissions.PIPELINE_RULE_READ, rule.id())) + .collect(Collectors.toList()); + } + + @ApiOperation(value = "Modify a processing rule", notes = "It can take up to a second until the change is applied") + @Path("/{id}") + @PUT + @AuditEvent(type = PipelineProcessorAuditEventTypes.RULE_UPDATE) + public RuleSource update(@ApiParam(name = "id") @PathParam("id") String id, + @ApiParam(name = "rule", required = true) @NotNull RuleSource update) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_RULE_EDIT, id); + + final RuleDao ruleDao = ruleService.load(id); + final Rule rule; + try { + rule = pipelineRuleParser.parseRule(id, update.source(), false); + } catch (ParseException e) { + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST).entity(e.getErrors()).build()); + } + final RuleDao toSave = ruleDao.toBuilder() + .title(rule.name()) + .description(update.description()) + .source(update.source()) + .modifiedAt(DateTime.now(DateTimeZone.UTC)) + .build(); + final RuleDao savedRule = ruleService.save(toSave); + + // TODO determine which pipelines could change because of this updated rule + clusterBus.post(RulesChangedEvent.updatedRuleId(savedRule.id())); + + return RuleSource.fromDao(pipelineRuleParser, savedRule); + } + + @ApiOperation(value = "Delete a processing rule", notes = "It can take up to a second until the change is applied") + @Path("/{id}") + @DELETE + @AuditEvent(type = PipelineProcessorAuditEventTypes.RULE_DELETE) + public void delete(@ApiParam(name = "id") @PathParam("id") String id) throws NotFoundException { + checkPermission(PipelineRestPermissions.PIPELINE_RULE_DELETE, id); + ruleService.load(id); + ruleService.delete(id); + + // TODO determine which pipelines could change because of this deleted rule, causing them to recompile + clusterBus.post(RulesChangedEvent.deletedRuleId(id)); + } + + + @ApiOperation("Get function descriptors") + @Path("/functions") + @GET + public Collection functionDescriptors() { + return functionRegistry.all().stream() + .map(Function::descriptor) + .collect(Collectors.toList()); + } + +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java new file mode 100644 index 000000000000..6280e2680188 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/RuleSource.java @@ -0,0 +1,128 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; +import org.joda.time.DateTime; +import org.mongojack.Id; +import org.mongojack.ObjectId; + +import javax.annotation.Nullable; +import java.util.Set; + +@AutoValue +@JsonAutoDetect +public abstract class RuleSource { + + @JsonProperty("id") + @Nullable + @Id + @ObjectId + public abstract String id(); + + @JsonProperty + @Nullable + public abstract String title(); + + @JsonProperty + @Nullable + public abstract String description(); + + @JsonProperty + public abstract String source(); + + @JsonProperty + @Nullable + public abstract DateTime createdAt(); + + @JsonProperty + @Nullable + public abstract DateTime modifiedAt(); + + @JsonProperty + @Nullable + public abstract Set errors(); + + public static Builder builder() { + return new AutoValue_RuleSource.Builder(); + } + + public abstract Builder toBuilder(); + + @JsonCreator + public static RuleSource create(@JsonProperty("id") @Id @ObjectId @Nullable String id, + @JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @JsonProperty("created_at") @Nullable DateTime createdAt, + @JsonProperty("modified_at") @Nullable DateTime modifiedAt) { + return builder() + .id(id) + .source(source) + .title(title) + .description(description) + .createdAt(createdAt) + .modifiedAt(modifiedAt) + .build(); + } + + public static RuleSource fromDao(PipelineRuleParser parser, RuleDao dao) { + Set errors = null; + try { + parser.parseRule(dao.id(), dao.source(), false); + + } catch (ParseException e) { + errors = e.getErrors(); + } + + return builder() + .id(dao.id()) + .source(dao.source()) + .title(dao.title()) + .description(dao.description()) + .createdAt(dao.createdAt()) + .modifiedAt(dao.modifiedAt()) + .errors(errors) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract RuleSource build(); + + public abstract Builder id(String id); + + public abstract Builder title(String title); + + public abstract Builder description(String description); + + public abstract Builder source(String source); + + public abstract Builder createdAt(DateTime createdAt); + + public abstract Builder modifiedAt(DateTime modifiedAt); + + public abstract Builder errors(Set errors); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java new file mode 100644 index 000000000000..e812e2856a32 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationRequest.java @@ -0,0 +1,65 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import javax.annotation.Nullable; +import java.util.Map; + +@AutoValue +@JsonAutoDetect +public abstract class SimulationRequest { + @JsonProperty + public abstract String streamId(); + + @JsonProperty + public abstract Map message(); + + @JsonProperty + @Nullable + public abstract String inputId(); + + public static Builder builder() { + return new AutoValue_SimulationRequest.Builder(); + } + + @JsonCreator + public static SimulationRequest create (@JsonProperty("stream_id") String streamId, + @JsonProperty("message") Map message, + @JsonProperty("input_id") @Nullable String inputId) { + return builder() + .streamId(streamId) + .message(message) + .inputId(inputId) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract SimulationRequest build(); + + public abstract Builder streamId(String streamId); + + public abstract Builder message(Map message); + + public abstract Builder inputId(String inputId); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java new file mode 100644 index 000000000000..baa0733997d1 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulationResponse.java @@ -0,0 +1,65 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTrace; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; + +import java.util.List; + +@AutoValue +@JsonAutoDetect +public abstract class SimulationResponse { + @JsonProperty + public abstract List messages(); + + @JsonProperty + public abstract List simulationTrace(); + + @JsonProperty + public abstract long tookMicroseconds(); + + public static SimulationResponse.Builder builder() { + return new AutoValue_SimulationResponse.Builder(); + } + + @JsonCreator + public static SimulationResponse create (@JsonProperty("messages") List messages, + @JsonProperty("simulation_trace") List simulationTrace, + @JsonProperty("took_microseconds") long tookMicroseconds) { + return builder() + .messages(messages) + .simulationTrace(simulationTrace) + .tookMicroseconds(tookMicroseconds) + .build(); + } + + @AutoValue.Builder + public abstract static class Builder { + public abstract SimulationResponse build(); + + public abstract SimulationResponse.Builder messages(List messages); + + public abstract SimulationResponse.Builder simulationTrace(List trace); + + public abstract SimulationResponse.Builder tookMicroseconds(long tookMicroseconds); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java new file mode 100644 index 000000000000..81d30d58024f --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/SimulatorResource.java @@ -0,0 +1,98 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import org.apache.shiro.authz.annotation.RequiresAuthentication; +import org.apache.shiro.authz.annotation.RequiresPermissions; +import org.elasticsearch.common.Strings; +import org.graylog.plugins.pipelineprocessor.processors.ConfigurationStateUpdater; +import org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter; +import org.graylog.plugins.pipelineprocessor.simulator.PipelineInterpreterTracer; +import org.graylog2.audit.jersey.NoAuditEvent; +import org.graylog2.database.NotFoundException; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.rest.PluginRestResource; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.rest.models.messages.responses.ResultMessageSummary; +import org.graylog2.shared.rest.resources.RestResource; +import org.graylog2.shared.security.RestPermissions; +import org.graylog2.streams.StreamService; + +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Inject; +import javax.validation.constraints.NotNull; +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; + +@Api(value = "Pipelines/Simulator", description = "Simulate pipeline message processor") +@Path("/system/pipelines/simulate") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@RequiresAuthentication +public class SimulatorResource extends RestResource implements PluginRestResource { + private final ConfigurationStateUpdater pipelineStateUpdater; + private final StreamService streamService; + private final PipelineInterpreter pipelineInterpreter; + + @Inject + public SimulatorResource(PipelineInterpreter pipelineInterpreter, + ConfigurationStateUpdater pipelineStateUpdater, + StreamService streamService) { + this.pipelineInterpreter = pipelineInterpreter; + this.pipelineStateUpdater = pipelineStateUpdater; + this.streamService = streamService; + } + + @ApiOperation(value = "Simulate the execution of the pipeline message processor") + @POST + @RequiresPermissions(PipelineRestPermissions.PIPELINE_RULE_READ) + @NoAuditEvent("only used to test pipelines, no changes made in the system") + public SimulationResponse simulate(@ApiParam(name = "simulation", required = true) @NotNull SimulationRequest request) throws NotFoundException { + checkPermission(RestPermissions.STREAMS_READ, request.streamId()); + + final Message message = new Message(request.message()); + final Stream stream = streamService.load(request.streamId()); + message.addStream(stream); + + if (!Strings.isNullOrEmpty(request.inputId())) { + message.setSourceInputId(request.inputId()); + } + + final List simulationResults = new ArrayList<>(); + final PipelineInterpreterTracer pipelineInterpreterTracer = new PipelineInterpreterTracer(); + + org.graylog2.plugin.Messages processedMessages = pipelineInterpreter.process(message, + pipelineInterpreterTracer.getSimulatorInterpreterListener(), + pipelineStateUpdater.getLatestState()); + for (Message processedMessage : processedMessages) { + simulationResults.add(ResultMessageSummary.create(null, processedMessage.getFields(), "")); + } + + return SimulationResponse.create(simulationResults, + pipelineInterpreterTracer.getExecutionTrace(), + pipelineInterpreterTracer.took()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java new file mode 100644 index 000000000000..d66e35408b99 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/rest/StageSource.java @@ -0,0 +1,67 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +import javax.validation.constraints.Min; +import java.util.List; + +@AutoValue +@JsonAutoDetect +public abstract class StageSource { + @Min(0) + @JsonProperty("stage") + public abstract int stage(); + + @JsonProperty("match_all") + public abstract boolean matchAll(); + + @JsonProperty("rules") + public abstract List rules(); + + @JsonCreator + public static StageSource create(@JsonProperty("stage") @Min(0) int stage, + @JsonProperty("match_all") boolean matchAll, + @JsonProperty("rules") List rules) { + return builder() + .stage(stage) + .matchAll(matchAll) + .rules(rules) + .build(); + } + + public static Builder builder() { + return new AutoValue_StageSource.Builder(); + } + + public abstract Builder toBuilder(); + + @AutoValue.Builder + public abstract static class Builder { + public abstract StageSource build(); + + public abstract Builder stage(int stageNumber); + + public abstract Builder matchAll(boolean mustMatchAll); + + public abstract Builder rules(List ruleRefs); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java new file mode 100644 index 000000000000..6aa87af6e743 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java @@ -0,0 +1,38 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.simulator; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; + +@AutoValue +@JsonAutoDetect +public abstract class PipelineInterpreterTrace { + @JsonProperty + public abstract long time(); + + @JsonProperty + public abstract String message(); + + @JsonCreator + public static PipelineInterpreterTrace create (@JsonProperty("time") long time, + @JsonProperty("message") String message) { + return new AutoValue_PipelineInterpreterTrace(time, message); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java new file mode 100644 index 000000000000..630610cfb0e5 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTracer.java @@ -0,0 +1,61 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.simulator; + +import com.google.common.base.Stopwatch; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class PipelineInterpreterTracer { + private final List executionTrace; + private final Stopwatch timer; + private final SimulatorInterpreterListener simulatorInterpreterListener; + + public PipelineInterpreterTracer() { + executionTrace = new ArrayList<>(); + timer = Stopwatch.createUnstarted(); + simulatorInterpreterListener = new SimulatorInterpreterListener(this); + } + + public SimulatorInterpreterListener getSimulatorInterpreterListener() { + return simulatorInterpreterListener; + } + + public List getExecutionTrace() { + return executionTrace; + } + + public long took() { + return timer.elapsed(TimeUnit.MICROSECONDS); + } + + public void addTrace(String message) { + executionTrace.add(PipelineInterpreterTrace.create(timer.elapsed(TimeUnit.MICROSECONDS), message)); + } + + public void startProcessing(String message) { + timer.start(); + addTrace(message); + } + + public void finishProcessing(String message) { + timer.stop(); + addTrace(message); + } +} diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java new file mode 100644 index 000000000000..3ca4794d7557 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/SimulatorInterpreterListener.java @@ -0,0 +1,98 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.simulator; + +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.processors.listeners.InterpreterListener; +import org.graylog2.plugin.Message; + +import java.util.Set; + +class SimulatorInterpreterListener implements InterpreterListener { + private final PipelineInterpreterTracer executionTrace; + + SimulatorInterpreterListener(PipelineInterpreterTracer executionTrace) { + this.executionTrace = executionTrace; + } + + @Override + public void startProcessing() { + executionTrace.startProcessing("Starting message processing"); + } + + @Override + public void finishProcessing() { + executionTrace.finishProcessing("Finished message processing"); + } + + @Override + public void processStreams(Message message, Set pipelines, Set streams) { + executionTrace.addTrace("Message " + message.getId() + " running " + pipelines + " for streams " + streams); + } + + @Override + public void enterStage(Stage stage) { + executionTrace.addTrace("Enter " + stage); + } + + @Override + public void exitStage(Stage stage) { + executionTrace.addTrace("Exit " + stage); + } + + @Override + public void evaluateRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Evaluate " + rule + " in " + pipeline); + } + + @Override + public void failEvaluateRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Failed evaluation " + rule + " in " + pipeline); + } + + @Override + public void satisfyRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Evaluation satisfied " + rule + " in " + pipeline); + } + + @Override + public void dissatisfyRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Evaluation not satisfied " + rule + " in " + pipeline); + } + + @Override + public void executeRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Execute " + rule + " in " + pipeline); + } + + @Override + public void failExecuteRule(Rule rule, Pipeline pipeline) { + executionTrace.addTrace("Failed execution " + rule + " in " + pipeline); + } + + @Override + public void continuePipelineExecution(Pipeline pipeline, Stage stage) { + executionTrace.addTrace("Completed " + stage + " for " + pipeline + ", continuing to next stage"); + } + + @Override + public void stopPipelineExecution(Pipeline pipeline, Stage stage) { + executionTrace.addTrace("Completed " + stage + " for " + pipeline + ", NOT continuing to next stage"); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/commands/Server.java b/graylog2-server/src/main/java/org/graylog2/commands/Server.java index 4773c7f0092b..4ebe85974f55 100644 --- a/graylog2-server/src/main/java/org/graylog2/commands/Server.java +++ b/graylog2-server/src/main/java/org/graylog2/commands/Server.java @@ -24,6 +24,7 @@ import com.google.inject.Module; import com.google.inject.spi.Message; import com.mongodb.MongoException; +import org.graylog.plugins.pipelineprocessor.PipelineConfig; import org.graylog2.Configuration; import org.graylog2.alerts.AlertConditionBindings; import org.graylog2.audit.AuditActor; @@ -96,6 +97,7 @@ public class Server extends ServerBootstrap { private final VersionCheckConfiguration versionCheckConfiguration = new VersionCheckConfiguration(); private final KafkaJournalConfiguration kafkaJournalConfiguration = new KafkaJournalConfiguration(); private final NettyTransportConfiguration nettyTransportConfiguration = new NettyTransportConfiguration(); + private final PipelineConfig pipelineConfiguration = new PipelineConfig(); public Server() { super("server", configuration); @@ -150,7 +152,8 @@ protected List getCommandConfigurationBeans() { mongoDbConfiguration, versionCheckConfiguration, kafkaJournalConfiguration, - nettyTransportConfiguration); + nettyTransportConfiguration, + pipelineConfiguration); } @Override diff --git a/graylog2-server/src/main/java/org/graylog2/messageprocessors/MessageProcessorModule.java b/graylog2-server/src/main/java/org/graylog2/messageprocessors/MessageProcessorModule.java index 939e01143367..534cd07b7333 100644 --- a/graylog2-server/src/main/java/org/graylog2/messageprocessors/MessageProcessorModule.java +++ b/graylog2-server/src/main/java/org/graylog2/messageprocessors/MessageProcessorModule.java @@ -17,6 +17,8 @@ package org.graylog2.messageprocessors; import com.google.inject.Scopes; +import org.graylog.plugins.pipelineprocessor.PipelineProcessorModule; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbServicesModule; import org.graylog2.plugin.PluginModule; public class MessageProcessorModule extends PluginModule { @@ -25,6 +27,8 @@ protected void configure() { addMessageProcessor(MessageFilterChainProcessor.class, MessageFilterChainProcessor.Descriptor.class); // must not be a singleton, because each thread should get an isolated copy of the processors bind(OrderedMessageProcessors.class).in(Scopes.NO_SCOPE); - } + install(new PipelineProcessorModule()); + install(new MongoDbServicesModule()); + } } diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java new file mode 100644 index 000000000000..3f87806db81e --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/BaseParserTest.java @@ -0,0 +1,162 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor; + +import com.google.common.collect.Maps; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.statements.Statement; +import org.graylog.plugins.pipelineprocessor.codegen.CodeGenerator; +import org.graylog.plugins.pipelineprocessor.codegen.GeneratedRule; +import org.graylog.plugins.pipelineprocessor.codegen.compiler.JavaCompiler; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.streams.Stream; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.rules.TestName; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +import static com.google.common.collect.ImmutableList.of; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BaseParserTest { + protected static final AtomicBoolean actionsTriggered = new AtomicBoolean(false); + protected static FunctionRegistry functionRegistry; + protected static Stream defaultStream; + + @org.junit.Rule + public TestName name = new TestName(); + protected PipelineRuleParser parser; + + protected static HashMap> commonFunctions() { + final HashMap> functions = Maps.newHashMap(); + functions.put("trigger_test", new AbstractFunction() { + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + actionsTriggered.set(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("trigger_test") + .returnType(Void.class) + .params(of()) + .build(); + } + }); + return functions; + } + + @BeforeClass + public static void init() { + defaultStream = mock(Stream.class, "Default stream"); + when(defaultStream.isPaused()).thenReturn(false); + when(defaultStream.getTitle()).thenReturn("default stream"); + when(defaultStream.getId()).thenReturn(Stream.DEFAULT_STREAM_ID); + } + + @Before + public void setup() { + parser = new PipelineRuleParser(functionRegistry, new CodeGenerator(JavaCompiler::new)); + // initialize before every test! + actionsTriggered.set(false); + } + + protected EvaluationContext contextForRuleEval(Rule rule, Message message) { + final EvaluationContext context = new EvaluationContext(message); + final GeneratedRule generatedRule = rule.generatedRule(); + if (generatedRule != null) { + if (generatedRule.when(context)) { + generatedRule.then(context); + } + } else { + if (rule.when().evaluateBool(context)) { + for (Statement statement : rule.then()) { + statement.evaluate(context); + } + } + } + return context; + } + + protected Message evaluateRule(Rule rule, Message message) { + final EvaluationContext context = new EvaluationContext(message); + final GeneratedRule generatedRule = rule.generatedRule(); + if (generatedRule != null) { + if (generatedRule.when(context)) { + generatedRule.then(context); + return context.currentMessage(); + } else { + return null; + } + } + if (rule.when().evaluateBool(context)) { + + for (Statement statement : rule.then()) { + statement.evaluate(context); + } + return context.currentMessage(); + } else { + return null; + } + } + + @Nullable + protected Message evaluateRule(Rule rule) { + return evaluateRule(rule, (msg) -> {}); + } + + @Nullable + protected Message evaluateRule(Rule rule, Consumer messageModifier) { + final Message message = new Message("hello test", "source", DateTime.now(DateTimeZone.UTC)); + message.addStream(defaultStream); + messageModifier.accept(message); + return evaluateRule(rule, message); + } + + protected String ruleForTest() { + try { + final URL resource = this.getClass().getResource(name.getMethodName().concat(".txt")); + final Path path = Paths.get(resource.toURI()); + final byte[] bytes = Files.readAllBytes(path); + return new String(bytes, StandardCharsets.UTF_8); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java new file mode 100644 index 000000000000..74e341fa6031 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/ast/expressions/IndexedAccessExpressionTest.java @@ -0,0 +1,127 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.ast.expressions; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Token; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog2.plugin.Message; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +public class IndexedAccessExpressionTest { + + public static final Token START = new CommonToken(-1); + private EvaluationContext context; + + @Before + public void setup() { + context = new EvaluationContext(new Message("test message", "test", DateTime.parse("2010-07-30T16:03:25Z"))); + } + + @Test + public void accessArray() { + int ary[] = new int[] {23}; + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(ary), num(0)); + + final Object evaluate = idxExpr.evaluateUnsafe(context); + assertThat(evaluate).isOfAnyClassIn(Integer.class); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void accessList() { + final ImmutableList list = ImmutableList.of(23); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(list), num(0)); + + final Object evaluate = idxExpr.evaluateUnsafe(context); + assertThat(evaluate).isOfAnyClassIn(Integer.class); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void accessIterable() { + final Iterable iterable = () -> new AbstractIterator() { + private boolean done = false; + + @Override + protected Integer computeNext() { + if (done) { + return endOfData(); + } + done = true; + return 23; + } + }; + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(iterable), num(0)); + + final Object evaluate = idxExpr.evaluateUnsafe(context); + assertThat(evaluate).isOfAnyClassIn(Integer.class); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void accessMap() { + final ImmutableMap map = ImmutableMap.of("string", 23); + final IndexedAccessExpression idxExpr = new IndexedAccessExpression(START, obj(map), string("string")); + + final Object evaluate = idxExpr.evaluateUnsafe(context); + assertThat(evaluate).isEqualTo(23); + } + + @Test + public void invalidObject() { + final IndexedAccessExpression expression = new IndexedAccessExpression(START, obj(23), num(0)); + + // this should throw an exception + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> expression.evaluateUnsafe(context)); + } + + private static Expression num(long idx) { + return new LongExpression(START, idx); + } + + private static Expression string(String string) { + return new StringExpression(START, string); + } + + private static ConstantObjectExpression obj(Object object) { + return new ConstantObjectExpression(object); + } + + private static class ConstantObjectExpression extends ConstantExpression { + private final Object object; + + protected ConstantObjectExpression(Object object) { + super(START, object.getClass()); + this.object = object; + } + + @Override + public Object evaluateUnsafe(EvaluationContext context) { + return object; + } + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java new file mode 100644 index 000000000000..072e622a98e8 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java @@ -0,0 +1,120 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.db.memory; + +import com.google.common.collect.ImmutableList; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog2.database.NotFoundException; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class InMemoryRuleServiceTest { + private InMemoryRuleService service; + + @Before + public void setup() { + service = new InMemoryRuleService(); + } + + @Test + public void notFoundException() { + + try { + service.load("1"); + fail("Should throw an exception"); + } catch (NotFoundException e) { + assertThat(e).hasMessage("No such rule with id 1"); + } + } + + @Test + public void storeRetrieve() { + RuleDao rule = RuleDao.create(null, "test", "description", "rule \"test\" when true then end", null, null); + final RuleDao savedRule = service.save(rule); + + // saving should create a copy with an id + assertThat(savedRule).isNotEqualTo(rule); + assertThat(savedRule.id()).isNotNull(); + + RuleDao loaded; + try { + loaded = service.load(savedRule.id()); + } catch (NotFoundException e) { + fail("The rule should be found"); + loaded = null; + } + assertThat(loaded).isNotNull(); + assertThat(loaded).isEqualTo(savedRule); + + service.delete(loaded.id()); + try { + service.load(loaded.id()); + fail("Deleted rules should not be found anymore"); + } catch (NotFoundException ignored) { + } + } + + @Test + public void uniqueTitles() { + RuleDao rule = RuleDao.create(null, "test", "description", "rule \"test\" when true then end", null, null); + RuleDao rule2 = RuleDao.create(null, + "test", + "some other description", + "rule \"test\" when false then end", + null, + null); + + final RuleDao saved = service.save(rule); + try { + service.save(rule2); + fail("Titles must be unique for two different rules"); + } catch (IllegalArgumentException ignored) { + } + + try { + service.save(saved.toBuilder().createdAt(DateTime.now(DateTimeZone.UTC)).build()); + } catch (IllegalArgumentException e) { + fail("Updating an existing rule should be possible"); + } + + service.delete(saved.id()); + try { + service.save(rule); + } catch (IllegalArgumentException e) { + fail("Removing a rule should clean up the title index."); + } + } + + + @Test + public void loadMultiple() { + + RuleDao rule1 = service.save(RuleDao.create(null, "test1", "description", "rule \"test1\" when true then end", null, null)); + RuleDao rule2 = service.save(RuleDao.create(null, "test2", "description", "rule \"test2\" when true then end", null, null)); + RuleDao rule3 = service.save(RuleDao.create(null, "test3", "description", "rule \"test3\" when true then end", null, null)); + RuleDao rule4 = service.save(RuleDao.create(null, "test4", "description", "rule \"test4\" when true then end", null, null)); + + assertThat(service.loadAll()).containsExactlyInAnyOrder(rule1, rule2, rule3, rule4); + + assertThat(service.loadNamed(ImmutableList.of("test3", "test2"))).containsExactlyInAnyOrder(rule2, rule3); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java new file mode 100644 index 000000000000..2a97f42ae8f5 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java @@ -0,0 +1,949 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.common.net.InetAddresses; +import org.graylog.plugins.pipelineprocessor.BaseParserTest; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.functions.conversion.BooleanConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.DoubleConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsBoolean; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsCollection; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsDouble; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsList; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsLong; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsMap; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsNumber; +import org.graylog.plugins.pipelineprocessor.functions.conversion.IsString; +import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.MapConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.DateConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.FlexParseDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.FormatDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.IsDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.Now; +import org.graylog.plugins.pipelineprocessor.functions.dates.ParseDate; +import org.graylog.plugins.pipelineprocessor.functions.dates.ParseUnixMilliseconds; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Days; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Hours; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.IsPeriod; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Millis; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Minutes; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Months; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.PeriodParseFunction; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Seconds; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Weeks; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Years; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base16Decode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base16Encode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32Decode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32Encode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32HumanDecode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base32HumanEncode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64Decode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64Encode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64UrlDecode; +import org.graylog.plugins.pipelineprocessor.functions.encoding.Base64UrlEncode; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32; +import org.graylog.plugins.pipelineprocessor.functions.hashing.CRC32C; +import org.graylog.plugins.pipelineprocessor.functions.hashing.MD5; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_128; +import org.graylog.plugins.pipelineprocessor.functions.hashing.Murmur3_32; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA1; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA256; +import org.graylog.plugins.pipelineprocessor.functions.hashing.SHA512; +import org.graylog.plugins.pipelineprocessor.functions.ips.CidrMatch; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddress; +import org.graylog.plugins.pipelineprocessor.functions.ips.IpAddressConversion; +import org.graylog.plugins.pipelineprocessor.functions.ips.IsIp; +import org.graylog.plugins.pipelineprocessor.functions.json.IsJson; +import org.graylog.plugins.pipelineprocessor.functions.json.JsonParse; +import org.graylog.plugins.pipelineprocessor.functions.json.SelectJsonPath; +import org.graylog.plugins.pipelineprocessor.functions.messages.CloneMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RemoveFromStream; +import org.graylog.plugins.pipelineprocessor.functions.messages.RenameField; +import org.graylog.plugins.pipelineprocessor.functions.messages.RouteToStream; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetFields; +import org.graylog.plugins.pipelineprocessor.functions.messages.StreamCacheService; +import org.graylog.plugins.pipelineprocessor.functions.strings.Abbreviate; +import org.graylog.plugins.pipelineprocessor.functions.strings.Capitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Concat; +import org.graylog.plugins.pipelineprocessor.functions.strings.Contains; +import org.graylog.plugins.pipelineprocessor.functions.strings.EndsWith; +import org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.KeyValue; +import org.graylog.plugins.pipelineprocessor.functions.strings.Lowercase; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; +import org.graylog.plugins.pipelineprocessor.functions.strings.Split; +import org.graylog.plugins.pipelineprocessor.functions.strings.StartsWith; +import org.graylog.plugins.pipelineprocessor.functions.strings.Substring; +import org.graylog.plugins.pipelineprocessor.functions.strings.Swapcase; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uncapitalize; +import org.graylog.plugins.pipelineprocessor.functions.strings.Uppercase; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogFacilityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogLevelConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityConversion; +import org.graylog.plugins.pipelineprocessor.functions.syslog.SyslogPriorityToStringConversion; +import org.graylog.plugins.pipelineprocessor.functions.urls.IsUrl; +import org.graylog.plugins.pipelineprocessor.functions.urls.UrlConversion; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.ParseException; +import org.graylog2.database.NotFoundException; +import org.graylog2.grok.GrokPattern; +import org.graylog2.grok.GrokPatternRegistry; +import org.graylog2.grok.GrokPatternService; +import org.graylog2.plugin.InstantMillisProvider; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Tools; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.shared.SuppressForbidden; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.graylog2.streams.StreamService; +import org.joda.time.DateTime; +import org.joda.time.DateTimeUtils; +import org.joda.time.Duration; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentMatchers; + +import javax.inject.Provider; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FunctionsSnippetsTest extends BaseParserTest { + + public static final DateTime GRAYLOG_EPOCH = DateTime.parse("2010-07-30T16:03:25Z"); + private static final EventBus eventBus = new EventBus(); + private static StreamCacheService streamCacheService; + private static Stream otherStream; + + @BeforeClass + @SuppressForbidden("Allow using default thread factory") + public static void registerFunctions() { + final Map> functions = commonFunctions(); + + functions.put(BooleanConversion.NAME, new BooleanConversion()); + functions.put(DoubleConversion.NAME, new DoubleConversion()); + functions.put(LongConversion.NAME, new LongConversion()); + functions.put(StringConversion.NAME, new StringConversion()); + functions.put(MapConversion.NAME, new MapConversion()); + + // message related functions + functions.put(HasField.NAME, new HasField()); + functions.put(SetField.NAME, new SetField()); + functions.put(SetFields.NAME, new SetFields()); + functions.put(RenameField.NAME, new RenameField()); + functions.put(RemoveField.NAME, new RemoveField()); + + functions.put(DropMessage.NAME, new DropMessage()); + functions.put(CreateMessage.NAME, new CreateMessage()); + functions.put(CloneMessage.NAME, new CloneMessage()); + + // route to stream mocks + final StreamService streamService = mock(StreamService.class); + + otherStream = mock(Stream.class, "some stream id2"); + when(otherStream.isPaused()).thenReturn(false); + when(otherStream.getTitle()).thenReturn("some name"); + when(otherStream.getId()).thenReturn("id2"); + + when(streamService.loadAll()).thenReturn(Lists.newArrayList(defaultStream, otherStream)); + when(streamService.loadAllEnabled()).thenReturn(Lists.newArrayList(defaultStream, otherStream)); + try { + when(streamService.load(anyString())).thenThrow(new NotFoundException()); + when(streamService.load(ArgumentMatchers.eq(Stream.DEFAULT_STREAM_ID))).thenReturn(defaultStream); + when(streamService.load(ArgumentMatchers.eq("id2"))).thenReturn(otherStream); + } catch (NotFoundException ignored) { + // oh well, checked exceptions <3 + } + streamCacheService = new StreamCacheService(eventBus, streamService, null); + streamCacheService.startAsync().awaitRunning(); + final Provider defaultStreamProvider = () -> defaultStream; + functions.put(RouteToStream.NAME, new RouteToStream(streamCacheService, defaultStreamProvider)); + functions.put(RemoveFromStream.NAME, new RemoveFromStream(streamCacheService, defaultStreamProvider)); + // input related functions + // TODO needs mock + //functions.put(FromInput.NAME, new FromInput()); + + // generic functions + functions.put(RegexMatch.NAME, new RegexMatch()); + + // string functions + functions.put(Abbreviate.NAME, new Abbreviate()); + functions.put(Capitalize.NAME, new Capitalize()); + functions.put(Concat.NAME, new Concat()); + functions.put(Contains.NAME, new Contains()); + functions.put(EndsWith.NAME, new EndsWith()); + functions.put(Lowercase.NAME, new Lowercase()); + functions.put(Substring.NAME, new Substring()); + functions.put(Swapcase.NAME, new Swapcase()); + functions.put(Uncapitalize.NAME, new Uncapitalize()); + functions.put(Uppercase.NAME, new Uppercase()); + functions.put(KeyValue.NAME, new KeyValue()); + functions.put(Split.NAME, new Split()); + functions.put(StartsWith.NAME, new StartsWith()); + + final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + functions.put(JsonParse.NAME, new JsonParse(objectMapper)); + functions.put(SelectJsonPath.NAME, new SelectJsonPath(objectMapper)); + + functions.put(DateConversion.NAME, new DateConversion()); + functions.put(Now.NAME, new Now()); + functions.put(FlexParseDate.NAME, new FlexParseDate()); + functions.put(ParseDate.NAME, new ParseDate()); + functions.put(ParseUnixMilliseconds.NAME, new ParseUnixMilliseconds()); + functions.put(FormatDate.NAME, new FormatDate()); + + functions.put(Years.NAME, new Years()); + functions.put(Months.NAME, new Months()); + functions.put(Weeks.NAME, new Weeks()); + functions.put(Days.NAME, new Days()); + functions.put(Hours.NAME, new Hours()); + functions.put(Minutes.NAME, new Minutes()); + functions.put(Seconds.NAME, new Seconds()); + functions.put(Millis.NAME, new Millis()); + functions.put(PeriodParseFunction.NAME, new PeriodParseFunction()); + + functions.put(CRC32.NAME, new CRC32()); + functions.put(CRC32C.NAME, new CRC32C()); + functions.put(MD5.NAME, new MD5()); + functions.put(Murmur3_32.NAME, new Murmur3_32()); + functions.put(Murmur3_128.NAME, new Murmur3_128()); + functions.put(SHA1.NAME, new SHA1()); + functions.put(SHA256.NAME, new SHA256()); + functions.put(SHA512.NAME, new SHA512()); + + functions.put(Base16Encode.NAME, new Base16Encode()); + functions.put(Base16Decode.NAME, new Base16Decode()); + functions.put(Base32Encode.NAME, new Base32Encode()); + functions.put(Base32Decode.NAME, new Base32Decode()); + functions.put(Base32HumanEncode.NAME, new Base32HumanEncode()); + functions.put(Base32HumanDecode.NAME, new Base32HumanDecode()); + functions.put(Base64Encode.NAME, new Base64Encode()); + functions.put(Base64Decode.NAME, new Base64Decode()); + functions.put(Base64UrlEncode.NAME, new Base64UrlEncode()); + functions.put(Base64UrlDecode.NAME, new Base64UrlDecode()); + + functions.put(IpAddressConversion.NAME, new IpAddressConversion()); + functions.put(CidrMatch.NAME, new CidrMatch()); + + functions.put(IsNull.NAME, new IsNull()); + functions.put(IsNotNull.NAME, new IsNotNull()); + + functions.put(SyslogPriorityConversion.NAME, new SyslogPriorityConversion()); + functions.put(SyslogPriorityToStringConversion.NAME, new SyslogPriorityToStringConversion()); + functions.put(SyslogFacilityConversion.NAME, new SyslogFacilityConversion()); + functions.put(SyslogLevelConversion.NAME, new SyslogLevelConversion()); + + functions.put(UrlConversion.NAME, new UrlConversion()); + + functions.put(IsBoolean.NAME, new IsBoolean()); + functions.put(IsNumber.NAME, new IsNumber()); + functions.put(IsDouble.NAME, new IsDouble()); + functions.put(IsLong.NAME, new IsLong()); + functions.put(IsString.NAME, new IsString()); + functions.put(IsCollection.NAME, new IsCollection()); + functions.put(IsList.NAME, new IsList()); + functions.put(IsMap.NAME, new IsMap()); + functions.put(IsDate.NAME, new IsDate()); + functions.put(IsPeriod.NAME, new IsPeriod()); + functions.put(IsIp.NAME, new IsIp()); + functions.put(IsJson.NAME, new IsJson()); + functions.put(IsUrl.NAME, new IsUrl()); + + final GrokPatternService grokPatternService = mock(GrokPatternService.class); + Set patterns = Sets.newHashSet( + GrokPattern.create("GREEDY", ".*"), + GrokPattern.create("BASE10NUM", "(?[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"), + GrokPattern.create("NUMBER", "(?:%{BASE10NUM:UNWANTED})"), + GrokPattern.create("NUM", "%{BASE10NUM}") + ); + when(grokPatternService.loadAll()).thenReturn(patterns); + final EventBus clusterBus = new EventBus(); + final GrokPatternRegistry grokPatternRegistry = new GrokPatternRegistry(clusterBus, + grokPatternService, + Executors.newScheduledThreadPool(1)); + functions.put(GrokMatch.NAME, new GrokMatch(grokPatternRegistry)); + + functionRegistry = new FunctionRegistry(functions); + } + + @Test + public void jsonpath() { + final String json = "{\n" + + " \"store\": {\n" + + " \"book\": [\n" + + " {\n" + + " \"category\": \"reference\",\n" + + " \"author\": \"Nigel Rees\",\n" + + " \"title\": \"Sayings of the Century\",\n" + + " \"price\": 8.95\n" + + " },\n" + + " {\n" + + " \"category\": \"fiction\",\n" + + " \"author\": \"Evelyn Waugh\",\n" + + " \"title\": \"Sword of Honour\",\n" + + " \"price\": 12.99\n" + + " },\n" + + " {\n" + + " \"category\": \"fiction\",\n" + + " \"author\": \"Herman Melville\",\n" + + " \"title\": \"Moby Dick\",\n" + + " \"isbn\": \"0-553-21311-3\",\n" + + " \"price\": 8.99\n" + + " },\n" + + " {\n" + + " \"category\": \"fiction\",\n" + + " \"author\": \"J. R. R. Tolkien\",\n" + + " \"title\": \"The Lord of the Rings\",\n" + + " \"isbn\": \"0-395-19395-8\",\n" + + " \"price\": 22.99\n" + + " }\n" + + " ],\n" + + " \"bicycle\": {\n" + + " \"color\": \"red\",\n" + + " \"price\": 19.95\n" + + " }\n" + + " },\n" + + " \"expensive\": 10\n" + + "}"; + + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule, new Message(json, "test", Tools.nowUTC())); + + assertThat(message.hasField("author_first")).isTrue(); + assertThat(message.getField("author_first")).isEqualTo("Nigel Rees"); + assertThat(message.hasField("author_last")).isTrue(); + assertThat(message.hasField("this_should_exist")).isTrue(); + } + + @Test + public void json() { + final String flatJson = "{\"str\":\"foobar\",\"int\":42,\"float\":2.5,\"bool\":true,\"array\":[1,2,3]}"; + final String nestedJson = "{\n" + + " \"store\": {\n" + + " \"book\": {\n" + + " \"category\": \"reference\",\n" + + " \"author\": \"Nigel Rees\",\n" + + " \"title\": \"Sayings of the Century\",\n" + + " \"price\": 8.95\n" + + " },\n" + + " \"bicycle\": {\n" + + " \"color\": \"red\",\n" + + " \"price\": 19.95\n" + + " }\n" + + " },\n" + + " \"expensive\": 10\n" + + "}"; + + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = new Message("JSON", "test", Tools.nowUTC()); + message.addField("flat_json", flatJson); + message.addField("nested_json", nestedJson); + final Message evaluatedMessage = evaluateRule(rule, message); + + assertThat(evaluatedMessage.getField("message")).isEqualTo("JSON"); + assertThat(evaluatedMessage.getField("flat_json")).isEqualTo(flatJson); + assertThat(evaluatedMessage.getField("nested_json")).isEqualTo(nestedJson); + assertThat(evaluatedMessage.getField("str")).isEqualTo("foobar"); + assertThat(evaluatedMessage.getField("int")).isEqualTo(42); + assertThat(evaluatedMessage.getField("float")).isEqualTo(2.5); + assertThat(evaluatedMessage.getField("bool")).isEqualTo(true); + assertThat(evaluatedMessage.getField("array")).isEqualTo(Arrays.asList(1, 2, 3)); + assertThat(evaluatedMessage.getField("store")).isInstanceOf(Map.class); + assertThat(evaluatedMessage.getField("expensive")).isEqualTo(10); + } + + @Test + public void substring() { + final Rule rule = parser.parseRule(ruleForTest(), false); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } + + @Test + public void dates() { + final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH); + DateTimeUtils.setCurrentMillisProvider(clock); + + try { + final Rule rule; + try { + rule = parser.parseRule(ruleForTest(), false); + } catch (ParseException e) { + fail("Should not fail to parse", e); + return; + } + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message).isNotEmpty(); + assertThat(message.hasField("year")).isTrue(); + assertThat(message.getField("year")).isEqualTo(2010); + assertThat(message.getField("timezone")).isEqualTo("UTC"); + + // Date parsing locales + assertThat(message.getField("german_year")).isEqualTo(1983); + assertThat(message.getField("german_month")).isEqualTo(7); + assertThat(message.getField("german_day")).isEqualTo(24); + assertThat(message.getField("english_year")).isEqualTo(1983); + assertThat(message.getField("english_month")).isEqualTo(7); + assertThat(message.getField("english_day")).isEqualTo(24); + assertThat(message.getField("french_year")).isEqualTo(1983); + assertThat(message.getField("french_month")).isEqualTo(7); + assertThat(message.getField("french_day")).isEqualTo(24); + + assertThat(message.getField("ts_hour")).isEqualTo(16); + assertThat(message.getField("ts_minute")).isEqualTo(3); + assertThat(message.getField("ts_second")).isEqualTo(25); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + } + + @Test + public void datesUnixTimestamps() { + final Rule rule = parser.parseRule(ruleForTest(), false); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } + + @Test + public void digests() { + final Rule rule = parser.parseRule(ruleForTest(), false); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } + + @Test + public void encodings() { + final Rule rule = parser.parseRule(ruleForTest(), false); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } + + @Test + public void regexMatch() { + try { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + assertNotNull(message); + assertTrue(message.hasField("matched_regex")); + assertTrue(message.hasField("group_1")); + assertThat((String) message.getField("named_group")).isEqualTo("cd.e"); + } catch (ParseException e) { + Assert.fail("Should parse"); + } + } + + @Test + public void strings() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("has_xyz")).isInstanceOf(Boolean.class); + assertThat((boolean) message.getField("has_xyz")).isFalse(); + assertThat(message.getField("string_literal")).isInstanceOf(String.class); + assertThat((String) message.getField("string_literal")).isEqualTo("abcd\\.e\tfg\u03a9\363"); + } + + @Test + public void split() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("limit_0")).isInstanceOf(String[].class); + assertThat((String[]) message.getField("limit_0")) + .isNotEmpty() + .containsExactly("foo", "bar", "baz"); + assertThat(message.getField("limit_1")).isInstanceOf(String[].class); + assertThat((String[]) message.getField("limit_1")) + .isNotEmpty() + .containsExactly("foo:bar:baz"); + assertThat(message.getField("limit_2")).isInstanceOf(String[].class); + assertThat((String[]) message.getField("limit_2")) + .isNotEmpty() + .containsExactly("foo", "bar|baz"); + } + + @Test + public void ipMatching() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message in = new Message("test", "test", Tools.nowUTC()); + in.addField("ip", "192.168.1.20"); + final Message message = evaluateRule(rule, in); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("ip_anon")).isEqualTo("192.168.1.0"); + assertThat(message.getField("ipv6_anon")).isEqualTo("2001:db8::"); + } + + @Test + public void evalErrorSuppressed() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final Message message = new Message("test", "test", Tools.nowUTC()); + message.addField("this_field_was_set", true); + final EvaluationContext context = contextForRuleEval(rule, message); + + assertThat(context).isNotNull(); + assertThat(context.hasEvaluationErrors()).isFalse(); + assertThat(actionsTriggered.get()).isTrue(); + } + + @Test + public void newlyCreatedMessage() { + final Message message = new Message("test", "test", Tools.nowUTC()); + message.addField("foo", "bar"); + message.addStream(mock(Stream.class)); + final Rule rule = parser.parseRule(ruleForTest(), false); + final EvaluationContext context = contextForRuleEval(rule, message); + + final Message origMessage = context.currentMessage(); + final Message newMessage = Iterables.getOnlyElement(context.createdMessages()); + + assertThat(origMessage).isNotSameAs(newMessage); + assertThat(newMessage.getMessage()).isEqualTo("new"); + assertThat(newMessage.getSource()).isEqualTo("synthetic"); + assertThat(newMessage.getStreams()).isEmpty(); + assertThat(newMessage.hasField("removed_again")).isFalse(); + assertThat(newMessage.getFieldAs(Boolean.class, "has_source")).isTrue(); + assertThat(newMessage.getFieldAs(String.class, "only_in")).isEqualTo("new message"); + assertThat(newMessage.getFieldAs(String.class, "multi")).isEqualTo("new message"); + assertThat(newMessage.getFieldAs(String.class, "foo")).isNull(); + } + + @Test + public void clonedMessage() { + final Message message = new Message("test", "test", Tools.nowUTC()); + message.addField("foo", "bar"); + message.addStream(mock(Stream.class)); + final Rule rule = parser.parseRule(ruleForTest(), false); + final EvaluationContext context = contextForRuleEval(rule, message); + + final Message origMessage = context.currentMessage(); + final Message clonedMessage = Iterables.get(context.createdMessages(), 0); + final Message otherMessage = Iterables.get(context.createdMessages(), 1); + + assertThat(origMessage).isNotSameAs(clonedMessage); + assertThat(clonedMessage).isNotNull(); + assertThat(clonedMessage.getMessage()).isEqualTo(origMessage.getMessage()); + assertThat(clonedMessage.getSource()).isEqualTo(origMessage.getSource()); + assertThat(clonedMessage.getTimestamp()).isEqualTo(origMessage.getTimestamp()); + assertThat(clonedMessage.getStreams()).isEqualTo(origMessage.getStreams()); + assertThat(clonedMessage.hasField("removed_again")).isFalse(); + assertThat(clonedMessage.getFieldAs(Boolean.class, "has_source")).isTrue(); + assertThat(clonedMessage.getFieldAs(String.class, "only_in")).isEqualTo("new message"); + assertThat(clonedMessage.getFieldAs(String.class, "multi")).isEqualTo("new message"); + assertThat(clonedMessage.getFieldAs(String.class, "foo")).isEqualTo("bar"); + assertThat(otherMessage).isNotNull(); + assertThat(otherMessage.getMessage()).isEqualTo("foo"); + assertThat(otherMessage.getSource()).isEqualTo("source"); + } + + @Test + public void clonedMessageWithInvalidTimestamp() { + final Message message = new Message("test", "test", Tools.nowUTC()); + message.addField("timestamp", "foobar"); + final Rule rule = parser.parseRule(ruleForTest(), false); + final EvaluationContext context = contextForRuleEval(rule, message); + + final Message origMessage = context.currentMessage(); + final Message clonedMessage = Iterables.get(context.createdMessages(), 0); + + assertThat(origMessage).isNotEqualTo(clonedMessage); + assertThat(origMessage.getField("timestamp")).isNotInstanceOf(DateTime.class); + + assertThat(clonedMessage).isNotNull(); + assertThat(clonedMessage.getMessage()).isEqualTo(origMessage.getMessage()); + assertThat(clonedMessage.getSource()).isEqualTo(origMessage.getSource()); + assertThat(clonedMessage.getStreams()).isEqualTo(origMessage.getStreams()); + assertThat(clonedMessage.getTimestamp()).isNotNull(); + assertThat(clonedMessage.getField("gl2_original_timestamp")).isEqualTo(origMessage.getField("timestamp")); + } + + @Test + public void grok() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + + assertThat(message).isNotNull(); + assertThat(message.getFieldCount()).isEqualTo(5); + assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2015-07-31T10:05:36.773Z")); + // named captures only + assertThat(message.hasField("num")).isTrue(); + assertThat(message.hasField("BASE10NUM")).isFalse(); + } + + @Test + public void urls() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("protocol")).isEqualTo("https"); + assertThat(message.getField("user_info")).isEqualTo("admin:s3cr31"); + assertThat(message.getField("host")).isEqualTo("some.host.with.lots.of.subdomains.com"); + assertThat(message.getField("port")).isEqualTo(9999); + assertThat(message.getField("file")).isEqualTo( + "/path1/path2/three?q1=something&with_spaces=hello%20graylog&equal=can=containanotherone"); + assertThat(message.getField("fragment")).isEqualTo("anchorstuff"); + assertThat(message.getField("query")).isEqualTo( + "q1=something&with_spaces=hello%20graylog&equal=can=containanotherone"); + assertThat(message.getField("q1")).isEqualTo("something"); + assertThat(message.getField("with_spaces")).isEqualTo("hello graylog"); + assertThat(message.getField("equal")).isEqualTo("can=containanotherone"); + assertThat(message.getField("authority")).isEqualTo("admin:s3cr31@some.host.with.lots.of.subdomains.com:9999"); + } + + @Test + public void syslog() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + + assertThat(message.getField("level0")).isEqualTo("Emergency"); + assertThat(message.getField("level1")).isEqualTo("Alert"); + assertThat(message.getField("level2")).isEqualTo("Critical"); + assertThat(message.getField("level3")).isEqualTo("Error"); + assertThat(message.getField("level4")).isEqualTo("Warning"); + assertThat(message.getField("level5")).isEqualTo("Notice"); + assertThat(message.getField("level6")).isEqualTo("Informational"); + assertThat(message.getField("level7")).isEqualTo("Debug"); + + assertThat(message.getField("facility0")).isEqualTo("kern"); + assertThat(message.getField("facility1")).isEqualTo("user"); + assertThat(message.getField("facility2")).isEqualTo("mail"); + assertThat(message.getField("facility3")).isEqualTo("daemon"); + assertThat(message.getField("facility4")).isEqualTo("auth"); + assertThat(message.getField("facility5")).isEqualTo("syslog"); + assertThat(message.getField("facility6")).isEqualTo("lpr"); + assertThat(message.getField("facility7")).isEqualTo("news"); + assertThat(message.getField("facility8")).isEqualTo("uucp"); + assertThat(message.getField("facility9")).isEqualTo("clock"); + assertThat(message.getField("facility10")).isEqualTo("authpriv"); + assertThat(message.getField("facility11")).isEqualTo("ftp"); + assertThat(message.getField("facility12")).isEqualTo("ntp"); + assertThat(message.getField("facility13")).isEqualTo("log audit"); + assertThat(message.getField("facility14")).isEqualTo("log alert"); + assertThat(message.getField("facility15")).isEqualTo("cron"); + assertThat(message.getField("facility16")).isEqualTo("local0"); + assertThat(message.getField("facility17")).isEqualTo("local1"); + assertThat(message.getField("facility18")).isEqualTo("local2"); + assertThat(message.getField("facility19")).isEqualTo("local3"); + assertThat(message.getField("facility20")).isEqualTo("local4"); + assertThat(message.getField("facility21")).isEqualTo("local5"); + assertThat(message.getField("facility22")).isEqualTo("local6"); + assertThat(message.getField("facility23")).isEqualTo("local7"); + + assertThat(message.getField("prio1_facility")).isEqualTo(0); + assertThat(message.getField("prio1_level")).isEqualTo(0); + assertThat(message.getField("prio2_facility")).isEqualTo(20); + assertThat(message.getField("prio2_level")).isEqualTo(5); + assertThat(message.getField("prio3_facility")).isEqualTo("kern"); + assertThat(message.getField("prio3_level")).isEqualTo("Emergency"); + assertThat(message.getField("prio4_facility")).isEqualTo("local4"); + assertThat(message.getField("prio4_level")).isEqualTo("Notice"); + } + + @Test + public void ipMatchingIssue28() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final Message in = new Message("some message", "somehost.graylog.org", Tools.nowUTC()); + evaluateRule(rule, in); + + assertThat(actionsTriggered.get()).isFalse(); + } + + @Test + public void fieldRenaming() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final Message in = new Message("some message", "somehost.graylog.org", Tools.nowUTC()); + in.addField("field_a", "fieldAContent"); + in.addField("field_b", "not deleted"); + + final Message message = evaluateRule(rule, in); + + assertThat(message.hasField("field_1")).isFalse(); + assertThat(message.hasField("field_2")).isTrue(); + assertThat(message.hasField("field_b")).isTrue(); + } + + @Test + public void comparisons() { + final Rule rule = parser.parseRule(ruleForTest(), false); + final EvaluationContext context = contextForRuleEval(rule, new Message("", "", Tools.nowUTC())); + assertThat(context.hasEvaluationErrors()).isFalse(); + assertThat(evaluateRule(rule)).isNotNull(); + assertThat(actionsTriggered.get()).isTrue(); + } + + @Test + public void conversions() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final EvaluationContext context = contextForRuleEval(rule, new Message("test", "test", Tools.nowUTC())); + + assertThat(context.evaluationErrors()).isEmpty(); + final Message message = context.currentMessage(); + + assertNotNull(message); + assertThat(message.getField("string_1")).isEqualTo("1"); + assertThat(message.getField("string_2")).isEqualTo("2"); + // special case, Message doesn't allow adding fields with empty string values + assertThat(message.hasField("string_3")).isFalse(); + assertThat(message.getField("string_4")).isEqualTo("default"); + assertThat(message.getField("string_5")).isEqualTo("false"); + assertThat(message.getField("string_6")).isEqualTo("42"); + assertThat(message.getField("string_7")).isEqualTo("23.42"); + + assertThat(message.getField("long_1")).isEqualTo(1L); + assertThat(message.getField("long_2")).isEqualTo(2L); + assertThat(message.getField("long_3")).isEqualTo(0L); + assertThat(message.getField("long_4")).isEqualTo(1L); + assertThat(message.getField("long_5")).isEqualTo(23L); + assertThat(message.getField("long_6")).isEqualTo(23L); + assertThat(message.getField("long_7")).isEqualTo(1L); + assertThat(message.getField("long_min1")).isEqualTo(Long.MIN_VALUE); + assertThat(message.getField("long_min2")).isEqualTo(1L); + assertThat(message.getField("long_max1")).isEqualTo(Long.MAX_VALUE); + assertThat(message.getField("long_max2")).isEqualTo(1L); + + assertThat(message.getField("double_1")).isEqualTo(1d); + assertThat(message.getField("double_2")).isEqualTo(2d); + assertThat(message.getField("double_3")).isEqualTo(0d); + assertThat(message.getField("double_4")).isEqualTo(1d); + assertThat(message.getField("double_5")).isEqualTo(23d); + assertThat(message.getField("double_6")).isEqualTo(23d); + assertThat(message.getField("double_7")).isEqualTo(23.42d); + assertThat(message.getField("double_min1")).isEqualTo(Double.MIN_VALUE); + assertThat(message.getField("double_min2")).isEqualTo(0d); + assertThat(message.getField("double_max1")).isEqualTo(Double.MAX_VALUE); + assertThat(message.getField("double_inf1")).isEqualTo(Double.POSITIVE_INFINITY); + assertThat(message.getField("double_inf2")).isEqualTo(Double.NEGATIVE_INFINITY); + assertThat(message.getField("double_inf3")).isEqualTo(Double.POSITIVE_INFINITY); + assertThat(message.getField("double_inf4")).isEqualTo(Double.NEGATIVE_INFINITY); + + assertThat(message.getField("bool_1")).isEqualTo(true); + assertThat(message.getField("bool_2")).isEqualTo(false); + assertThat(message.getField("bool_3")).isEqualTo(false); + assertThat(message.getField("bool_4")).isEqualTo(true); + + // the is wrapped in our own class for safety in rules + assertThat(message.getField("ip_1")).isEqualTo(new IpAddress(InetAddresses.forString("127.0.0.1"))); + assertThat(message.getField("ip_2")).isEqualTo(new IpAddress(InetAddresses.forString("127.0.0.1"))); + assertThat(message.getField("ip_3")).isEqualTo(new IpAddress(InetAddresses.forString("0.0.0.0"))); + assertThat(message.getField("ip_4")).isEqualTo(new IpAddress(InetAddresses.forString("::1"))); + + assertThat(message.getField("map_1")).isEqualTo(Collections.singletonMap("foo", "bar")); + assertThat(message.getField("map_2")).isEqualTo(Collections.emptyMap()); + assertThat(message.getField("map_3")).isEqualTo(Collections.emptyMap()); + assertThat(message.getField("map_4")).isEqualTo(Collections.emptyMap()); + assertThat(message.getField("map_5")).isEqualTo(Collections.emptyMap()); + assertThat(message.getField("map_6")).isEqualTo(Collections.emptyMap()); + } + + @Test + public void fieldPrefixSuffix() { + final Rule rule = parser.parseRule(ruleForTest(), false); + + final Message message = evaluateRule(rule); + + assertThat(message).isNotNull(); + + assertThat(message.getField("field")).isEqualTo("1"); + assertThat(message.getField("prae_field_sueff")).isEqualTo("2"); + assertThat(message.getField("field_sueff")).isEqualTo("3"); + assertThat(message.getField("prae_field")).isEqualTo("4"); + assertThat(message.getField("pre_field1_suff")).isEqualTo("5"); + assertThat(message.getField("pre_field2_suff")).isEqualTo("6"); + assertThat(message.getField("pre_field1")).isEqualTo("7"); + assertThat(message.getField("pre_field2")).isEqualTo("8"); + assertThat(message.getField("field1_suff")).isEqualTo("9"); + assertThat(message.getField("field2_suff")).isEqualTo("10"); + } + + @Test + public void keyValue() { + final Rule rule = parser.parseRule(ruleForTest(), true); + + final EvaluationContext context = contextForRuleEval(rule, new Message("", "", Tools.nowUTC())); + + assertThat(context).isNotNull(); + assertThat(context.evaluationErrors()).isEmpty(); + final Message message = context.currentMessage(); + assertThat(message).isNotNull(); + + + assertThat(message.getField("a")).isEqualTo("1,4"); + assertThat(message.getField("b")).isEqualTo("2"); + assertThat(message.getField("c")).isEqualTo("3"); + assertThat(message.getField("d")).isEqualTo("44"); + assertThat(message.getField("e")).isEqualTo("4"); + assertThat(message.getField("f")).isEqualTo("1"); + assertThat(message.getField("g")).isEqualTo("3"); + assertThat(message.hasField("h")).isFalse(); + + assertThat(message.getField("dup_first")).isEqualTo("1"); + assertThat(message.getField("dup_last")).isEqualTo("2"); + } + + @Test + public void keyValueFailure() { + final Rule rule = parser.parseRule(ruleForTest(), true); + final EvaluationContext context = contextForRuleEval(rule, new Message("", "", Tools.nowUTC())); + + assertThat(context.hasEvaluationErrors()).isTrue(); + } + + @Test + public void timezones() { + final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH); + DateTimeUtils.setCurrentMillisProvider(clock); + try { + final Rule rule = parser.parseRule(ruleForTest(), true); + evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + } + + @Test + public void dateArithmetic() { + final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH); + DateTimeUtils.setCurrentMillisProvider(clock); + try { + final Rule rule = parser.parseRule(ruleForTest(), true); + final Message message = evaluateRule(rule); + + assertThat(actionsTriggered.get()).isTrue(); + assertThat(message).isNotNull(); + assertThat(message.getField("interval")) + .isInstanceOf(Duration.class) + .matches(o -> ((Duration)o).isEqual(Duration.standardDays(1)), "Exactly one day difference"); + assertThat(message.getField("years")).isEqualTo(Period.years(2)); + assertThat(message.getField("months")).isEqualTo(Period.months(2)); + assertThat(message.getField("weeks")).isEqualTo(Period.weeks(2)); + assertThat(message.getField("days")).isEqualTo(Period.days(2)); + assertThat(message.getField("hours")).isEqualTo(Period.hours(2)); + assertThat(message.getField("minutes")).isEqualTo(Period.minutes(2)); + assertThat(message.getField("seconds")).isEqualTo(Period.seconds(2)); + assertThat(message.getField("millis")).isEqualTo(Period.millis(2)); + assertThat(message.getField("period")).isEqualTo(Period.parse("P1YT1M")); + + + assertThat(message.getFieldAs(DateTime.class, "long_time_ago")).matches(date -> date.plus(Period.years(10000)).equals(GRAYLOG_EPOCH)); + + assertThat(message.getTimestamp()).isEqualTo(GRAYLOG_EPOCH.plusHours(1)); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + } + + @Test + public void routeToStream() { + final Rule rule = parser.parseRule(ruleForTest(), true); + final Message message = evaluateRule(rule); + + assertThat(message).isNotNull(); + assertThat(message.getStreams()).isNotEmpty(); + assertThat(message.getStreams().size()).isEqualTo(2); + + streamCacheService.updateStreams(ImmutableSet.of("id")); + + final Message message2 = evaluateRule(rule); + assertThat(message2).isNotNull(); + assertThat(message2.getStreams().size()).isEqualTo(2); + } + + @Test + public void routeToStreamRemoveDefault() { + final Rule rule = parser.parseRule(ruleForTest(), true); + final Message message = evaluateRule(rule); + + assertThat(message).isNotNull(); + assertThat(message.getStreams()).isNotEmpty(); + assertThat(message.getStreams().size()).isEqualTo(1); + + streamCacheService.updateStreams(ImmutableSet.of(Stream.DEFAULT_STREAM_ID)); + + final Message message2 = evaluateRule(rule); + assertThat(message2).isNotNull(); + assertThat(message2.getStreams().size()).isEqualTo(1); + } + + @Test + public void removeFromStream() { + final Rule rule = parser.parseRule(ruleForTest(), true); + final Message message = evaluateRule(rule, msg -> msg.addStream(otherStream)); + + assertThat(message).isNotNull(); + assertThat(message.getStreams()).containsOnly(defaultStream); + } + + @Test + public void removeFromStreamRetainDefault() { + final Rule rule = parser.parseRule(ruleForTest(), true); + final Message message = evaluateRule(rule, msg -> msg.addStream(otherStream)); + + assertThat(message).isNotNull(); + assertThat(message.getStreams()).containsOnly(defaultStream); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheServiceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheServiceTest.java new file mode 100644 index 000000000000..e6f0cef2944f --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheServiceTest.java @@ -0,0 +1,43 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.functions.messages; + +import com.google.common.eventbus.EventBus; + +import org.graylog2.plugin.streams.Stream; +import org.graylog2.shared.SuppressForbidden; +import org.graylog2.streams.StreamService; +import org.junit.Test; + +import java.util.Collection; +import java.util.concurrent.Executors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +public class StreamCacheServiceTest { + @Test + @SuppressForbidden("Allow using default thread factory") + public void getByName() throws Exception { + final StreamCacheService streamCacheService = new StreamCacheService(new EventBus(), mock(StreamService.class), Executors.newSingleThreadScheduledExecutor()); + + // make sure getByName always returns a collection + final Collection streams = streamCacheService.getByName("nonexisting"); + assertThat(streams).isNotNull().isEmpty(); + } + +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/CodegenPipelineRuleParserTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/CodegenPipelineRuleParserTest.java new file mode 100644 index 000000000000..1044016e58ff --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/CodegenPipelineRuleParserTest.java @@ -0,0 +1,29 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; +import org.junit.Ignore; + +@Ignore("code generation disabled") +public class CodegenPipelineRuleParserTest extends PipelineRuleParserTest { + + // runs the same tests as in PipelineRuleParserTest but with dynamic code generation turned on. + public CodegenPipelineRuleParserTest() { + classLoader = new PipelineClassloader(); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java new file mode 100644 index 000000000000..c791c18eb0ec --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PipelineRuleParserTest.java @@ -0,0 +1,715 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import org.graylog.plugins.pipelineprocessor.BaseParserTest; +import org.graylog.plugins.pipelineprocessor.EvaluationContext; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; +import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; +import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; +import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; +import org.graylog.plugins.pipelineprocessor.functions.conversion.LongConversion; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.dates.Now; +import org.graylog.plugins.pipelineprocessor.functions.dates.TimezoneAwareFunction; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Days; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Hours; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Millis; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Minutes; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Months; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.PeriodParseFunction; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Seconds; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Weeks; +import org.graylog.plugins.pipelineprocessor.functions.dates.periods.Years; +import org.graylog.plugins.pipelineprocessor.functions.messages.HasField; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.functions.strings.RegexMatch; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleArgumentType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleIndexType; +import org.graylog.plugins.pipelineprocessor.parser.errors.IncompatibleTypes; +import org.graylog.plugins.pipelineprocessor.parser.errors.InvalidFunctionArgument; +import org.graylog.plugins.pipelineprocessor.parser.errors.InvalidOperation; +import org.graylog.plugins.pipelineprocessor.parser.errors.NonIndexableType; +import org.graylog.plugins.pipelineprocessor.parser.errors.OptionalParametersMustBeNamed; +import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; +import org.graylog.plugins.pipelineprocessor.parser.errors.SyntaxError; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredFunction; +import org.graylog.plugins.pipelineprocessor.parser.errors.UndeclaredVariable; +import org.graylog2.plugin.InstantMillisProvider; +import org.graylog2.plugin.Message; +import org.joda.time.DateTime; +import org.joda.time.DateTimeUtils; +import org.joda.time.DateTimeZone; +import org.junit.After; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.collect.ImmutableList.of; +import static org.graylog.plugins.pipelineprocessor.functions.FunctionsSnippetsTest.GRAYLOG_EPOCH; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PipelineRuleParserTest extends BaseParserTest { + + protected static PipelineClassloader classLoader; + + @BeforeClass + public static void registerFunctions() { + final Map> functions = commonFunctions(); + functions.put("nein", new NeinFunction()); + functions.put("doch", new DochFunction()); + functions.put("double_valued_func", new DoubleValuedFunction()); + functions.put("one_arg", new OneArgFunction()); + functions.put("concat", new ConcatFunction()); + functions.put("trigger_test", new TriggerTestFunction()); + functions.put("optional", new OptionalFunction()); + functions.put("customObject", new CustomObjectFunction()); + functions.put("beanObject", new BeanObjectFunction()); + functions.put("keys", new KeysFunction()); + functions.put("sort", new SortFunction()); + functions.put(LongConversion.NAME, new LongConversion()); + functions.put(StringConversion.NAME, new StringConversion()); + functions.put(SetField.NAME, new SetField()); + functions.put(HasField.NAME, new HasField()); + functions.put(RegexMatch.NAME, new RegexMatch()); + functions.put("now_in_tz", new NowInTimezoneFunction()); + + functions.put(Now.NAME, new Now()); + functions.put(Years.NAME, new Years()); + functions.put(Months.NAME, new Months()); + functions.put(Weeks.NAME, new Weeks()); + functions.put(Days.NAME, new Days()); + functions.put(Hours.NAME, new Hours()); + functions.put(Minutes.NAME, new Minutes()); + functions.put(Seconds.NAME, new Seconds()); + functions.put(Millis.NAME, new Millis()); + functions.put(PeriodParseFunction.NAME, new PeriodParseFunction()); + + functionRegistry = new FunctionRegistry(functions); + } + + @After + public void tearDown() { + parser = null; + } + + private Rule parseRuleWithOptionalCodegen() { + return parser.parseRule(ruleForTest(), false, classLoader); + } + + @Test + public void basicRule() throws Exception { + final Rule rule = parseRuleWithOptionalCodegen(); + Assert.assertNotNull("rule should be successfully parsed", rule); + } + + @Test + public void undeclaredIdentifier() throws Exception { + try { + parseRuleWithOptionalCodegen(); + fail("should throw error: undeclared variable x"); + } catch (ParseException e) { + assertEquals(2, + e.getErrors().size()); // undeclared var and incompatible type, but we only care about the undeclared one here + assertTrue("Should find error UndeclaredVariable", + e.getErrors().stream().anyMatch(error -> error instanceof UndeclaredVariable)); + } + } + + @Test + public void declaredFunction() throws Exception { + try { + parseRuleWithOptionalCodegen(); + } catch (ParseException e) { + fail("Should not fail to resolve function 'false'"); + } + } + + @Test + public void undeclaredFunction() throws Exception { + try { + parseRuleWithOptionalCodegen(); + fail("should throw error: undeclared function 'unknown'"); + } catch (ParseException e) { + assertTrue("Should find error UndeclaredFunction", + e.getErrors().stream().anyMatch(input -> input instanceof UndeclaredFunction)); + } + } + + @Test + public void singleArgFunction() throws Exception { + try { + final Rule rule = parseRuleWithOptionalCodegen(); + final Message message = evaluateRule(rule); + + assertNotNull(message); + assertTrue("actions should have triggered", actionsTriggered.get()); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + + @Test + public void positionalArguments() throws Exception { + try { + final Rule rule = parseRuleWithOptionalCodegen(); + evaluateRule(rule); + + assertTrue(actionsTriggered.get()); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + + @Test + public void inferVariableType() throws Exception { + try { + final Rule rule = parseRuleWithOptionalCodegen(); + + evaluateRule(rule); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + + @Test + public void invalidArgType() throws Exception { + try { + parseRuleWithOptionalCodegen(); + } catch (ParseException e) { + assertEquals(2, e.getErrors().size()); + assertTrue("Should only find IncompatibleArgumentType errors", + e.getErrors().stream().allMatch(input -> input instanceof IncompatibleArgumentType)); + } + } + + @Test + public void booleanValuedFunctionAsCondition() throws Exception { + try { + final Rule rule = parseRuleWithOptionalCodegen(); + + evaluateRule(rule); + assertTrue("actions should have triggered", actionsTriggered.get()); + } catch (ParseException e) { + fail("Should not fail to parse"); + } + } + + @Test + public void messageRef() throws Exception { + final Rule rule = parseRuleWithOptionalCodegen(); + Message message = new Message("hello test", "source", DateTime.now(DateTimeZone.UTC)); + message.addField("responseCode", 500); + final Message processedMsg = evaluateRule(rule, message); + + assertNotNull(processedMsg); + assertEquals("server_error", processedMsg.getField("response_category")); + } + + @Test + public void messageRefQuotedField() throws Exception { + final Rule rule = parseRuleWithOptionalCodegen(); + Message message = new Message("hello test", "source", DateTime.now(DateTimeZone.UTC)); + message.addField("@specialfieldname", "string"); + evaluateRule(rule, message); + + assertTrue(actionsTriggered.get()); + } + + @Test + public void optionalArguments() throws Exception { + final Rule rule = parseRuleWithOptionalCodegen(); + + Message message = new Message("hello test", "source", DateTime.now(DateTimeZone.UTC)); + evaluateRule(rule, message); + assertTrue(actionsTriggered.get()); + } + + @Test + public void optionalParamsMustBeNamed() throws Exception { + try { + parseRuleWithOptionalCodegen(); + } catch (ParseException e) { + assertEquals(1, e.getErrors().stream().count()); + assertTrue(e.getErrors().stream().allMatch(error -> error instanceof OptionalParametersMustBeNamed)); + } + + } + + @Test + public void mapArrayLiteral() { + final Rule rule = parseRuleWithOptionalCodegen(); + Message message = new Message("hello test", "source", DateTime.now(DateTimeZone.UTC)); + evaluateRule(rule, message); + assertTrue(actionsTriggered.get()); + } + + @Test + public void typedFieldAccess() throws Exception { + try { + final Rule rule = parseRuleWithOptionalCodegen(); + evaluateRule(rule, new Message("hallo", "test", DateTime.now(DateTimeZone.UTC))); + assertTrue("condition should be true", actionsTriggered.get()); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + + @Test + public void nestedFieldAccess() throws Exception { + try { + final Rule rule = parseRuleWithOptionalCodegen(); + evaluateRule(rule, new Message("hello", "world", DateTime.now(DateTimeZone.UTC))); + assertTrue("condition should be true", actionsTriggered.get()); + } catch (ParseException e) { + fail(e.getMessage()); + } + } + + @Test + public void pipelineDeclaration() throws Exception { + final List pipelines = parser.parsePipelines(ruleForTest()); + assertEquals(1, pipelines.size()); + final Pipeline pipeline = Iterables.getOnlyElement(pipelines); + assertEquals("cisco", pipeline.name()); + assertEquals(2, pipeline.stages().size()); + final Stage stage1 = pipeline.stages().first(); + final Stage stage2 = pipeline.stages().last(); + + assertEquals(true, stage1.matchAll()); + assertEquals(1, stage1.stage()); + assertArrayEquals(new Object[]{"check_ip_whitelist", "cisco_device"}, stage1.ruleReferences().toArray()); + + assertEquals(false, stage2.matchAll()); + assertEquals(2, stage2.stage()); + assertArrayEquals(new Object[]{"parse_cisco_time", "extract_src_dest", "normalize_src_dest", "lookup_ips", "resolve_ips"}, + stage2.ruleReferences().toArray()); + } + + @Test + public void indexedAccess() { + final Rule rule = parseRuleWithOptionalCodegen(); + + evaluateRule(rule, new Message("hallo", "test", DateTime.now(DateTimeZone.UTC))); + assertTrue("condition should be true", actionsTriggered.get()); + } + + @Test + public void indexedAccessWrongType() { + try { + parseRuleWithOptionalCodegen(); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(NonIndexableType.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + + @Test + public void indexedAccessWrongIndexType() { + try { + parseRuleWithOptionalCodegen(); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(IncompatibleIndexType.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + + @Test + public void invalidArgumentValue() { + try { + parseRuleWithOptionalCodegen(); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + final ParseError parseError = Iterables.getOnlyElement(e.getErrors()); + assertEquals("Unable to pre-compute value for 1st argument timezone in call to function now_in_tz: The datetime zone id '123' is not recognised", parseError.toString()); + assertEquals(InvalidFunctionArgument.class, parseError.getClass()); + } + } + + @Test + public void arithmetic() { + final Rule rule = parseRuleWithOptionalCodegen(); + evaluateRule(rule); + + assertTrue(actionsTriggered.get()); + } + + @Test + public void mismatchedNumericTypes() { + try { + parseRuleWithOptionalCodegen(); + fail("Should have thrown parse exception"); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(IncompatibleTypes.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + + @Test + public void booleanNot() { + final Rule rule = parseRuleWithOptionalCodegen(); + evaluateRule(rule); + + assertFalse(actionsTriggered.get()); + } + + @Test + public void dateArithmetic() { + final InstantMillisProvider clock = new InstantMillisProvider(GRAYLOG_EPOCH); + DateTimeUtils.setCurrentMillisProvider(clock); + try { + final Rule rule = parseRuleWithOptionalCodegen(); + final Message message = evaluateRule(rule); + assertNotNull(message); + assertTrue(actionsTriggered.get()); + } finally { + DateTimeUtils.setCurrentMillisSystem(); + } + + } + + @Test + public void invalidDateAddition() { + try { + parseRuleWithOptionalCodegen(); + fail("Should have thrown parse exception"); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(InvalidOperation.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + + @Test + public void issue185() { + try { + parseRuleWithOptionalCodegen(); + fail("Should have thrown parse exception"); + } catch (ParseException e) { + assertEquals(1, e.getErrors().size()); + assertEquals(SyntaxError.class, Iterables.getOnlyElement(e.getErrors()).getClass()); + } + } + + + public static class CustomObject { + private final String id; + + public CustomObject(String id) { + this.id = id; + } + + public String getId() { + return id; + } + } + + public static class BeanObject { + private final String id; + private final NestedBeanObject theName; + + public BeanObject(String id, String firstName, String lastName) { + this.id = id; + this.theName = new NestedBeanObject(firstName, lastName); + } + + public String getId() { + return id; + } + + public NestedBeanObject getTheName() { + return theName; + } + + public static class NestedBeanObject { + private final String firstName; + private final String lastName; + + NestedBeanObject(String firstName, String lastName) { + this.firstName = firstName; + this.lastName = lastName; + } + + public String getFirstName() { + return firstName; + } + + public String getLastName() { + return lastName; + } + } + } + + public static class NeinFunction extends AbstractFunction { + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + return false; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("nein") + .returnType(Boolean.class) + .params(of()) + .build(); + } + } + + public static class DochFunction extends AbstractFunction { + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + return true; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("doch") + .returnType(Boolean.class) + .params(of()) + .build(); + } + } + + public static class DoubleValuedFunction extends AbstractFunction { + @Override + public Double evaluate(FunctionArgs args, EvaluationContext context) { + return 0d; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("double_valued_func") + .returnType(Double.class) + .params(of()) + .build(); + } + } + + public static class OneArgFunction extends AbstractFunction { + + private final ParameterDescriptor one = ParameterDescriptor.string("one").build(); + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + return one.optional(args, context).orElse(""); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("one_arg") + .returnType(String.class) + .params(of(one)) + .build(); + } + } + + public static class ConcatFunction extends AbstractFunction { + + private final ParameterDescriptor three = ParameterDescriptor.object("three").build(); + private final ParameterDescriptor two = ParameterDescriptor.object("two").build(); + private final ParameterDescriptor one = ParameterDescriptor.string("one").build(); + + @Override + public String evaluate(FunctionArgs args, EvaluationContext context) { + final Object one = this.one.optional(args, context).orElse(""); + final Object two = this.two.optional(args, context).orElse(""); + final Object three = this.three.optional(args, context).orElse(""); + return one.toString() + two.toString() + three.toString(); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("concat") + .returnType(String.class) + .params(of( + one, + two, + three + )) + .build(); + } + } + + public static class TriggerTestFunction extends AbstractFunction { + @Override + public Void evaluate(FunctionArgs args, EvaluationContext context) { + actionsTriggered.set(true); + return null; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("trigger_test") + .returnType(Void.class) + .params(of()) + .build(); + } + } + + public static class OptionalFunction extends AbstractFunction { + @Override + public Boolean evaluate(FunctionArgs args, EvaluationContext context) { + return true; + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("optional") + .returnType(Boolean.class) + .params(of( + ParameterDescriptor.bool("a").build(), + ParameterDescriptor.string("b").build(), + ParameterDescriptor.floating("c").optional().build(), + ParameterDescriptor.integer("d").build() + )) + .build(); + } + } + + public static class CustomObjectFunction extends AbstractFunction { + + private final ParameterDescriptor aDefault = ParameterDescriptor.string("default").build(); + + @Override + public CustomObject evaluate(FunctionArgs args, EvaluationContext context) { + return new CustomObject(aDefault.optional(args, context).orElse("")); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("customObject") + .returnType(CustomObject.class) + .params(of(aDefault)) + .build(); + } + } + + public static class BeanObjectFunction extends AbstractFunction { + + private final ParameterDescriptor id = ParameterDescriptor.string("id").build(); + private final ParameterDescriptor firstName = ParameterDescriptor.string("firstName").build(); + private final ParameterDescriptor lastName = ParameterDescriptor.string("lastName").build(); + + @Override + public BeanObject evaluate(FunctionArgs args, EvaluationContext context) { + return new BeanObject( + id.optional(args, context).orElse(""), + firstName.optional(args, context).orElse(""), + lastName.optional(args, context).orElse("") + ); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("beanObject") + .returnType(BeanObject.class) + .params(of(id, firstName, lastName)) + .build(); + } + } + + public static class KeysFunction extends AbstractFunction { + + private final ParameterDescriptor map = ParameterDescriptor.type("map", Map.class).build(); + + @Override + public List evaluate(FunctionArgs args, EvaluationContext context) { + final Optional map = this.map.optional(args, context); + return Lists.newArrayList(map.orElse(Collections.emptyMap()).keySet()); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("keys") + .returnType(List.class) + .params(of(map)) + .build(); + } + } + + public static class SortFunction extends AbstractFunction { + + private final ParameterDescriptor collection = ParameterDescriptor.type("collection", + Collection.class).build(); + + @Override + public Collection evaluate(FunctionArgs args, EvaluationContext context) { + final Collection collection = this.collection.optional(args, context).orElse(Collections.emptyList()); + return Ordering.natural().sortedCopy(collection); + } + + @Override + public FunctionDescriptor descriptor() { + return FunctionDescriptor.builder() + .name("sort") + .returnType(Collection.class) + .params(of(collection)) + .build(); + } + } + + public static class NowInTimezoneFunction extends TimezoneAwareFunction { + @Override + protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { + return DateTime.now(timezone); + } + + @Override + protected String description() { + return "Now in the given timezone"; + } + + @Override + protected String getName() { + return "now_in_tz"; + } + + @Override + protected ImmutableList params() { + return ImmutableList.of(); + } + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java new file mode 100644 index 000000000000..53ab1b442650 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/parser/PrecedenceTest.java @@ -0,0 +1,134 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.parser; + +import org.graylog.plugins.pipelineprocessor.BaseParserTest; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.expressions.AndExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.BooleanExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.ComparisonExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.EqualityExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.NotExpression; +import org.graylog.plugins.pipelineprocessor.ast.expressions.OrExpression; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.codegen.CodeGenerator; +import org.graylog.plugins.pipelineprocessor.codegen.compiler.JavaCompiler; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Tools; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PrecedenceTest extends BaseParserTest { + + @BeforeClass + public static void registerFunctions() { + final Map> functions = commonFunctions(); + + functions.put(StringConversion.NAME, new StringConversion()); + functionRegistry = new FunctionRegistry(functions); + } + + @Test + public void orVsEquality() { + final Rule rule = parseRule("rule \"test\" when true == false || true then end"); + final LogicalExpression when = rule.when(); + + assertThat(when).isInstanceOf(OrExpression.class); + OrExpression orEprx = (OrExpression) when; + + assertThat(orEprx.left()).isInstanceOf(EqualityExpression.class); + assertThat(orEprx.right()).isInstanceOf(BooleanExpression.class); + } + + @Test + public void andVsEquality() { + final Rule rule = parseRule("rule \"test\" when true == false && true then end"); + final LogicalExpression when = rule.when(); + + assertThat(when).isInstanceOf(AndExpression.class); + AndExpression andExpr = (AndExpression) when; + + assertThat(andExpr.left()).isInstanceOf(EqualityExpression.class); + assertThat(andExpr.right()).isInstanceOf(BooleanExpression.class); + } + + @Test + public void parenGroup() { + final Rule rule = parseRule("rule \"test\" when true == (false == false) then end"); + final LogicalExpression when = rule.when(); + + assertThat(when).isInstanceOf(EqualityExpression.class); + EqualityExpression topEqual = (EqualityExpression) when; + + assertThat(topEqual.left()).isInstanceOf(BooleanExpression.class); + assertThat(topEqual.right()).isInstanceOf(EqualityExpression.class); + + final BooleanExpression trueExpr = (BooleanExpression) topEqual.left(); + assertThat(trueExpr.evaluateBool(null)).isTrue(); + final EqualityExpression falseFalse = (EqualityExpression) topEqual.right(); + assertThat(falseFalse.evaluateBool(null)).isTrue(); + } + + @Test + public void comparisonVsEqual() { + final Rule rule = parseRule("rule \"test\" when 1 > 2 == false then end"); + final LogicalExpression when = rule.when(); + + assertThat(when).isInstanceOf(EqualityExpression.class); + + EqualityExpression topEqual = (EqualityExpression) when; + assertThat(topEqual.left()).isInstanceOf(ComparisonExpression.class); + assertThat(topEqual.right()).isInstanceOf(BooleanExpression.class); + } + + @Test + public void notVsAndOr() { + final Rule rule = parseRule("rule \"test\" when !true && false then end"); + final LogicalExpression when = rule.when(); + + assertThat(when).isInstanceOf(AndExpression.class); + AndExpression and = (AndExpression) when; + assertThat(and.left()).isInstanceOf(NotExpression.class); + assertThat(and.right()).isInstanceOf(BooleanExpression.class); + } + + @Test(expected = ParseException.class) + public void literalsMustBeQuotedInFieldref() { + final Rule rule = parseRule("rule \"test\" when to_string($message.true) == to_string($message.false) then end"); + } + + @Test + public void quotedLiteralInFieldRef() { + final Rule rule = parseRule("rule \"test\" when to_string($message.`true`) == \"true\" then end"); + final Message message = new Message("hallo", "test", Tools.nowUTC()); + message.addField("true", "true"); + final Message result = evaluateRule(rule, message); + + assertThat(result).isNotNull(); + } + + private static Rule parseRule(String rule) { + final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry, new CodeGenerator(JavaCompiler::new)); + return parser.parseRule(rule, true); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/ConfigurationStateUpdaterTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/ConfigurationStateUpdaterTest.java new file mode 100644 index 000000000000..01649303b314 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/ConfigurationStateUpdaterTest.java @@ -0,0 +1,301 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ + /** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Maps; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.assistedinject.FactoryModuleBuilder; +import com.google.inject.name.Names; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricRegistry; + +import org.assertj.core.api.Assertions; +import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog.plugins.pipelineprocessor.db.memory.InMemoryServicesModule; +import org.graylog.plugins.pipelineprocessor.functions.ProcessorFunctionsModule; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog2.database.NotFoundException; +import org.graylog2.grok.GrokPatternService; +import org.graylog2.grok.InMemoryGrokPatternService; +import org.graylog2.plugin.Tools; +import org.graylog2.plugin.alarms.AlertCondition; +import org.graylog2.plugin.database.Persisted; +import org.graylog2.plugin.database.ValidationException; +import org.graylog2.plugin.database.validators.ValidationResult; +import org.graylog2.plugin.database.validators.Validator; +import org.graylog2.plugin.streams.Output; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.plugin.streams.StreamRule; +import org.graylog2.rest.resources.streams.requests.CreateStreamRequest; +import org.graylog2.shared.SuppressForbidden; +import org.graylog2.shared.bindings.SchedulerBindings; +import org.graylog2.shared.bindings.providers.MetricRegistryProvider; +import org.graylog2.streams.StreamImpl; +import org.graylog2.streams.StreamService; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Ignore("code generation disabled") +public class ConfigurationStateUpdaterTest { + private static final Logger log = LoggerFactory.getLogger(ConfigurationStateUpdaterTest.class); + private PipelineInterpreter.State reload; + + @SuppressForbidden("Allow calling System#gc()") + @Test + public void testClassUnloading() { + + final Injector injector = Guice.createInjector( + new InMemoryServicesModule(), + new ProcessorFunctionsModule(), + new SchedulerBindings(), + binder -> binder.install(new FactoryModuleBuilder().build(PipelineInterpreter.State.Factory.class)), + binder -> binder.bindConstant().annotatedWith(Names.named("generate_native_code")).to(true), + binder -> binder.bindConstant().annotatedWith(Names.named("cached_stageiterators")).to(true), + binder -> binder.bindConstant().annotatedWith(Names.named("processbuffer_processors")).to(1), + binder -> binder.bind(StreamService.class).to(DummyStreamService.class), + binder -> binder.bind(GrokPatternService.class).to(InMemoryGrokPatternService.class), + binder -> binder.bind(FunctionRegistry.class).asEagerSingleton(), + binder -> binder.bind(MetricRegistry.class).toProvider(MetricRegistryProvider.class).asEagerSingleton() + ); + + final MetricRegistry metricRegistry = injector.getInstance(MetricRegistry.class); + final Gauge pipelineLoadedClasses = metricRegistry.register("jvm.cl.loaded-classes", (Gauge) () -> PipelineClassloader.loadedClasses.get()); + + final RuleService ruleService = injector.getInstance(RuleService.class); + ruleService.save(RuleDao.create("00001", "some rule", "awesome rule", "rule \"arrsome\" when true then let x = now(); end", null, null)); + final ConfigurationStateUpdater updater = injector.getInstance(ConfigurationStateUpdater.class); + + //noinspection unchecked + final Gauge unloadedClasses = metricRegistry.getGauges((name, metric) -> name.startsWith("jvm.cl.unloaded")).get("jvm.cl.unloaded"); + long i = 0; + final Long initialLoaded = pipelineLoadedClasses.getValue(); + while (i++ < 100) { + final long initialUnloaded = unloadedClasses.getValue(); + this.reload = null; + reload = updater.reload(); + + if (i % 10 == 0) { + System.gc(); + log.info("\nClassloading metrics:\n====================="); + metricRegistry.getGauges((name, metric) -> name.startsWith("jvm.cl")).forEach((s, gauge) -> { + log.info("{} : {}", s, gauge.getValue()); + }); + Assertions.assertThat(unloadedClasses.getValue()).isGreaterThan(initialUnloaded); + } + } + Assertions.assertThat(pipelineLoadedClasses.getValue()).isGreaterThan(initialLoaded); + } + + + private static class DummyStreamService implements StreamService { + + private final Map store = new MapMaker().makeMap(); + + @Override + public Stream create(Map fields) { + return new StreamImpl(fields); + } + + @Override + public Stream create(CreateStreamRequest cr, String userId) { + Map streamData = Maps.newHashMap(); + streamData.put(StreamImpl.FIELD_TITLE, cr.title()); + streamData.put(StreamImpl.FIELD_DESCRIPTION, cr.description()); + streamData.put(StreamImpl.FIELD_CREATOR_USER_ID, userId); + streamData.put(StreamImpl.FIELD_CREATED_AT, Tools.nowUTC()); + streamData.put(StreamImpl.FIELD_CONTENT_PACK, cr.contentPack()); + streamData.put(StreamImpl.FIELD_MATCHING_TYPE, cr.matchingType().toString()); + + return create(streamData); + } + + @Override + public Stream load(String id) throws NotFoundException { + final Stream stream = store.get(id); + if (stream == null) { + throw new NotFoundException(); + } + return stream; + } + + @Override + public void destroy(Stream stream) throws NotFoundException { + if (store.remove(stream.getId()) == null) { + throw new NotFoundException(); + } + } + + @Override + public List loadAll() { + return ImmutableList.copyOf(store.values()); + } + + @Override + public List loadAllEnabled() { + return store.values().stream().filter(stream -> !stream.getDisabled()).collect(Collectors.toList()); + } + + @Override + public long count() { + return store.size(); + } + + @Override + public void pause(Stream stream) throws ValidationException { + throw new IllegalStateException("no implemented"); + } + + @Override + public void resume(Stream stream) throws ValidationException { + throw new IllegalStateException("no implemented"); + } + + @Override + public List getStreamRules(Stream stream) throws NotFoundException { + throw new IllegalStateException("no implemented"); + } + + @Override + public List loadAllWithConfiguredAlertConditions() { + throw new IllegalStateException("no implemented"); + } + + @Override + public List getAlertConditions(Stream stream) { + throw new IllegalStateException("no implemented"); + } + + @Override + public AlertCondition getAlertCondition(Stream stream, + String conditionId) throws NotFoundException { + throw new IllegalStateException("no implemented"); + } + + @Override + public void addAlertCondition(Stream stream, + AlertCondition condition) throws ValidationException { + throw new IllegalStateException("no implemented"); + } + + @Override + public void updateAlertCondition(Stream stream, + AlertCondition condition) throws ValidationException { + throw new IllegalStateException("no implemented"); + } + + @Override + public void removeAlertCondition(Stream stream, String conditionId) { + throw new IllegalStateException("no implemented"); + } + + @Override + public void addAlertReceiver(Stream stream, String type, String name) { + throw new IllegalStateException("no implemented"); + } + + @Override + public void removeAlertReceiver(Stream stream, String type, String name) { + throw new IllegalStateException("no implemented"); + } + + @Override + public void addOutput(Stream stream, Output output) { + throw new IllegalStateException("no implemented"); + } + + @Override + public void removeOutput(Stream stream, Output output) { + throw new IllegalStateException("no implemented"); + } + + @Override + public void removeOutputFromAllStreams(Output output) { + throw new IllegalStateException("no implemented"); + } + + @Override + public List loadAllWithIndexSet(String indexSetId) { + throw new IllegalStateException("no implemented"); + } + + @Override + public int destroy(T model) { + throw new IllegalStateException("no implemented"); + } + + @Override + public int destroyAll(Class modelClass) { + throw new IllegalStateException("no implemented"); + } + + @Override + public String save(T model) throws ValidationException { + store.put(model.getId(), (Stream) model); + return model.getId(); + } + + @Override + public String saveWithoutValidation(T model) { + throw new IllegalStateException("no implemented"); + } + + @Override + public Map> validate(T model, + Map fields) { + throw new IllegalStateException("no implemented"); + } + + @Override + public Map> validate(T model) { + throw new IllegalStateException("no implemented"); + } + + @Override + public Map> validate(Map validators, + Map fields) { + throw new IllegalStateException("no implemented"); + } + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java new file mode 100644 index 000000000000..130da63c2e55 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/PipelineInterpreterTest.java @@ -0,0 +1,380 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.functions.Function; +import org.graylog.plugins.pipelineprocessor.codegen.CodeGenerator; +import org.graylog.plugins.pipelineprocessor.codegen.compiler.JavaCompiler; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineService; +import org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.memory.InMemoryRuleService; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService; +import org.graylog.plugins.pipelineprocessor.functions.conversion.StringConversion; +import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage; +import org.graylog.plugins.pipelineprocessor.functions.messages.SetField; +import org.graylog.plugins.pipelineprocessor.parser.FunctionRegistry; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.plugin.Message; +import org.graylog2.plugin.Messages; +import org.graylog2.plugin.Tools; +import org.graylog2.plugin.streams.Stream; +import org.graylog2.shared.SuppressForbidden; +import org.graylog2.shared.journal.Journal; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.concurrent.Executors; + +import static com.codahale.metrics.MetricRegistry.name; +import static org.assertj.core.api.Assertions.assertThat; +import static org.graylog2.plugin.streams.Stream.DEFAULT_STREAM_ID; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PipelineInterpreterTest { + private static final RuleDao RULE_TRUE = RuleDao.create("true", "true", "true", + "rule \"true\"\n" + + "when true\n" + + "then\n" + + "end", null, null); + private static final RuleDao RULE_FALSE = RuleDao.create("false", "false", "false", + "rule \"false\"\n" + + "when false\n" + + "then\n" + + "end", null, null); + private static final RuleDao RULE_ADD_FOOBAR = RuleDao.create("add_foobar", "add_foobar", "add_foobar", + "rule \"add_foobar\"\n" + + "when true\n" + + "then\n" + + " set_field(\"foobar\", \"covfefe\");\n" + + "end", null, null); + + @Test + public void testCreateMessage() { + final RuleService ruleService = mock(MongoDbRuleService.class); + when(ruleService.loadAll()).thenReturn(Collections.singleton( + RuleDao.create("abc", + "title", + "description", + "rule \"creates message\"\n" + + "when to_string($message.message) == \"original message\"\n" + + "then\n" + + " create_message(\"derived message\");\n" + + "end", + Tools.nowUTC(), + null) + )); + + final PipelineService pipelineService = mock(MongoDbPipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("p1", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"creates message\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final Map> functions = ImmutableMap.of( + CreateMessage.NAME, new CreateMessage(), + StringConversion.NAME, new StringConversion()); + + final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions); + + Message msg = messageInDefaultStream("original message", "test"); + final Messages processed = interpreter.process(msg); + + final Message[] messages = Iterables.toArray(processed, Message.class); + assertEquals(2, messages.length); + } + + @Test + public void testMatchAllContinuesIfAllRulesMatched() { + final RuleService ruleService = mock(MongoDbRuleService.class); + when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR)); + + final PipelineService pipelineService = mock(MongoDbPipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("p1", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"true\";\n" + + "stage 1 match either\n" + + " rule \"add_foobar\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final Map> functions = ImmutableMap.of(SetField.NAME, new SetField()); + final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions); + + final Messages processed = interpreter.process(messageInDefaultStream("message", "test")); + + final List messages = ImmutableList.copyOf(processed); + assertThat(messages).hasSize(1); + + final Message actualMessage = messages.get(0); + assertThat(actualMessage.getFieldAs(String.class, "foobar")).isEqualTo("covfefe"); + } + + @Test + public void testMatchAllDoesNotContinueIfNotAllRulesMatched() { + final RuleService ruleService = mock(MongoDbRuleService.class); + when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR)); + + final PipelineService pipelineService = mock(MongoDbPipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("p1", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"true\";\n" + + " rule \"false\";\n" + + "stage 1 match either\n" + + " rule \"add_foobar\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final Map> functions = ImmutableMap.of(SetField.NAME, new SetField()); + final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions); + + final Messages processed = interpreter.process(messageInDefaultStream("message", "test")); + + final List messages = ImmutableList.copyOf(processed); + assertThat(messages).hasSize(1); + + final Message actualMessage = messages.get(0); + assertThat(actualMessage.hasField("foobar")).isFalse(); + } + + @Test + public void testMatchEitherContinuesIfOneRuleMatched() { + final RuleService ruleService = mock(MongoDbRuleService.class); + when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR)); + + final PipelineService pipelineService = mock(MongoDbPipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("p1", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match either\n" + + " rule \"true\";\n" + + " rule \"false\";\n" + + "stage 1 match either\n" + + " rule \"add_foobar\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final Map> functions = ImmutableMap.of(SetField.NAME, new SetField()); + final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions); + + final Messages processed = interpreter.process(messageInDefaultStream("message", "test")); + + final List messages = ImmutableList.copyOf(processed); + assertThat(messages).hasSize(1); + + final Message actualMessage = messages.get(0); + assertThat(actualMessage.getFieldAs(String.class, "foobar")).isEqualTo("covfefe"); + } + + @Test + public void testMatchEitherStopsIfNoRuleMatched() { + final RuleService ruleService = mock(MongoDbRuleService.class); + when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR)); + + final PipelineService pipelineService = mock(MongoDbPipelineService.class); + when(pipelineService.loadAll()).thenReturn(Collections.singleton( + PipelineDao.create("p1", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match either\n" + + " rule \"false\";\n" + + "stage 1 match either\n" + + " rule \"add_foobar\";\n" + + "end\n", + Tools.nowUTC(), + null) + )); + + final Map> functions = ImmutableMap.of(SetField.NAME, new SetField()); + final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions); + + final Messages processed = interpreter.process(messageInDefaultStream("message", "test")); + + final List messages = ImmutableList.copyOf(processed); + assertThat(messages).hasSize(1); + + final Message actualMessage = messages.get(0); + assertThat(actualMessage.hasField("foobar")).isFalse(); + } + + @SuppressForbidden("Allow using default thread factory") + private PipelineInterpreter createPipelineInterpreter(RuleService ruleService, PipelineService pipelineService, Map> functions) { + final PipelineStreamConnectionsService pipelineStreamConnectionsService = mock(MongoDbPipelineStreamConnectionsService.class); + final PipelineConnections pipelineConnections = PipelineConnections.create("p1", DEFAULT_STREAM_ID, Collections.singleton("p1")); + when(pipelineStreamConnectionsService.loadAll()).thenReturn(Collections.singleton(pipelineConnections)); + + final FunctionRegistry functionRegistry = new FunctionRegistry(functions); + final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry, new CodeGenerator(JavaCompiler::new)); + + final ConfigurationStateUpdater stateUpdater = new ConfigurationStateUpdater(ruleService, + pipelineService, + pipelineStreamConnectionsService, + parser, + new MetricRegistry(), + functionRegistry, + Executors.newScheduledThreadPool(1), + mock(EventBus.class), + (currentPipelines, streamPipelineConnections) -> new PipelineInterpreter.State(currentPipelines, streamPipelineConnections, new MetricRegistry(), 1, true), + false); + return new PipelineInterpreter( + mock(Journal.class), + new MetricRegistry(), + stateUpdater + ); + } + + @Test + @SuppressForbidden("Allow using default thread factory") + public void testMetrics() { + final RuleService ruleService = new InMemoryRuleService(); + ruleService.save(RuleDao.create("abc", + "title", + "description", + "rule \"match_all\"\n" + + "when true\n" + + "then\n" + + "end", + Tools.nowUTC(), + null) + ); + + final PipelineService pipelineService = new InMemoryPipelineService(); + pipelineService.save(PipelineDao.create("cde", "title", "description", + "pipeline \"pipeline\"\n" + + "stage 0 match all\n" + + " rule \"match_all\";\n" + + "stage 1 match all\n" + + " rule \"match_all\";\n" + + "end\n", + Tools.nowUTC(), + null) + ); + + final PipelineStreamConnectionsService pipelineStreamConnectionsService = new InMemoryPipelineStreamConnectionsService(); + pipelineStreamConnectionsService.save(PipelineConnections.create(null, + DEFAULT_STREAM_ID, + Collections.singleton("cde"))); + + final FunctionRegistry functionRegistry = new FunctionRegistry(Collections.emptyMap()); + final PipelineRuleParser parser = new PipelineRuleParser(functionRegistry, new CodeGenerator(JavaCompiler::new)); + + final MetricRegistry metricRegistry = new MetricRegistry(); + final ConfigurationStateUpdater stateUpdater = new ConfigurationStateUpdater(ruleService, + pipelineService, + pipelineStreamConnectionsService, + parser, + metricRegistry, + functionRegistry, + Executors.newScheduledThreadPool(1), + mock(EventBus.class), + (currentPipelines, streamPipelineConnections) -> new PipelineInterpreter.State(currentPipelines, streamPipelineConnections, new MetricRegistry(), 1, true), + false); + final PipelineInterpreter interpreter = new PipelineInterpreter( + mock(Journal.class), + metricRegistry, + stateUpdater + ); + + interpreter.process(messageInDefaultStream("", "")); + + final SortedMap meters = metricRegistry.getMeters((name, metric) -> name.startsWith(name(Pipeline.class, "cde")) || name.startsWith(name(Rule.class, "abc"))); + + assertThat(meters.keySet()).containsExactlyInAnyOrder( + name(Pipeline.class, "cde", "executed"), + name(Pipeline.class, "cde", "stage", "0", "executed"), + name(Pipeline.class, "cde", "stage", "1", "executed"), + name(Rule.class, "abc", "executed"), + name(Rule.class, "abc", "cde", "0", "executed"), + name(Rule.class, "abc", "cde", "1", "executed"), + name(Rule.class, "abc", "matched"), + name(Rule.class, "abc", "cde", "0", "matched"), + name(Rule.class, "abc", "cde", "1", "matched"), + name(Rule.class, "abc", "not-matched"), + name(Rule.class, "abc", "cde", "0", "not-matched"), + name(Rule.class, "abc", "cde", "1", "not-matched"), + name(Rule.class, "abc", "failed"), + name(Rule.class, "abc", "cde", "0", "failed"), + name(Rule.class, "abc", "cde", "1", "failed") + ); + + assertThat(meters.get(name(Pipeline.class, "cde", "executed")).getCount()).isEqualTo(1L); + assertThat(meters.get(name(Pipeline.class, "cde", "stage", "0", "executed")).getCount()).isEqualTo(1L); + assertThat(meters.get(name(Pipeline.class, "cde", "stage", "1", "executed")).getCount()).isEqualTo(1L); + + assertThat(meters.get(name(Rule.class, "abc", "executed")).getCount()).isEqualTo(2L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "executed")).getCount()).isEqualTo(1L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "executed")).getCount()).isEqualTo(1L); + + assertThat(meters.get(name(Rule.class, "abc", "matched")).getCount()).isEqualTo(2L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "matched")).getCount()).isEqualTo(1L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "matched")).getCount()).isEqualTo(1L); + + assertThat(meters.get(name(Rule.class, "abc", "not-matched")).getCount()).isEqualTo(0L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "not-matched")).getCount()).isEqualTo(0L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "not-matched")).getCount()).isEqualTo(0L); + + assertThat(meters.get(name(Rule.class, "abc", "failed")).getCount()).isEqualTo(0L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "0", "failed")).getCount()).isEqualTo(0L); + assertThat(meters.get(name(Rule.class, "abc", "cde", "1", "failed")).getCount()).isEqualTo(0L); + + } + + private Message messageInDefaultStream(String message, String source) { + final Message msg = new Message(message, source, Tools.nowUTC()); + + final Stream mockedStream = mock(Stream.class); + when(mockedStream.getId()).thenReturn(DEFAULT_STREAM_ID); + msg.addStream(mockedStream); + + return msg; + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java new file mode 100644 index 000000000000..8308f8cf0f0b --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/processors/StageIteratorTest.java @@ -0,0 +1,162 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.processors; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.jooq.lambda.Seq; +import org.junit.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static com.google.common.collect.ImmutableSortedSet.of; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class StageIteratorTest { + + @Test + public void singleEmptyPipeline() { + final ImmutableSet empty = ImmutableSet.of(Pipeline.empty("empty")); + final StageIterator iterator = new StageIterator(empty); + + assertFalse(iterator.hasNext()); + } + + @Test + public void singlePipelineNoStage() { + + final ImmutableSet input = + ImmutableSet.of(Pipeline.builder() + .name("hallo") + .stages(of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build())) + .build()); + final StageIterator iterator = new StageIterator(input); + assertTrue(iterator.hasNext()); + final List nextStages = iterator.next(); + assertEquals(1, nextStages.size()); + + final Stage stage = Iterables.getOnlyElement(nextStages); + assertEquals(0, stage.ruleReferences().size()); + } + + @Test + public void singlePipelineTwoStages() { + final ImmutableSet input = + ImmutableSet.of(Pipeline.builder() + .name("hallo") + .stages(of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(10) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build() + )).build()); + final StageIterator iterator = new StageIterator(input); + //noinspection unchecked + final List[] stages = Iterators.toArray(iterator, List.class); + + assertEquals(2, stages.length); + assertEquals(1, stages[0].size()); + assertEquals("last set of stages are on stage 0", 0, Iterables.getOnlyElement(stages[0]).stage()); + assertEquals(1, stages[1].size()); + assertEquals("last set of stages are on stage 1", 10, Iterables.getOnlyElement(stages[1]).stage()); + } + + + @Test + public void multiplePipelines() { + final ImmutableSortedSet stages1 = + of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(10) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build() + ); + final ImmutableSortedSet stages2 = + of(Stage.builder() + .stage(-1) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(4) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build(), + Stage.builder() + .stage(11) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build() + ); + final ImmutableSortedSet stages3 = + of(Stage.builder() + .stage(0) + .matchAll(true) + .ruleReferences(Collections.emptyList()) + .build()); + + final ImmutableSet input = + ImmutableSet.of(Pipeline.builder() + .name("p1") + .stages(stages1).build(), + Pipeline.builder() + .name("p2") + .stages(stages2).build() + ,Pipeline.builder() + .name("p3") + .stages(stages3).build() + ); + final StageIterator iterator = new StageIterator(input); + + final List> stageSets = Lists.newArrayList(iterator); + + assertEquals("5 different stages to execute", 5, stageSets.size()); + + for (List stageSet : stageSets) { + assertEquals("Each stage set should only contain stages with the same number", + 1, + Seq.seq(stageSet).groupBy(Stage::stage).keySet().size()); + } + assertArrayEquals("Stages must be sorted numerically", + new int[] {-1, 0, 4, 10, 11}, + stageSets.stream().flatMap(Collection::stream).mapToInt(Stage::stage).distinct().toArray()); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSourceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSourceTest.java new file mode 100644 index 000000000000..097e5fceecf0 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rest/PipelineSourceTest.java @@ -0,0 +1,87 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.junit.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PipelineSourceTest { + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + @Test + public void testSerialization() throws Exception { + final StageSource stageSource = StageSource.create(23, true, Collections.singletonList("some-rule")); + final PipelineSource pipelineSource = PipelineSource.create( + "id", + "title", + "description", + "source", + Collections.singletonList(stageSource), + new DateTime(2017, 7, 4, 15, 0, DateTimeZone.UTC), + new DateTime(2017, 7, 4, 15, 0, DateTimeZone.UTC) + ); + final JsonNode json = objectMapper.convertValue(pipelineSource, JsonNode.class); + + assertThat(json.path("id").asText()).isEqualTo("id"); + assertThat(json.path("title").asText()).isEqualTo("title"); + assertThat(json.path("description").asText()).isEqualTo("description"); + assertThat(json.path("source").asText()).isEqualTo("source"); + assertThat(json.path("created_at").asText()).isEqualTo("2017-07-04T15:00:00.000Z"); + assertThat(json.path("modified_at").asText()).isEqualTo("2017-07-04T15:00:00.000Z"); + assertThat(json.path("stages").isArray()).isTrue(); + assertThat(json.path("stages")).hasSize(1); + + final JsonNode stageNode = json.path("stages").get(0); + assertThat(stageNode.path("stage").asInt()).isEqualTo(23); + assertThat(stageNode.path("match_all").asBoolean()).isTrue(); + assertThat(stageNode.path("rules").isArray()).isTrue(); + assertThat(stageNode.path("rules")).hasSize(1); + assertThat(stageNode.path("rules").get(0).asText()).isEqualTo("some-rule"); + } + + @Test + public void testDeserialization() throws Exception { + final String json = "{" + + "\"id\":\"id\"," + + "\"title\":\"title\"," + + "\"description\":\"description\"," + + "\"source\":\"source\"," + + "\"created_at\":\"2017-07-04T15:00:00.000Z\"," + + "\"modified_at\":\"2017-07-04T15:00:00.000Z\"," + + "\"stages\":[{\"stage\":23,\"match_all\":true,\"rules\":[\"some-rule\"]}]" + + "}"; + + final PipelineSource pipelineSource = objectMapper.readValue(json, PipelineSource.class); + assertThat(pipelineSource.id()).isEqualTo("id"); + assertThat(pipelineSource.title()).isEqualTo("title"); + assertThat(pipelineSource.description()).isEqualTo("description"); + assertThat(pipelineSource.source()).isEqualTo("source"); + assertThat(pipelineSource.createdAt()).isEqualTo(new DateTime(2017, 7, 4, 15, 0, DateTimeZone.UTC)); + assertThat(pipelineSource.modifiedAt()).isEqualTo(new DateTime(2017, 7, 4, 15, 0, DateTimeZone.UTC)); + assertThat(pipelineSource.stages()) + .hasSize(1) + .containsOnly(StageSource.create(23, true, Collections.singletonList("some-rule"))); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rest/StageSourceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rest/StageSourceTest.java new file mode 100644 index 000000000000..ea73e2606703 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/rest/StageSourceTest.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog.plugins.pipelineprocessor.rest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.junit.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +public class StageSourceTest { + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + @Test + public void testSerialization() throws Exception { + final StageSource stageSource = StageSource.create(23, true, Collections.singletonList("some-rule")); + final JsonNode json = objectMapper.convertValue(stageSource, JsonNode.class); + + assertThat(json.path("stage").asInt()).isEqualTo(23); + assertThat(json.path("match_all").asBoolean()).isTrue(); + assertThat(json.path("rules").isArray()).isTrue(); + assertThat(json.path("rules")).hasSize(1); + assertThat(json.path("rules").get(0).asText()).isEqualTo("some-rule"); + } + + @Test + public void testDeserialization() throws Exception { + final String json = "{\"stage\":23,\"match_all\":true,\"rules\":[\"some-rule\"]}"; + final StageSource stageSource = objectMapper.readValue(json, StageSource.class); + assertThat(stageSource.stage()).isEqualTo(23); + assertThat(stageSource.matchAll()).isTrue(); + assertThat(stageSource.rules()) + .hasSize(1) + .containsOnly("some-rule"); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/audit/AuditCoverageTest.java b/graylog2-server/src/test/java/org/graylog2/audit/AuditCoverageTest.java index 6b1a96439e7e..63ce771a6f32 100644 --- a/graylog2-server/src/test/java/org/graylog2/audit/AuditCoverageTest.java +++ b/graylog2-server/src/test/java/org/graylog2/audit/AuditCoverageTest.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import org.graylog.plugins.pipelineprocessor.audit.PipelineProcessorAuditEventTypes; import org.graylog2.audit.jersey.AuditEvent; import org.graylog2.audit.jersey.NoAuditEvent; import org.junit.Test; @@ -41,7 +42,11 @@ public void testAuditCoverage() throws Exception { final ConfigurationBuilder configurationBuilder = new ConfigurationBuilder() .setUrls(ClasspathHelper.forPackage("org.graylog2")) .setScanners(new MethodAnnotationsScanner()); - final Set auditEventTypes = new AuditEventTypes().auditEventTypes(); + // TODO: Dynamically discover event types? + final Set auditEventTypes = ImmutableSet.builder() + .addAll(new AuditEventTypes().auditEventTypes()) + .addAll(new PipelineProcessorAuditEventTypes().auditEventTypes()) + .build(); final Reflections reflections = new Reflections(configurationBuilder); final ImmutableSet.Builder methods = ImmutableSet.builder(); diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/codegen/runCodegen.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/codegen/runCodegen.txt new file mode 100644 index 000000000000..483ec31b37ae --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/codegen/runCodegen.txt @@ -0,0 +1,11 @@ +rule "grok jenkins extraction" +when + to_string($message.source) == "jenkins.torch.sh" && + (regex("#\\d+", to_string($message.message)).matches == true || !has_field("something_that_doesnt_exist")) +then + let number = 1; + let string = "sadfasdf"; + let fields = {some_identifier: 1, `something with spaces`: "some expression"}; + let ary = [1,3,4,5,"object", string]; + set_fields(fields); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessage.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessage.txt new file mode 100644 index 000000000000..45462e9b26bd --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessage.txt @@ -0,0 +1,14 @@ +rule "operate on cloned message" +when true +then + let x = clone_message(); + let new = create_message("foo", "source"); + let cloned = clone_message(new); + + set_field(field: "removed_again", value: "foo", message: x); + set_field(field: "only_in", value: "new message", message: x); + set_fields(fields: { multi: "new message" }, message: x); + set_field(field: "has_source", value: has_field("source", x), message: x); + route_to_stream(name: "some stream", message: x); + remove_field("removed_again", x); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessageWithInvalidTimestamp.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessageWithInvalidTimestamp.txt new file mode 100644 index 000000000000..06cd26655672 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessageWithInvalidTimestamp.txt @@ -0,0 +1,5 @@ +rule "operate on cloned message" +when true +then + let cloned = clone_message(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/comparisons.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/comparisons.txt new file mode 100644 index 000000000000..e7f4a0e3a606 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/comparisons.txt @@ -0,0 +1,54 @@ +rule "comparison" +when + is_string("") == true && + is_string("foobar") == true && + is_string(false) == false && + is_string(1000) == false && + is_string(1.234d) == false && + + is_bool(true) == true && + is_bool(false) == true && + is_bool("foobar") == false && + is_bool(1234) == false && + is_bool(23.42) == false && + + is_double(23.42) == true && + is_double(23) == false && + is_double(true) == false && + is_double("foobar") == false && + + is_long(23) == true && + is_long(23.42) == false && + is_long(true) == false && + is_long("foobar") == false && + + is_number(23) == true && + is_number(23.42) == true && + is_number(true) == false && + is_number("foobar") == false && + + is_collection(["foobar", "foobaz"]) == true && + is_collection({foo:"bar"}) == false && + is_collection("foobar") == false && + is_collection(true) == false && + is_collection(23) == false && + is_collection(23.42) == false && + is_collection("foobar") == false && + + is_list(["foobar", "foobaz"]) == true && + is_list({foo:"bar"}) == false && + is_list("foobar") == false && + is_list(true) == false && + is_list(23) == false && + is_list(23.42) == false && + is_list("foobar") == false && + + is_map({foo:"bar"}) == true && + is_map(["foobar", "foobaz"]) == false && + is_map(true) == false && + is_map(23) == false && + is_map(23.42) == false && + is_map("foobar") == false +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt new file mode 100644 index 000000000000..efc378898875 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/conversions.txt @@ -0,0 +1,57 @@ +rule "conversions" +when true +then + set_fields({ + string_1: to_string("1"), // "1" + string_2: to_string("2", "default"), // "2" + string_3: to_string($message.not_there), // "" -> not being set in message! + string_4: to_string($message.not_there, "default"), // "default" + string_5: to_string(false), // "false" + string_6: to_string(42), // "42" + string_7: to_string(23.42d), // "23.42" + + long_1: to_long(1), // 1L + long_2: to_long(2, 1), // 2L + long_3: to_long($message.not_there), // 0L + long_4: to_long($message.not_there, 1), // 1L + long_5: to_long(23.42d), // 23L + long_6: to_long("23"), // 23L + long_7: to_long("23.42", 1), // 1L + long_min1: to_long("-9223372036854775808", 1), // Long.MIN_VALUE + long_min2: to_long("-9223372036854775809", 1), // 1L + long_max1: to_long("9223372036854775807", 1), // Long.MAX_VALUE + long_max2: to_long("9223372036854775808", 1), // 1L + + double_1: to_double(1d), // 1d + double_2: to_double(2d, 1d), // 2d + double_3: to_double($message.not_there), // 0d + double_4: to_double($message.not_there, 1d), // 1d + double_5: to_double(23), // 23d + double_6: to_double("23"), // 23d + double_7: to_double("23.42"), // 23.42d + double_min1: to_double("4.9E-324"), // Double.MIN_VALUE + double_min2: to_double("4.9E-325", 1d), // 0d + double_max1: to_double("1.7976931348623157E308"), // Double.MAX_VALUE + double_inf1: to_double("1.7976931348623157E309", 1d), // Infinity + double_inf2: to_double("-1.7976931348623157E309", 1d), // -Infinity + double_inf3: to_double("Infinity", 1d), // Infinity + double_inf4: to_double("-Infinity", 1d), // -Infinity + + bool_1: to_bool("true"), // true + bool_2: to_bool("false", true), // false + bool_3: to_bool($message.not_there), // false + bool_4: to_bool($message.not_there, true), // true + + ip_1: to_ip("127.0.0.1"), // 127.0.0.1 + ip_2: to_ip("127.0.0.1", "2001:db8::1"), // 127.0.0.1 + ip_3: to_ip($message.not_there), // 0.0.0.0 + ip_4: to_ip($message.not_there, "::1"), // ::1 (v6) + + map_1: to_map({foo:"bar"}), // Map.of("foo", "bar") + map_2: to_map("foobar"), // empty map + map_3: to_map(23), // empty map + map_4: to_map(23.42), // empty map + map_5: to_map(true), // empty map + map_6: to_map($message.not_there) // empty map + }); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateArithmetic.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateArithmetic.txt new file mode 100644 index 000000000000..522f9e443f93 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateArithmetic.txt @@ -0,0 +1,54 @@ +// now() is fixed to "2010-07-30T18:03:25+02:00" to provide a better testing experience +rule "date math" +when + now() + years(1) > now() && + now() + months(1) > now() && + now() + weeks(1) > now() && + now() + days(1) > now() && + now() + hours(1) > now() && + now() + minutes(1) > now() && + now() + seconds(1) > now() && + now() + millis(1) > now() && + now() + period("P1YT1M") > now() && + + now() - years(1) < now() && + now() - months(1) < now() && + now() - weeks(1) < now() && + now() - days(1) < now() && + now() - hours(1) < now() && + now() - minutes(1) < now() && + now() - seconds(1) < now() && + now() - millis(1) < now() && + now() - period("P1YT1M") < now() && + + is_period(years(1)) == true && + is_period(months(1)) == true && + is_period(weeks(1)) == true && + is_period(days(1)) == true && + is_period(hours(1)) == true && + is_period(minutes(1)) == true && + is_period(seconds(1)) == true && + is_period(millis(1)) == true && + is_period(period("P1YT1M")) == true && + is_period("foobar") == false && + is_period(1234) == false && + is_period(12.34) == false && + is_period(true) == false +then + set_field("interval", now() - (now() - days(1))); // is a duration of 1 day + set_field("long_time_ago", now() - years(10000)); + set_fields({ + years: years(2), + months: months(2), + weeks: weeks(2), + days: days(2), + hours: hours(2), + minutes: minutes(2), + seconds: seconds(2), + millis: millis(2), + period: period("P1YT1M") + }); + set_field("timestamp", to_date($message.timestamp) + hours(1)); + + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt new file mode 100644 index 000000000000..c142ec1d6203 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dates.txt @@ -0,0 +1,47 @@ +// now() is fixed, test uses different millisprovider! + +rule "dates" +when + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") == parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + now("CET") == now("UTC") && + now("CET") == now() && + flex_parse_date(value: "30th July 2010 18:03:25 ", timezone: "CET") == parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") && + format_date(flex_parse_date("30th July 2010 18:03:25"), "yyyy-MM-dd") == "2010-07-30" && + parse_date("2010-07-30T18:03:24+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") < parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:24+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") >= parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") > parse_date("2010-07-30T16:03:24Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") <= parse_date("2010-07-30T16:03:24Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") <= parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") > parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") >= parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ") && + !(parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ") < parse_date("2010-07-30T16:03:25Z", "yyyy-MM-dd'T'HH:mm:ssZZ")) && + is_date(now("CET")) == true && + is_date("foobar") == false && + is_date(1234) == false && + is_date(12.34) == false +then + trigger_test(); + let date = parse_date("2010-07-30T18:03:25+02:00", "yyyy-MM-dd'T'HH:mm:ssZZ"); + set_field("year", date.year); + set_field("timezone", to_string(date.zone)); + + // Date parsing locales + let germanDate = parse_date(value: "24. Juli 1983", pattern: "dd. MMM yyyy", locale: "de"); + set_field("german_day", germanDate.dayOfMonth); + set_field("german_month", germanDate.monthOfYear); + set_field("german_year", germanDate.year); + + let englishDate = parse_date(value: "July 24, 1983", pattern: "MMM dd, yyyy", locale: "en"); + set_field("english_day", englishDate.dayOfMonth); + set_field("english_month", englishDate.monthOfYear); + set_field("english_year", englishDate.year); + + let frenchDate = parse_date(value: "24 juillet 1983", pattern: "dd MMM yyyy", locale: "fr"); + set_field("french_day", frenchDate.dayOfMonth); + set_field("french_month", frenchDate.monthOfYear); + set_field("french_year", frenchDate.year); + + set_field("ts_hour", $message.timestamp.hourOfDay); + set_field("ts_minute", $message.timestamp.minuteOfHour); + set_field("ts_second", $message.timestamp.secondOfMinute); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/datesUnixTimestamps.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/datesUnixTimestamps.txt new file mode 100644 index 000000000000..fd04608480d4 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/datesUnixTimestamps.txt @@ -0,0 +1,8 @@ +rule "dates" +when + parse_unix_milliseconds(0) == parse_date("1970-01-01T00:00:00.000Z", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") && + parse_unix_milliseconds(1516272143555) == parse_date("2018-01-18T10:42:23.555Z", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") && + parse_unix_milliseconds(1516272143555, "Europe/Berlin") == parse_date(value: "2018-01-18T11:42:23.555", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSS", timezone: "Europe/Berlin") +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt new file mode 100644 index 000000000000..3e938e60fea0 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt @@ -0,0 +1,13 @@ +rule "digests" +when + crc32("graylog") == "e3018c57" && + crc32c("graylog") == "82390e89" && + md5("graylog") == "6f9efb466e043b9f3635827ce446e13c" && + murmur3_32("graylog") == "67285534" && + murmur3_128("graylog") == "945d5b1aaa8fdfe9b880b31e814972b3" && + sha1("graylog") == "6d88bccf40bf65b911fe79d78c7af98e382f0c1a" && + sha256("graylog") == "4bbdd5a829dba09d7a7ff4c1367be7d36a017b4267d728d31bd264f63debeaa6" && + sha512("graylog") == "f6cb3a96450fb9c9174299a651333c926cd67b6f5c25d8daeede1589ffa006f4dd31da4f0625b7f281051a34c8352b3a9c1a9babf90020360e911a380b5c3f4f" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/encodings.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/encodings.txt new file mode 100644 index 000000000000..f9d110a33f5c --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/encodings.txt @@ -0,0 +1,21 @@ +rule "digests" +when + base16_encode(value: "graylog") == "677261796C6F67" && + base16_decode(value: "677261796C6F67") == "graylog" && + base16_encode(value: "graylog", omit_padding: true) == "677261796C6F67" && + base16_decode(value: "677261796C6F67", omit_padding: true) == "graylog" && + base32_encode(value: "graylog") == "CTP62UBCDTJG====" && + base32_decode(value: "CTP62UBCDTJG====") == "graylog" && + base32_encode(value: "graylog", omit_padding: true) == "CTP62UBCDTJG" && + base32_decode(value: "CTP62UBCDTJG", omit_padding: true) == "graylog" && + base32human_encode(value: "graylog") == "M5ZGC6LMN5TQ====" && + base32human_decode(value: "M5ZGC6LMN5TQ====") == "graylog" && + base32human_encode(value: "graylog", omit_padding: true) == "M5ZGC6LMN5TQ" && + base32human_decode(value: "M5ZGC6LMN5TQ", omit_padding: true) == "graylog" && + base64_encode(value: "graylog") == "Z3JheWxvZw==" && + base64_decode(value: "Z3JheWxvZw==") == "graylog" && + base64url_encode(value: "graylog", omit_padding: true) == "Z3JheWxvZw" && + base64url_decode(value: "Z3JheWxvZw", omit_padding: true) == "graylog" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt new file mode 100644 index 000000000000..ea0989acdecd --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt @@ -0,0 +1,6 @@ +rule "suppressing exceptions/nulls" +when + is_null(to_ip($message.does_not_exist, "d.f.f.f")) && is_not_null($message.this_field_was_set) +then + trigger_test(); +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt new file mode 100644 index 000000000000..8e94a920e621 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt @@ -0,0 +1,40 @@ +rule "prefixsuffix" +when true +then + // plain set field + set_field("field", "1"); + // both prefix and suffix, doesn't touch the above + set_field("field", "2", "prae_", "_sueff"); + + // combinations of optional prefix, suffix + set_field(field: "field", value: "3", suffix: "_sueff"); + set_field(field: "field", value: "4", prefix: "prae_"); + + // set multiple fields with the same prefix + set_fields( + fields: { + field1: "5", + field2: "6" + }, + prefix: "pre_", + suffix: "_suff" + ); + + // set multiple fields with the same prefix, suffix optional + set_fields( + fields: { + field1: "7", + field2: "8" + }, + prefix: "pre_" + ); + // set multiple fields with the same suffix, prefix optional + set_fields( + fields: { + field1: "9", + field2: "10" + }, + suffix: "_suff" + ); +end + diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt new file mode 100644 index 000000000000..1fc8c285c622 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt @@ -0,0 +1,8 @@ +rule "fieldRenaming" +when true +then + + rename_field("no_such_field", "field_1"); + rename_field("field_a", "field_2"); + rename_field("field_b", "field_b"); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt new file mode 100644 index 000000000000..49e57025f301 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt @@ -0,0 +1,10 @@ +rule "grok" +when true +then + let matches = grok(pattern: "%{GREEDY:timestamp;date;yyyy-MM-dd'T'HH:mm:ss.SSSX}", value: "2015-07-31T10:05:36.773Z"); + set_fields(matches); + + // only named captures + let matches1 = grok("%{NUM:num}", "10", true); + set_fields(matches1); +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt new file mode 100644 index 000000000000..78578eb39b00 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt @@ -0,0 +1,14 @@ +rule "ip handling" +when + cidr_match("192.0.0.0/8", to_ip("192.168.1.50")) && + ! cidr_match("191.0.0.0/8", to_ip("192.168.1.50")) && + is_ip(to_ip("127.0.0.1")) == true && + is_ip("foobar") == false && + is_ip(1234) == false && + is_ip(12.34) == false && + is_ip(true) == false +then + set_field("ip_anon", to_string(to_ip($message.ip).anonymized)); + set_field("ipv6_anon", to_string(to_ip("2001:db8::1").anonymized)); + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt new file mode 100644 index 000000000000..7dcd035e63c5 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt @@ -0,0 +1,6 @@ +rule "IP subnet" +when + cidr_match("10.20.30.0/24", to_ip($message.source)) +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/json.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/json.txt new file mode 100644 index 000000000000..6e1475564f4e --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/json.txt @@ -0,0 +1,19 @@ +rule "json" +when + true +then + let json = parse_json(to_string($message.flat_json)); + set_fields(to_map(json)); + + // Don't fail on invalid input + let invalid_json = parse_json("#FOOBAR#"); + set_fields(to_map(invalid_json)); + + // Don't fail on empty input + let empty_json = parse_json(""); + set_fields(to_map(empty_json)); + + // Don't fail on nested input + let nested_json = parse_json(to_string($message.nested_json)); + set_fields(to_map(nested_json)); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt new file mode 100644 index 000000000000..1f97ba272a8b --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt @@ -0,0 +1,24 @@ +rule "jsonpath" +when + is_json(parse_json("{}")) == true && + is_json("foobar") == false && + is_json(1234) == false && + is_json(12.34) == false && + is_json(true) == false +then + let x = parse_json(to_string($message.message)); + let new_fields = select_jsonpath(x, + { author_first: "$['store']['book'][0]['author']", + author_last: "$['store']['book'][-1:]['author']" + }); + set_fields(new_fields); + + // Don't fail on empty input + let invalid_json = parse_json("#FOOBAR#"); + let invalid_json_fields = select_jsonpath(invalid_json, { some_field: "$.message" }); + set_fields(invalid_json_fields); + + // Don't fail on missing field + let missing_fields = select_jsonpath(x, { some_field: "$.i_dont_exist", this_should_exist: "$['store']['book'][-1:]['author']" }); + set_fields(missing_fields); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt new file mode 100644 index 000000000000..95f7f983ce1c --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt @@ -0,0 +1,23 @@ +rule "kv" +when true +then + set_fields(key_value( + value: "a='1' =2 \n 'c'=3 [d]=44 a=4 \"e\"=4 [f=1][[g]:3] h=", + delimiters: " \t\n\r[", + kv_delimiters: "=:", + ignore_empty_values: true, + trim_key_chars: "\"[]<>'", + trim_value_chars: "']", + allow_dup_keys: true, // the default + handle_dup_keys: "," // meaning concat, default "take_first" + )); + + set_fields(key_value( + value: "dup_first=1 dup_first=2", + handle_dup_keys: "take_first" + )); + set_fields(key_value( + value: "dup_last=1 dup_last=2", + handle_dup_keys: "take_last" + )); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt new file mode 100644 index 000000000000..0e1f18107de3 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt @@ -0,0 +1,12 @@ +rule "kv" +when true +then + set_fields(key_value( + value: "dup_first=1 dup_first=2", + allow_dup_keys: false + )); + set_fields(key_value( + value: "dup_last=", + ignore_empty_values: false + )); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt new file mode 100644 index 000000000000..f4f32ae20c25 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt @@ -0,0 +1,12 @@ +rule "operate on newly created message" +when true +then + let x = create_message("new", "synthetic", now()); + + set_field(field: "removed_again", value: "foo", message: x); + set_field(field: "only_in", value: "new message", message: x); + set_fields(fields: { multi: "new message" }, message: x); + set_field(field: "has_source", value: has_field("source", x), message: x); + route_to_stream(name: "some stream", message: x); + remove_field("removed_again", x); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt new file mode 100644 index 000000000000..07124cfca4df --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt @@ -0,0 +1,13 @@ +rule "regexMatch" +when + regex("^.*(cde\\.)(:(\\d+))?.*$", "abcde.fg").matches == true && + regex(".*(cde\\.)(:(\\d+))?.*", "abcde.fg").matches == true && + regex("(cde\\.)(:(\\d+))?", "abcde.fg").matches == true && + regex("^(cde\\.)(:(\\d+))?$", "abcde.fg").matches == false +then + let result = regex("(cd\\.e)", "abcd.efg"); + set_field("group_1", result["0"]); + let result = regex("(cd\\.e)", "abcd.efg", ["name"]); + set_field("named_group", result["name"]); + set_field("matched_regex", result.matches); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStream.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStream.txt new file mode 100644 index 000000000000..9fc938c8bf24 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStream.txt @@ -0,0 +1,5 @@ +rule "stream routing" +when true +then + remove_from_stream(name: "some name"); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStreamRetainDefault.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStreamRetainDefault.txt new file mode 100644 index 000000000000..c31106569568 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStreamRetainDefault.txt @@ -0,0 +1,7 @@ +rule "stream routing" +when true +then + remove_from_stream(name: "some name"); + // if a message is taken off all stream it was on, the default stream will be added back to avoid dropping the message + remove_from_stream(id: "000000000000000000000001"); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStream.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStream.txt new file mode 100644 index 000000000000..794d0e465b94 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStream.txt @@ -0,0 +1,5 @@ +rule "stream routing" +when true +then + route_to_stream(name: "some name"); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStreamRemoveDefault.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStreamRemoveDefault.txt new file mode 100644 index 000000000000..97bd97b8624b --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStreamRemoveDefault.txt @@ -0,0 +1,5 @@ +rule "stream routing" +when true +then + route_to_stream(name: "some name", remove_from_default: true); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/split.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/split.txt new file mode 100644 index 000000000000..6ea533e7bdc1 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/split.txt @@ -0,0 +1,9 @@ +rule "split" +when + true +then + set_field("limit_0", split("_", "foo_bar_baz")); + set_field("limit_1", split(":", "foo:bar:baz", 1)); + set_field("limit_2", split("\\|", "foo|bar|baz", 2)); + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt new file mode 100644 index 000000000000..0b01a7523b52 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt @@ -0,0 +1,34 @@ +// various string functions +rule "string tests" +when + contains("abcdef", "bc") && + lowercase("a MIXED bag of chArs") == "a mixed bag of chars" && + uppercase("a MIXED bag of chArs") == "A MIXED BAG OF CHARS" && + swapcase("Capitalized") == "cAPITALIZED" && + capitalize("hello") == "Hello" && + capitalize("hEllo") == "HEllo" && + uncapitalize("Hello") == "hello" && + uncapitalize("HEllo") == "hEllo" && + abbreviate("", 4) == "" && + abbreviate("abcdefg", 6) == "abc..." && + abbreviate("abcdefg", 7) == "abcdefg" && + abbreviate("abcdefg", 8) == "abcdefg" && + abbreviate("abcdefg", 4) == "a..." && + concat("foo", "bar") == "foobar" && + starts_with("foobar", "foo") == true && + starts_with("foobar", "") == true && + starts_with("", "foo") == false && + starts_with("foobar", "abc") == false && + starts_with("foobar", "FOO") == false && + starts_with("foobar", "FOO", true) == true && + ends_with("foobar", "bar") == true && + ends_with("foobar", "") == true && + ends_with("", "bar") == false && + ends_with("foobar", "abc") == false && + ends_with("foobar", "BAR") == false && + ends_with("foobar", "BAR", true) == true +then + set_field("has_xyz", contains("abcdef", "xyz")); + set_field("string_literal", "abcd\\.e\tfg\u03a9\363"); + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt new file mode 100644 index 000000000000..7d253bdb10da --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt @@ -0,0 +1,15 @@ +rule "substrings" +when + substring("abc", 0, 2) == "ab" && + substring("abc", 0, 2) == "ab" && + substring("abc", 2, 0) == "" && + substring("abc", 2, 4) == "c" && + substring("abc", 4, 6) == "" && + substring("abc", 2, 2) == "" && + substring("abc", -2, -1) == "b" && + substring("abc", -4, 2) == "ab" && + substring("abc", 1) == "bc" && + substring("abc", 0, -1) == "ab" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt new file mode 100644 index 000000000000..c013f59a88f3 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/syslog.txt @@ -0,0 +1,50 @@ +rule "syslog tests" +when + true +then + set_field("level0", syslog_level(0)); + set_field("level1", syslog_level(1)); + set_field("level2", syslog_level(2)); + set_field("level3", syslog_level(3)); + set_field("level4", syslog_level(4)); + set_field("level5", syslog_level(5)); + set_field("level6", syslog_level(6)); + set_field("level7", syslog_level(7)); + + set_field("facility0", syslog_facility(0)); + set_field("facility1", syslog_facility(1)); + set_field("facility2", syslog_facility(2)); + set_field("facility3", syslog_facility(3)); + set_field("facility4", syslog_facility(4)); + set_field("facility5", syslog_facility(5)); + set_field("facility6", syslog_facility(6)); + set_field("facility7", syslog_facility(7)); + set_field("facility8", syslog_facility(8)); + set_field("facility9", syslog_facility(9)); + set_field("facility10", syslog_facility(10)); + set_field("facility11", syslog_facility(11)); + set_field("facility12", syslog_facility(12)); + set_field("facility13", syslog_facility(13)); + set_field("facility14", syslog_facility(14)); + set_field("facility15", syslog_facility(15)); + set_field("facility16", syslog_facility(16)); + set_field("facility17", syslog_facility(17)); + set_field("facility18", syslog_facility(18)); + set_field("facility19", syslog_facility(19)); + set_field("facility20", syslog_facility(20)); + set_field("facility21", syslog_facility(21)); + set_field("facility22", syslog_facility(22)); + set_field("facility23", syslog_facility(23)); + + let priority1 = expand_syslog_priority(0); + set_fields({prio1_facility: priority1.facility, prio1_level: priority1.level }); + let priority2 = expand_syslog_priority(165); + set_fields({prio2_facility: priority2.facility, prio2_level: priority2.level }); + + let priority3 = expand_syslog_priority_as_string(0); + set_fields({prio3_facility: priority3.facility, prio3_level: priority3.level }); + let priority4 = expand_syslog_priority_as_string(165); + set_fields({prio4_facility: priority4.facility, prio4_level: priority4.level }); + + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt new file mode 100644 index 000000000000..396f9d2ee393 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt @@ -0,0 +1,16 @@ +// now() is fixed, test uses different millisprovider! + +rule "timezones" +when + now("CET") == now("UTC") && + now("utc") == now("UTC") && + now("Europe/Moscow") == now("europe/moscow") && + now("europe/MoSCOw") == now("msk") && + to_string(now("europe/MoSCOw").zone) == "Europe/Moscow" && + to_string(now("cet").zone) == "CET" && + to_string(now("Etc/gmt-14").zone) == "Etc/GMT-14" && + to_string(now("").zone) == "UTC" && + to_string(now("invalid-timezone").zone) == "UTC" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/urls.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/urls.txt new file mode 100644 index 000000000000..5608ff5e0af8 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/urls.txt @@ -0,0 +1,25 @@ +rule "urls" +when + is_url("foobar") == false && + is_url(1234) == false && + is_url(12.34) == false && + is_url(true) == false && + is_url(to_url("http://example.org/")) == true +then + let url = to_url("https://admin:s3cr31@some.host.with.lots.of.subdomains.com:9999/path1/path2/three?q1=something&with_spaces=hello%20graylog&equal=can=containanotherone#anchorstuff"); + set_fields({ + protocol: url.protocol, + authority: url.authority, + user_info: url.userInfo, + host: url.host, + port: url.port, + path: url.path, + file: url.file, + fragment: url.fragment, + query: url.query, + q1: url.queryParams.q1, + with_spaces: url.queryParams.with_spaces, + equal: url.queryParams.equal + }); + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/arithmetic.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/arithmetic.txt new file mode 100644 index 000000000000..f1473ca80bfb --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/arithmetic.txt @@ -0,0 +1,13 @@ +rule "arithmetic operators" +when + 1.0 + 1.0 == 2.0 && + 8 * 2 > 15 && + double_valued_func() / 20.0 == 0.0 && + 21 % 20 == 1 && + 10.0 / 20.0 == 0.5 && + +10.0 / -5.0 == -2.0 && + -double_valued_func() == -0.0 && + double_valued_func() + 1.0 > 0.0 +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt new file mode 100644 index 000000000000..f6019098a3fb --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt @@ -0,0 +1,5 @@ +rule "something" +when double_valued_func() > 1.0d AND false == true +then +double_valued_func(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanNot.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanNot.txt new file mode 100644 index 000000000000..f4c5132f7594 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanNot.txt @@ -0,0 +1,6 @@ +rule "booleanNot" +when + !false == false +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanValuedFunctionAsCondition.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanValuedFunctionAsCondition.txt new file mode 100644 index 000000000000..6c226ae823fc --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanValuedFunctionAsCondition.txt @@ -0,0 +1,5 @@ +rule "bool function as top level" +when doch() +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/dateArithmetic.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/dateArithmetic.txt new file mode 100644 index 000000000000..fc20d332fb6c --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/dateArithmetic.txt @@ -0,0 +1,27 @@ +// now() is fixed to "2010-07-30T18:03:25+02:00" to provide a better testing experience +rule "date math" +when + now() + years(1) > now() && + now() + months(1) > now() && + now() + weeks(1) > now() && + now() + days(1) > now() && + now() + hours(1) > now() && + now() + minutes(1) > now() && + now() + seconds(1) > now() && + now() + millis(1) > now() && + now() + period("P1YT1M") > now() && + + now() - years(1) < now() && + now() - months(1) < now() && + now() - weeks(1) < now() && + now() - days(1) < now() && + now() - hours(1) < now() && + now() - minutes(1) < now() && + now() - seconds(1) < now() && + now() - millis(1) < now() && + now() - period("P1YT1M") < now() + +then + set_field("interval", now() - (now() - days(1))); // is a duration of 1 day + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/declaredFunction.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/declaredFunction.txt new file mode 100644 index 000000000000..2804b06d62b3 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/declaredFunction.txt @@ -0,0 +1,4 @@ +rule "using declared function 'nein'" +when true == nein() +then +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt new file mode 100644 index 000000000000..db37fbb31e5e --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt @@ -0,0 +1,6 @@ +rule "indexed array and map access" +when + ["first","second"][0] == "first" and {third: "a value"}["third"] == "a value" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt new file mode 100644 index 000000000000..93e221c3073f --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt @@ -0,0 +1,6 @@ +rule "indexed array and map access" +when + ["first"][true] == "first" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt new file mode 100644 index 000000000000..2ffae9939978 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt @@ -0,0 +1,6 @@ +rule "indexed array and map access" +when + one_arg("not an array")[0] == "first" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/inferVariableType.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/inferVariableType.txt new file mode 100644 index 000000000000..d72935a40e0d --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/inferVariableType.txt @@ -0,0 +1,6 @@ +rule "infer" +when true +then + let x = one_arg("string"); + one_arg(x); +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgType.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgType.txt new file mode 100644 index 000000000000..373df5300893 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgType.txt @@ -0,0 +1,6 @@ +rule "invalid arg type" +when one_arg(0d) == "0" // one_arg needs a String argument, but 0d is Double +then + let x = double_valued_func(); + one_arg(x); // this needs a String argument, but x resolves to Double +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt new file mode 100644 index 000000000000..f1d96cfa30ee --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt @@ -0,0 +1,4 @@ +rule "invalid arg" +when now_in_tz("123") // this isn't a valid tz +then +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidDateAddition.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidDateAddition.txt new file mode 100644 index 000000000000..7d71413475c3 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidDateAddition.txt @@ -0,0 +1,4 @@ +rule "cannot add dates" +when + now() + now() == now() +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/issue185.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/issue185.txt new file mode 100644 index 000000000000..638fd599fd98 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/issue185.txt @@ -0,0 +1,6 @@ +rule "issue-185" +when + true +then + let a = "\s+$" +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mapArrayLiteral.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mapArrayLiteral.txt new file mode 100644 index 000000000000..a8fba5bfdb70 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mapArrayLiteral.txt @@ -0,0 +1,5 @@ +rule "mapliteral" +when sort(keys({some_identifier: 1, `something with spaces`: "some expression"})) == ["some_identifier", "something with spaces"] +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt new file mode 100644 index 000000000000..c34a6101026a --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt @@ -0,0 +1,5 @@ +rule "message field ref" +when to_long(value: $message.responseCode, default: 200) >= 500 +then + set_field(field: "response_category", value: "server_error"); +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt new file mode 100644 index 000000000000..c22284f1278a --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt @@ -0,0 +1,5 @@ +rule "test" +when to_string($message.`@specialfieldname`, "empty") == "string" +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mismatchedNumericTypes.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mismatchedNumericTypes.txt new file mode 100644 index 000000000000..0a26a42b89a7 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mismatchedNumericTypes.txt @@ -0,0 +1,6 @@ +rule "incompatible numeric types inference" +when + 1.0 + 10 == 11 // error: no automatic long -> double conversion! +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/nestedFieldAccess.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/nestedFieldAccess.txt new file mode 100644 index 000000000000..114f9d231343 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/nestedFieldAccess.txt @@ -0,0 +1,15 @@ +// Test nested field/bean access with camel case and snake case +rule "nested field access" +when + beanObject("1", "john", "doe").id == "1" && + beanObject("1", "john", "doe").theName.firstName == "john" && + beanObject("1", "john", "doe").theName.lastName == "doe" && + beanObject("1", "john", "doe").theName.first_name == "john" && + beanObject("1", "john", "doe").theName.last_name == "doe" && + beanObject("1", "john", "doe").the_name.firstName == "john" && + beanObject("1", "john", "doe").the_name.lastName == "doe" && + beanObject("1", "john", "doe").the_name.first_name == "john" && + beanObject("1", "john", "doe").the_name.last_name == "doe" +then + trigger_test(); +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalArguments.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalArguments.txt new file mode 100644 index 000000000000..913b30f7bc0d --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalArguments.txt @@ -0,0 +1,6 @@ +rule "optional function arguments" +when + optional(d: 3, a: true, b: "string") +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalParamsMustBeNamed.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalParamsMustBeNamed.txt new file mode 100644 index 000000000000..c0eb68be4bf7 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalParamsMustBeNamed.txt @@ -0,0 +1,5 @@ +rule "optionalParamsMustBeNamed" +when + optional(false, "string", 3) +then +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/pipelineDeclaration.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/pipelineDeclaration.txt new file mode 100644 index 000000000000..93f2037a2989 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/pipelineDeclaration.txt @@ -0,0 +1,11 @@ +pipeline "cisco" +stage 1 match all + rule "check_ip_whitelist" + rule "cisco_device" +stage 2 match either + rule "parse_cisco_time" + rule "extract_src_dest" + rule "normalize_src_dest" + rule "lookup_ips" + rule "resolve_ips" +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/positionalArguments.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/positionalArguments.txt new file mode 100644 index 000000000000..ac834b085e21 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/positionalArguments.txt @@ -0,0 +1,5 @@ +rule "positional args" +when concat("a", 1, true) == concat(one: "a", two: 1, three: true) +then +trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/singleArgFunction.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/singleArgFunction.txt new file mode 100644 index 000000000000..5c8d9d718661 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/singleArgFunction.txt @@ -0,0 +1,5 @@ +rule "single arg" +when one_arg("arg") == one_arg(one: "arg") +then + trigger_test(); +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt new file mode 100644 index 000000000000..9bec52c605a2 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt @@ -0,0 +1,6 @@ +rule "typed field access" +when + to_long(customObject("1").id, 0) < 2 +then + trigger_test(); +end diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredFunction.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredFunction.txt new file mode 100644 index 000000000000..419077cee77e --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredFunction.txt @@ -0,0 +1,4 @@ +rule "undeclared function" +when false == unknown() +then +end \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredIdentifier.txt b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredIdentifier.txt new file mode 100644 index 000000000000..b0e11a3f5e75 --- /dev/null +++ b/graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredIdentifier.txt @@ -0,0 +1,5 @@ +rule "undeclared variable" +when true +then + one_arg(one: x); +end \ No newline at end of file diff --git a/graylog2-web-interface/src/actions/pipelines/PipelineConnectionsActions.js b/graylog2-web-interface/src/actions/pipelines/PipelineConnectionsActions.js new file mode 100644 index 000000000000..0d7de6d6a579 --- /dev/null +++ b/graylog2-web-interface/src/actions/pipelines/PipelineConnectionsActions.js @@ -0,0 +1,9 @@ +import Reflux from 'reflux'; + +const PipelineConnectionsActions = Reflux.createActions({ + list: { asyncResult: true }, + connectToStream: { asyncResult: true }, + connectToPipeline: { asyncResult: true }, +}); + +export default PipelineConnectionsActions; diff --git a/graylog2-web-interface/src/actions/pipelines/PipelinesActions.jsx b/graylog2-web-interface/src/actions/pipelines/PipelinesActions.jsx new file mode 100644 index 000000000000..433843e23fd5 --- /dev/null +++ b/graylog2-web-interface/src/actions/pipelines/PipelinesActions.jsx @@ -0,0 +1,12 @@ +import Reflux from 'reflux'; + +const RulesActions = Reflux.createActions({ + 'delete': { asyncResult: true }, + 'list': { asyncResult: true }, + 'get': { asyncResult: true }, + 'save': { asyncResult: true }, + 'update': { asyncResult: true }, + 'parse' : { asyncResult: true }, +}); + +export default RulesActions; \ No newline at end of file diff --git a/graylog2-web-interface/src/actions/rules/RulesActions.jsx b/graylog2-web-interface/src/actions/rules/RulesActions.jsx new file mode 100644 index 000000000000..d749b974e3e7 --- /dev/null +++ b/graylog2-web-interface/src/actions/rules/RulesActions.jsx @@ -0,0 +1,14 @@ +import Reflux from 'reflux'; + +const RulesActions = Reflux.createActions({ + delete: { asyncResult: true }, + list: { asyncResult: true }, + get: { asyncResult: true }, + save: { asyncResult: true }, + update: { asyncResult: true }, + parse: { asyncResult: true }, + multiple: { asyncResult: true }, + loadFunctions: { asyncResult: true }, +}); + +export default RulesActions; \ No newline at end of file diff --git a/graylog2-web-interface/src/actions/simulator/SimulatorActions.js b/graylog2-web-interface/src/actions/simulator/SimulatorActions.js new file mode 100644 index 000000000000..19f374d7e818 --- /dev/null +++ b/graylog2-web-interface/src/actions/simulator/SimulatorActions.js @@ -0,0 +1,7 @@ +import Reflux from 'reflux'; + +const SimulatorActions = Reflux.createActions({ + simulate: { asyncResult: true }, +}); + +export default SimulatorActions; diff --git a/graylog2-web-interface/src/components/navigation/Navigation.jsx b/graylog2-web-interface/src/components/navigation/Navigation.jsx index 2421da6a2184..b7651b6d46c9 100644 --- a/graylog2-web-interface/src/components/navigation/Navigation.jsx +++ b/graylog2-web-interface/src/components/navigation/Navigation.jsx @@ -84,6 +84,9 @@ const Navigation = React.createClass({ if (this._isActive('/system/configurations')) { return `${prefix} / Configurations`; } + if (this._isActive('/system/pipelines')) { + return `${prefix} / Pipelines`; + } const pluginRoute = PluginStore.exports('systemnavigation').filter(route => this._isActive(route.path))[0]; if (pluginRoute) { @@ -233,6 +236,11 @@ const Navigation = React.createClass({ Lookup Tables } + {this.isPermitted(this.props.permissions, ['inputs:create']) && + + Pipelines + + } {pluginSystemNavigations} diff --git a/graylog2-web-interface/src/components/pipelines/NewPipeline.jsx b/graylog2-web-interface/src/components/pipelines/NewPipeline.jsx new file mode 100644 index 000000000000..2c911a692178 --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/NewPipeline.jsx @@ -0,0 +1,41 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Row, Col } from 'react-bootstrap'; + +import history from 'util/History'; +import PipelineDetails from './PipelineDetails'; + +import Routes from 'routing/Routes'; + +const NewPipeline = React.createClass({ + propTypes: { + onChange: PropTypes.func.isRequired, + }, + + _onChange(newPipeline) { + this.props.onChange(newPipeline, this._goToPipeline); + }, + + _goToPipeline(pipeline) { + history.push(Routes.SYSTEM.PIPELINES.PIPELINE(pipeline.id)); + }, + + _goBack() { + history.goBack(); + }, + + render() { + return ( + + +

+ Give a name and description to the new pipeline. You can add stages to it when you save the changes. +

+ + + + ); + }, +}); + +export default NewPipeline; diff --git a/graylog2-web-interface/src/components/pipelines/Pipeline.css b/graylog2-web-interface/src/components/pipelines/Pipeline.css new file mode 100644 index 000000000000..c1038d318aab --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/Pipeline.css @@ -0,0 +1,28 @@ +.pipeline-dl { + margin-bottom: 0; +} + +dl.pipeline-dl > dt { + text-align: left; + width: 140px; +} + +dl.pipeline-dl > dt:after { + content: ':'; +} + +dl.pipeline-dl > dd { + margin-left: 100px; +} + +.row-margin-top { + margin-top: 10px; +} + +.description-margin-top { + margin-top: 5px; +} + +.pipeline-no-connections-warning { + margin-bottom: 13px; +} diff --git a/graylog2-web-interface/src/components/pipelines/Pipeline.jsx b/graylog2-web-interface/src/components/pipelines/Pipeline.jsx new file mode 100644 index 000000000000..8f018397a2de --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/Pipeline.jsx @@ -0,0 +1,134 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Button, Row, Col, Alert } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { EntityList, Pluralize } from 'components/common'; +import Stage from './Stage'; +import StageForm from './StageForm'; +import PipelineDetails from './PipelineDetails'; +import PipelineConnectionsForm from './PipelineConnectionsForm'; +import PipelineConnectionsList from './PipelineConnectionsList'; + +import Routes from 'routing/Routes'; + +const Pipeline = React.createClass({ + propTypes: { + pipeline: PropTypes.object.isRequired, + connections: PropTypes.array.isRequired, + streams: PropTypes.array.isRequired, + onConnectionsChange: PropTypes.func.isRequired, + onStagesChange: PropTypes.func.isRequired, + onPipelineChange: PropTypes.func.isRequired, + }, + + componentDidMount() { + this.style.use(); + }, + + componentWillUnmount() { + this.style.unuse(); + }, + + style: require('!style/useable!css!./Pipeline.css'), + + _connections_warning() { + if(this.props.connections.length == 0) { + return ( + + This pipeline is currently not connected to any streams. You have to connect a pipeline to at least one + stream to make it process incoming messages. Note that this is not required if you intend to use this + pipeline only for search result transformation using decorators. + + ); + } + }, + + _saveStage(stage, callback) { + const newStages = this.props.pipeline.stages.slice(); + newStages.push(stage); + this.props.onStagesChange(newStages, callback); + }, + + _updateStage(prevStage) { + return (stage, callback) => { + const newStages = this.props.pipeline.stages.filter(s => s.stage !== prevStage.stage); + newStages.push(stage); + this.props.onStagesChange(newStages, callback); + }; + }, + + _deleteStage(stage) { + return () => { + if (confirm(`You are about to delete stage ${stage.stage}, are you sure you want to proceed?`)) { + const newStages = this.props.pipeline.stages.filter(s => s.stage !== stage.stage); + this.props.onStagesChange(newStages); + } + }; + }, + + _formatConnectedStreams(streams) { + const formattedStreams = streams.map(s => `"${s.title}"`); + const streamList = streams.length > 1 ? [formattedStreams.slice(0, -1).join(', '), formattedStreams.slice(-1)].join(' and ') : formattedStreams[0]; + return ( + + This pipeline is processing messages from the{' '} + {' '} + {streamList}. + + ); + }, + + _formatStage(stage, maxStage) { + return ( + + ); + }, + + render() { + const pipeline = this.props.pipeline; + + const maxStage = pipeline.stages.reduce((max, currentStage) => Math.max(max, currentStage.stage), -Infinity); + const formattedStages = pipeline.stages + .sort((s1, s2) => s1.stage - s2.stage) + .map(stage => this._formatStage(stage, maxStage)); + + return ( +
+ {this._connections_warning()} + + + +
+ +
+

Pipeline connections

+

+ +

+
+ + + + +
+ +
+

Pipeline Stages

+

+ Stages are groups of conditions and actions which need to run in order, and provide the necessary{' '} + control flow to decide whether or not to run the rest of a pipeline. +

+ + + +
+ ); + }, +}); + +export default Pipeline; diff --git a/graylog2-web-interface/src/components/pipelines/PipelineConnectionsForm.jsx b/graylog2-web-interface/src/components/pipelines/PipelineConnectionsForm.jsx new file mode 100644 index 000000000000..c45e29a847c7 --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/PipelineConnectionsForm.jsx @@ -0,0 +1,106 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Button, ControlLabel, FormGroup, HelpBlock } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; +import naturalSort from 'javascript-natural-sort'; + +import { SelectableList } from 'components/common'; +import BootstrapModalForm from 'components/bootstrap/BootstrapModalForm'; + +import Routes from 'routing/Routes'; + +const PipelineConnectionsForm = React.createClass({ + propTypes: { + pipeline: PropTypes.object.isRequired, + connections: PropTypes.array.isRequired, + streams: PropTypes.array.isRequired, + save: PropTypes.func.isRequired, + }, + + getInitialState() { + return { + connectedStreams: this._getFormattedStreams(this._getConnectedStreams(this.props.pipeline, this.props.connections, this.props.streams)), + }; + }, + + openModal() { + this.refs.modal.open(); + }, + + _onStreamsChange(newStreams) { + this.setState({ connectedStreams: newStreams.sort((s1, s2) => naturalSort(s1.label, s2.label)) }); + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _resetForm() { + this.setState(this.getInitialState()); + }, + + _saved() { + this._closeModal(); + }, + + _save() { + const streamIds = this.state.connectedStreams.map(cs => cs.value); + const newConnection = { + pipeline: this.props.pipeline.id, + streams: streamIds, + }; + + this.props.save(newConnection, this._saved); + }, + + _getConnectedStreams(pipeline, connections, streams) { + return connections + .filter(c => c.pipeline_ids && c.pipeline_ids.includes(pipeline.id)) // Get connections for this pipeline + .filter(c => streams.some(s => s.id === c.stream_id)) // Filter out deleted streams + .map(c => this.props.streams.find(s => s.id === c.stream_id)); + }, + + _getFormattedStreams(streams) { + return streams + .map(s => { + return { value: s.id, label: s.title }; + }) + .sort((s1, s2) => naturalSort(s1.label, s2.label)); + }, + + _getFilteredStreams(streams) { + return streams.filter(s => !this.state.connectedStreams.some(cs => cs.value.toLowerCase() === s.id.toLowerCase())); + }, + + render() { + const streamsHelp = ( + + Select the streams you want to connect to this pipeline, or create one in the{' '} + Streams page. + + ); + + return ( + + + Edit connections for {this.props.pipeline.title}} + onSubmitForm={this._save} onCancel={this._resetForm} submitButtonText="Save"> +
+ + Streams + + {streamsHelp} + +
+ + + ); + }, +}); + +export default PipelineConnectionsForm; diff --git a/graylog2-web-interface/src/components/pipelines/PipelineConnectionsList.jsx b/graylog2-web-interface/src/components/pipelines/PipelineConnectionsList.jsx new file mode 100644 index 000000000000..2d19ffb5a831 --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/PipelineConnectionsList.jsx @@ -0,0 +1,35 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import naturalSort from 'javascript-natural-sort'; + +const PipelineConnectionsList = React.createClass({ + propTypes: { + pipeline: PropTypes.object.isRequired, + connections: PropTypes.array.isRequired, + streams: PropTypes.array.isRequired, + streamsFormatter: PropTypes.func.isRequired, + noConnectionsMessage: PropTypes.any, + }, + + getDefaultProps() { + return { + noConnectionsMessage: 'Pipeline not connected to any streams', + }; + }, + + render() { + const streamsUsingPipeline = this.props.connections + .filter(c => c.pipeline_ids && c.pipeline_ids.includes(this.props.pipeline.id)) // Get connections for this pipeline + .filter(c => this.props.streams.some(s => s.id === c.stream_id)) // Filter out deleted streams + .map(c => this.props.streams.find(s => s.id === c.stream_id)) + .sort((s1, s2) => naturalSort(s1.title, s2.title)); + + return ( + + {streamsUsingPipeline.length === 0 ? this.props.noConnectionsMessage : this.props.streamsFormatter(streamsUsingPipeline)} + + ); + }, +}); + +export default PipelineConnectionsList; diff --git a/graylog2-web-interface/src/components/pipelines/PipelineDetails.jsx b/graylog2-web-interface/src/components/pipelines/PipelineDetails.jsx new file mode 100644 index 000000000000..4b170bd02d7e --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/PipelineDetails.jsx @@ -0,0 +1,56 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Row, Col } from 'react-bootstrap'; + +import { Timestamp } from 'components/common'; +import PipelineForm from './PipelineForm'; + +import { MetricContainer, CounterRate } from 'components/metrics'; + +const PipelineDetails = React.createClass({ + propTypes: { + pipeline: PropTypes.object, + create: PropTypes.bool, + onChange: PropTypes.func.isRequired, + onCancel: PropTypes.func, + }, + + render() { + if (this.props.create) { + return ; + } + + const pipeline = this.props.pipeline; + return ( +
+ + +
+ +
+

Details

+
+
Title
+
{pipeline.title}
+
Description
+
{pipeline.description}
+
Created
+
+
Last modified
+
+
Current throughput
+
+ + + +
+
+ + +
+
+ ); + }, +}); + +export default PipelineDetails; diff --git a/graylog2-web-interface/src/components/pipelines/PipelineForm.jsx b/graylog2-web-interface/src/components/pipelines/PipelineForm.jsx new file mode 100644 index 000000000000..cb2e712995fa --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/PipelineForm.jsx @@ -0,0 +1,137 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Row, Col, Button } from 'react-bootstrap'; + +import { BootstrapModalForm, Input } from 'components/bootstrap'; +import ObjectUtils from 'util/ObjectUtils'; +import FormsUtils from 'util/FormsUtils'; + +const PipelineForm = React.createClass({ + propTypes: { + pipeline: PropTypes.object, + create: PropTypes.bool, + modal: PropTypes.bool, + save: PropTypes.func.isRequired, + validatePipeline: PropTypes.func.isRequired, + onCancel: PropTypes.func, + }, + + getDefaultProps() { + return { + modal: true, + pipeline: { + id: undefined, + title: '', + description: '', + stages: [{ stage: 0, rules: [] }], + }, + }; + }, + + getInitialState() { + const pipeline = ObjectUtils.clone(this.props.pipeline); + return { + // when editing, take the pipeline that's been passed in + pipeline: { + id: pipeline.id, + title: pipeline.title, + description: pipeline.description, + stages: pipeline.stages, + }, + }; + }, + + openModal() { + this.refs.modal.open(); + }, + + _onChange(event) { + const pipeline = ObjectUtils.clone(this.state.pipeline); + pipeline[event.target.name] = FormsUtils.getValueFromInput(event.target); + this.setState({ pipeline }); + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _saved() { + if (this.props.modal) { + this._closeModal(); + } + + if (this.props.create) { + this.setState(this.getInitialState()); + } + }, + + _save(event) { + if (event) { + event.preventDefault(); + } + + this.props.save(this.state.pipeline, this._saved); + }, + + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Add new pipeline'; + } else { + triggerButtonContent = 'Edit pipeline details'; + } + + const content = ( +
+ + + +
+ ); + + if (this.props.modal) { + return ( + + + + {content} + + + ); + } + + return ( +
+ {content} + + + + + + +
+ ); + }, +}); + +export default PipelineForm; diff --git a/graylog2-web-interface/src/components/pipelines/ProcessingTimelineComponent.css b/graylog2-web-interface/src/components/pipelines/ProcessingTimelineComponent.css new file mode 100644 index 000000000000..592b4d322aa1 --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/ProcessingTimelineComponent.css @@ -0,0 +1,32 @@ +.pipeline-stage { + border: 1px solid #666; + border-radius: 4px; + display: inline-block; + margin-right: 15px; + padding: 20px; + text-align: center; + width: 120px; +} + +.pipeline-stage.idle-stage { + background-color: #E3E5E5; + border-color: #D0D4D4; +} + +.pipeline-stage.used-stage { + background-color: #FFFFFF; +} + +.pipeline-name { + max-width: 300px; + overflow-x: hidden; + text-overflow: ellipsis; + white-space: nowrap; + width: 300px; +} + +.stream-list { + max-width: 150px; + width: 150px; + word-wrap: break-word; +} \ No newline at end of file diff --git a/graylog2-web-interface/src/components/pipelines/ProcessingTimelineComponent.jsx b/graylog2-web-interface/src/components/pipelines/ProcessingTimelineComponent.jsx new file mode 100644 index 000000000000..d549d5ccb39c --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/ProcessingTimelineComponent.jsx @@ -0,0 +1,166 @@ +import React from 'react'; +import Reflux from 'reflux'; +import { Alert, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; +import naturalSort from 'javascript-natural-sort'; + +import { DataTable, Spinner } from 'components/common'; +import { MetricContainer, CounterRate } from 'components/metrics'; +import PipelineConnectionsList from './PipelineConnectionsList'; + +import PipelinesActions from 'actions/pipelines/PipelinesActions'; +import PipelinesStore from 'stores/pipelines/PipelinesStore'; +import PipelineConnectionsActions from 'actions/pipelines/PipelineConnectionsActions'; +import PipelineConnectionsStore from 'stores/pipelines/PipelineConnectionsStore'; + +import StoreProvider from 'injection/StoreProvider'; +const StreamsStore = StoreProvider.getStore('Streams'); + +import Routes from 'routing/Routes'; + +const ProcessingTimelineComponent = React.createClass({ + mixins: [Reflux.connect(PipelinesStore), Reflux.connect(PipelineConnectionsStore)], + + componentDidMount() { + this.style.use(); + PipelinesActions.list(); + PipelineConnectionsActions.list(); + + StreamsStore.listStreams().then((streams) => { + this.setState({ streams }); + }); + }, + + componentWillUnmount() { + this.style.unuse(); + }, + + style: require('!style/useable!css!./ProcessingTimelineComponent.css'), + + _calculateUsedStages(pipelines) { + return pipelines + .map(pipeline => pipeline.stages) + .reduce((usedStages, pipelineStages) => { + // Concat stages in a single array removing duplicates + return usedStages.concat(pipelineStages.map(stage => stage.stage).filter(stage => usedStages.indexOf(stage) === -1)); + }, []) + .sort(naturalSort); + }, + + _headerCellFormatter(header) { + let className; + if (header === 'Actions') { + className = 'actions'; + } + + return {header}; + }, + + _formatConnectedStreams(streams) { + return streams.map(s => s.title).join(', '); + }, + + _formatStages(pipeline, stages) { + const formattedStages = []; + const stageNumbers = stages.map(stage => stage.stage); + + this.usedStages.forEach(usedStage => { + if (stageNumbers.indexOf(usedStage) === -1) { + formattedStages.push( +
Idle
+ ); + } else { + formattedStages.push( +
Stage {usedStage}
+ ); + } + }, this); + + return formattedStages; + }, + + _pipelineFormatter(pipeline) { + return ( + + + {pipeline.title}
+ {pipeline.description} +
+ + + + + + Not connected} /> + + {this._formatStages(pipeline, pipeline.stages)} + + +   + + + + + + ); + }, + + _deletePipeline(pipeline) { + return () => { + if (confirm(`Do you really want to delete pipeline "${pipeline.title}"? This action cannot be undone.`)) { + PipelinesActions.delete(pipeline.id); + } + }; + }, + + _isLoading() { + return !this.state.pipelines || !this.state.streams || !this.state.connections; + }, + + render() { + if (this._isLoading()) { + return ; + } + + const addNewPipelineButton = ( +
+ + + +
+ ); + + if (this.state.pipelines.length === 0) { + return ( +
+ {addNewPipelineButton} + + There are no pipelines configured in your system. Create one to start processing your messages. + +
+ ); + } + + this.usedStages = this._calculateUsedStages(this.state.pipelines); + + const headers = ['Pipeline', 'Connected to Streams', 'Processing Timeline', 'Actions']; + return ( +
+ {addNewPipelineButton} + +
+ ); + }, +}); + +export default ProcessingTimelineComponent; diff --git a/graylog2-web-interface/src/components/pipelines/Stage.jsx b/graylog2-web-interface/src/components/pipelines/Stage.jsx new file mode 100644 index 000000000000..fda4676b78a5 --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/Stage.jsx @@ -0,0 +1,131 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import Reflux from 'reflux'; +import { Col, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { DataTable, EntityListItem, Spinner } from 'components/common'; +import RulesStore from 'stores/rules/RulesStore'; +import StageForm from './StageForm'; +import { MetricContainer, CounterRate } from 'components/metrics'; + +import Routes from 'routing/Routes'; + +const Stage = React.createClass({ + propTypes: { + stage: PropTypes.object.isRequired, + pipeline: PropTypes.object.isRequired, + isLastStage: PropTypes.bool, + onUpdate: PropTypes.func.isRequired, + onDelete: PropTypes.func.isRequired, + }, + mixins: [Reflux.connect(RulesStore)], + + _ruleHeaderFormatter(header) { + return {header}; + }, + + _ruleRowFormatter(stage, ruleArg, ruleIdx) { + let rule = ruleArg; + + let ruleTitle; + // this can happen when a rule has been renamed, but not all references are updated + if (!rule) { + rule = { + id: `invalid-${ruleIdx}`, + description: `Rule ${stage.rules[ruleIdx]} has been renamed or removed. This rule will be skipped.`, + }; + ruleTitle = {stage.rules[ruleIdx]}; + } else { + ruleTitle = ( + {rule.title} + + ); + + } + return ( + + + {ruleTitle} + + {rule.description} + + + + + + + + + + + + ); + }, + + _formatRules(stage, rules) { + const headers = ['Title', 'Description', 'Throughput', 'Errors']; + + return ( + this._ruleRowFormatter(stage, rule, i)} + noDataText="This stage has no rules yet. Click on edit to add some." + filterLabel="" + filterKeys={[]} /> + ); + }, + + render() { + const stage = this.props.stage; + + let suffix = `Contains ${(stage.rules.length === 1 ? '1 rule' : `${stage.rules.length} rules`)}`; + + const throughput = ( + + ); + + const actions = [ + , + , + ]; + + let description; + if (this.props.isLastStage) { + description = 'There are no further stages in this pipeline. Once rules in this stage are applied, the pipeline will have finished processing.'; + } else { + description = ( + + Messages satisfying {stage.match_all ? 'all rules' : 'at least one rule'}{' '} + in this stage, will continue to the next stage. + + ); + } + + let block = ( + {description} +
+ {throughput} + ); + let content; + // We check if we have the rules details before trying to render them + if (this.state.rules) { + content = this._formatRules(stage, this.props.stage.rules.map(name => this.state.rules.filter(r => r.title === name)[0])); + } else { + content = ; + } + + return ( + {content}} /> + ); + }, +}); + +export default Stage; diff --git a/graylog2-web-interface/src/components/pipelines/StageForm.jsx b/graylog2-web-interface/src/components/pipelines/StageForm.jsx new file mode 100644 index 000000000000..14b80b1f460a --- /dev/null +++ b/graylog2-web-interface/src/components/pipelines/StageForm.jsx @@ -0,0 +1,150 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import Reflux from 'reflux'; +import { Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { SelectableList } from 'components/common'; +import { BootstrapModalForm, Input } from 'components/bootstrap'; +import ObjectUtils from 'util/ObjectUtils'; +import FormsUtils from 'util/FormsUtils'; + +import RulesStore from 'stores/rules/RulesStore'; + +import Routes from 'routing/Routes'; + +const StageForm = React.createClass({ + propTypes: { + stage: PropTypes.object, + create: PropTypes.bool, + save: PropTypes.func.isRequired, + validateStage: PropTypes.func.isRequired, + }, + mixins: [Reflux.connect(RulesStore)], + + getDefaultProps() { + return { + stage: { + stage: 0, + match_all: false, + rules: [], + }, + }; + }, + + getInitialState() { + const stage = ObjectUtils.clone(this.props.stage); + return { + // when editing, take the stage that's been passed in + stage: { + stage: stage.stage, + match_all: stage.match_all, + rules: stage.rules, + }, + }; + }, + + openModal() { + this.refs.modal.open(); + }, + + _onChange(event) { + const stage = ObjectUtils.clone(this.state.stage); + stage[event.target.name] = FormsUtils.getValueFromInput(event.target); + this.setState({ stage }); + }, + + _onRulesChange(newRules) { + const stage = ObjectUtils.clone(this.state.stage); + stage.rules = newRules; + this.setState({ stage }); + }, + + _closeModal() { + this.refs.modal.close(); + }, + + _saved() { + this._closeModal(); + if (this.props.create) { + this.setState(this.getInitialState()); + } + }, + + _save() { + this.props.save(this.state.stage, this._saved); + }, + + _getFormattedOptions(rules) { + return rules ? rules.map(rule => { + return { value: rule.title, label: rule.title }; + }) : []; + }, + + render() { + let triggerButtonContent; + if (this.props.create) { + triggerButtonContent = 'Add new stage'; + } else { + triggerButtonContent = Edit; + } + + const rulesHelp = ( + + Select the rules evaluated on this stage, or create one in the{' '} + Pipeline Rules page. + + ); + + return ( + + + +
+ + + + + + + + + + + +
+ + + ); + }, +}); + +export default StageForm; diff --git a/graylog2-web-interface/src/components/rules/Rule.jsx b/graylog2-web-interface/src/components/rules/Rule.jsx new file mode 100644 index 000000000000..47b7557fefd7 --- /dev/null +++ b/graylog2-web-interface/src/components/rules/Rule.jsx @@ -0,0 +1,76 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Row, Col, Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { PageHeader } from 'components/common'; +import DocumentationLink from 'components/support/DocumentationLink'; + +import DocsHelper from 'util/DocsHelper'; + +import RuleForm from './RuleForm'; +import RuleHelper from './RuleHelper'; + +import Routes from 'routing/Routes'; + +const Rule = React.createClass({ + propTypes: { + rule: PropTypes.object, + usedInPipelines: PropTypes.array, + create: PropTypes.bool, + onSave: PropTypes.func.isRequired, + validateRule: PropTypes.func.isRequired, + }, + + render() { + let title; + if (this.props.create) { + title = 'Create pipeline rule'; + } else { + title = Pipeline rule {this.props.rule.title}; + } + + return ( +
+ + + Rules are a way of applying changes to messages in Graylog. A rule consists of a condition and a list{' '} + of actions.{' '} + Graylog evaluates the condition against a message and executes the actions if the condition is satisfied. + + + + Read more about Graylog pipeline rules in the . + + + + + + +   + + + +   + + + + + + + + + + + + + + +
+ ); + }, +}); + +export default Rule; diff --git a/graylog2-web-interface/src/components/rules/RuleForm.css b/graylog2-web-interface/src/components/rules/RuleForm.css new file mode 100644 index 000000000000..bbb8a8782e37 --- /dev/null +++ b/graylog2-web-interface/src/components/rules/RuleForm.css @@ -0,0 +1,17 @@ +:local(.usedInPipelines) { + margin: 0; + padding: 0; +} + +:local(.usedInPipelines li:not(:last-child)) { + float: left; +} + +:local(.usedInPipelines li:not(:last-child):after) { + content: ','; + margin-right: 5px; +} + +:local(.usedInPipelines li:last-child:after) { + content: '.'; +} \ No newline at end of file diff --git a/graylog2-web-interface/src/components/rules/RuleForm.jsx b/graylog2-web-interface/src/components/rules/RuleForm.jsx new file mode 100644 index 000000000000..76073ea9cff5 --- /dev/null +++ b/graylog2-web-interface/src/components/rules/RuleForm.jsx @@ -0,0 +1,185 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Button, Col, ControlLabel, FormControl, FormGroup, Row } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { SourceCodeEditor } from 'components/common'; +import { Input } from 'components/bootstrap'; +import Routes from 'routing/Routes'; + +import history from 'util/History'; + +import RuleFormStyle from './RuleForm.css'; + +const RuleForm = React.createClass({ + propTypes: { + rule: PropTypes.object, + usedInPipelines: PropTypes.array, + create: PropTypes.bool, + onSave: PropTypes.func.isRequired, + validateRule: PropTypes.func.isRequired, + }, + + getDefaultProps() { + return { + rule: { + id: '', + title: '', + description: '', + source: '', + }, + }; + }, + + getInitialState() { + const rule = this.props.rule; + return { + // when editing, take the rule that's been passed in + rule: { + id: rule.id, + title: rule.title, + description: rule.description, + source: rule.source, + }, + parseErrors: [], + }; + }, + + componentWillUnmount() { + if (this.parseTimer !== undefined) { + clearTimeout(this.parseTimer); + this.parseTimer = undefined; + } + }, + + parseTimer: undefined, + + _setParseErrors(errors) { + this.setState({ parseErrors: errors }); + }, + + _onSourceChange(value) { + // don't try to parse the previous value, gets reset below + if (this.parseTimer !== undefined) { + clearTimeout(this.parseTimer); + } + const rule = this.state.rule; + rule.source = value; + this.setState({ rule }); + + if (this.props.validateRule) { + // have the caller validate the rule after typing stopped for a while. usually this will mean send to server to parse + this.parseTimer = setTimeout(() => this.props.validateRule(rule, this._setParseErrors), 500); + } + }, + + _onDescriptionChange(event) { + const rule = this.state.rule; + rule.description = event.target.value; + this.setState({ rule }); + }, + + _onTitleChange(event) { + const rule = this.state.rule; + rule.title = event.target.value; + this.setState({ rule }); + }, + + _getId(prefixIdName) { + return this.state.name !== undefined ? prefixIdName + this.state.name : prefixIdName; + }, + + _goBack() { + history.goBack(); + }, + + _saved() { + history.push(Routes.SYSTEM.PIPELINES.RULES); + }, + + _save() { + if (this.state.parseErrors.length === 0) { + this.props.onSave(this.state.rule, this._saved); + } + }, + + _submit(event) { + event.preventDefault(); + this._save(); + }, + + _formatPipelinesUsingRule() { + if (this.props.usedInPipelines.length === 0) { + return 'This rule is not being used in any pipelines.'; + } + + const formattedPipelines = this.props.usedInPipelines.map(pipeline => { + return ( +
  • + + {pipeline.title} + +
    • + ); + }); + + return
      {formattedPipelines}
    ; + }, + + render() { + let pipelinesUsingRule; + if (!this.props.create) { + pipelinesUsingRule = ( + +
    + {this._formatPipelinesUsingRule()} +
    + + ); + } + + const annotations = this.state.parseErrors.map(e => { + return { row: e.line - 1, column: e.position_in_line - 1, text: e.reason, type: 'error' }; + }); + + return ( +
    +
    + + Title + You can set the rule title in the rule source. See the quick reference for more information. + + + + + {pipelinesUsingRule} + + + + +
    + + + +
    + + +
    + +     +
    + ); + }, +}); + +export default RuleForm; diff --git a/graylog2-web-interface/src/components/rules/RuleHelper.css b/graylog2-web-interface/src/components/rules/RuleHelper.css new file mode 100644 index 000000000000..cca21818a08f --- /dev/null +++ b/graylog2-web-interface/src/components/rules/RuleHelper.css @@ -0,0 +1,23 @@ +:local(.clickableRow) { + cursor: pointer; +} + +:local(.functionTableCell) { + width: 300px; +} + +:local(.marginQuickReferenceText) { + margin-top: 5px; +} + +:local(.marginTab) { + margin-top: 10px; +} + +:local(.exampleFunction) { + white-space: pre-wrap; +} + +:local(.adjustedTableCellWidth) { + width: 1%; +} \ No newline at end of file diff --git a/graylog2-web-interface/src/components/rules/RuleHelper.jsx b/graylog2-web-interface/src/components/rules/RuleHelper.jsx new file mode 100644 index 000000000000..d989c0f2cfee --- /dev/null +++ b/graylog2-web-interface/src/components/rules/RuleHelper.jsx @@ -0,0 +1,165 @@ +import React from 'react'; +import { Row, Col, Panel, Table, Tabs, Tab } from 'react-bootstrap'; + +import Reflux from 'reflux'; + +import RulesStore from 'stores/rules/RulesStore'; +import RulesActions from 'actions/rules/RulesActions'; +import ObjectUtils from 'util/ObjectUtils'; + +import DocumentationLink from 'components/support/DocumentationLink'; +import { PaginatedList, Spinner } from 'components/common'; + +import DocsHelper from 'util/DocsHelper'; + +import RuleHelperStyle from './RuleHelper.css'; + +const RuleHelper = React.createClass({ + mixins: [ + Reflux.connect(RulesStore), + ], + + getInitialState() { + return { + expanded: {}, + paginatedEntries: undefined, + filteredEntries: undefined, + currentPage: 1, + pageSize: 10, + }; + }, + + componentDidMount() { + RulesActions.loadFunctions(); + }, + + ruleTemplate: `rule "function howto" +when + has_field("transaction_date") +then + // the following date format assumes there's no time zone in the string + let new_date = parse_date(to_string($message.transaction_date), "yyyy-MM-dd HH:mm:ss"); + set_field("transaction_year", new_date.year); +end`, + + _niceType(typeName) { + return typeName.replace(/^.*\.(.*?)$/, '$1'); + }, + + _toggleFunctionDetail(functionName) { + const newState = ObjectUtils.clone(this.state.expanded); + newState[functionName] = !newState[functionName]; + this.setState({ expanded: newState }); + }, + + _functionSignature(descriptor) { + const args = descriptor.params.map(p => { return p.optional ? `[${p.name}]` : p.name; }); + return {`${descriptor.name}(${args.join(', ')}) : ${this._niceType(descriptor.return_type)}`}; + }, + + _parameters(descriptor) { + return descriptor.params.map(p => { + return ( + + {p.name} + {this._niceType(p.type)} + {p.optional ? null : } + {p.description} + ); + }); + }, + + _renderFunctions(descriptors) { + if (!descriptors) return []; + return descriptors.map((d) => { + let details = null; + if (this.state.expanded[d.name]) { + details = ( + + + + + + + + + + + + {this._parameters(d)} + +
    ParameterTypeRequiredDescription
    + + ); + } + return ( + this._toggleFunctionDetail(d.name)} className={RuleHelperStyle.clickableRow}> + {this._functionSignature(d)} + {d.description} + + {details} + ); + }); + }, + + _onPageChange(newPage, pageSize) { + this.setState({ currentPage: newPage, pageSize: pageSize }); + }, + + render() { + if (!this.state.functionDescriptors) { + return ; + } + + const pagedEntries = this.state.functionDescriptors.slice((this.state.currentPage - 1) * this.state.pageSize, this.state.currentPage * this.state.pageSize); + + return ( + + + +

    + Read the {' '} + to gain a better understanding of how Graylog pipeline rules work. +

    + +     + + + + +

    + This is a list of all available functions in pipeline rules. Click on a row to see more information + about the function parameters. +

    +
    + + + + + + + + + {this._renderFunctions(pagedEntries)} +
    FunctionDescription
    +     +
    +     + +

    + Do you want to see how a pipeline rule looks like? Take a look at this example: +

    + 
    +                  {this.ruleTemplate}
+
    +     +     + +     +     + ); + }, +}); + +export default RuleHelper; diff --git a/graylog2-web-interface/src/components/rules/RuleList.jsx b/graylog2-web-interface/src/components/rules/RuleList.jsx new file mode 100644 index 000000000000..ebc26a16fbfb --- /dev/null +++ b/graylog2-web-interface/src/components/rules/RuleList.jsx @@ -0,0 +1,93 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { DataTable, Timestamp } from 'components/common'; + +import { Button } from 'react-bootstrap'; +import { LinkContainer } from 'react-router-bootstrap'; + +import RulesActions from 'actions/rules/RulesActions'; + +import { MetricContainer, CounterRate } from 'components/metrics'; + +import Routes from 'routing/Routes'; + +const RuleList = React.createClass({ + propTypes: { + rules: PropTypes.array.isRequired, + }, + + _delete(rule) { + return () => { + if (window.confirm(`Do you really want to delete rule "${rule.title}"?`)) { + RulesActions.delete(rule); + } + }; + }, + + _headerCellFormatter(header) { + return {header}; + }, + + _ruleInfoFormatter(rule) { + const actions = [ + , +  , + + + , + ]; + + return ( + + + + {rule.title} + + + {rule.description} + + + + + + + + + + + + + {actions} + + ); + }, + render() { + const filterKeys = ['title', 'description']; + const headers = ['Title', 'Description', 'Created', 'Last modified', 'Throughput', 'Errors', 'Actions']; + + return ( +
    + +
    + + + +
    +     +
    + ); + }, +}); + +export default RuleList; diff --git a/graylog2-web-interface/src/components/rules/RulesComponent.jsx b/graylog2-web-interface/src/components/rules/RulesComponent.jsx new file mode 100644 index 000000000000..6393105dc2ff --- /dev/null +++ b/graylog2-web-interface/src/components/rules/RulesComponent.jsx @@ -0,0 +1,25 @@ +import PropTypes from 'prop-types'; +import React from 'react'; + +import { Spinner } from 'components/common'; +import RuleList from './RuleList'; + +const RulesComponent = React.createClass({ + propTypes: { + rules: PropTypes.array, + }, + + render() { + if (!this.props.rules) { + return ; + } + + return ( +
    + +
    + ); + }, +}); + +export default RulesComponent; \ No newline at end of file diff --git a/graylog2-web-interface/src/components/simulator/ProcessorSimulator.jsx b/graylog2-web-interface/src/components/simulator/ProcessorSimulator.jsx new file mode 100644 index 000000000000..1efaefe46bbd --- /dev/null +++ b/graylog2-web-interface/src/components/simulator/ProcessorSimulator.jsx @@ -0,0 +1,128 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import { Col, ControlLabel, FormGroup, HelpBlock, Panel, Row } from 'react-bootstrap'; +import naturalSort from 'javascript-natural-sort'; +import { LinkContainer } from 'react-router-bootstrap'; + +import { Select } from 'components/common'; +import RawMessageLoader from 'components/messageloaders/RawMessageLoader'; +import SimulationResults from './SimulationResults'; + +import Routes from 'routing/Routes'; + +import SimulatorActions from 'actions/simulator/SimulatorActions'; +// eslint-disable-next-line no-unused-vars +import SimulatorStore from 'stores/simulator/SimulatorStore'; + +const DEFAULT_STREAM_ID = '000000000000000000000001'; + +const ProcessorSimulator = React.createClass({ + propTypes: { + streams: PropTypes.array.isRequired, + }, + + getInitialState() { + // The default stream could not be present in a system. In that case we fallback to the first available stream. + this.defaultStream = this.props.streams.find(s => s.id === DEFAULT_STREAM_ID) || this.props.streams[0]; + + return { + message: undefined, + stream: this.defaultStream, + simulation: undefined, + loading: false, + error: undefined, + }; + }, + + _onMessageLoad(message, options) { + this.setState({ message: message, simulation: undefined, loading: true, error: undefined }); + + SimulatorActions.simulate + .triggerPromise(this.state.stream, message.fields, options.inputId) + .then( + response => { + this.setState({ simulation: response, loading: false }); + }, + error => { + this.setState({ loading: false, error: error }); + } + ); + }, + + _getFormattedStreams(streams) { + if (!streams) { + return []; + } + + return streams + .map(s => { + return { value: s.id, label: s.title }; + }) + .sort((s1, s2) => naturalSort(s1.label, s2.label)); + }, + + _onStreamSelect(selectedStream) { + const stream = this.props.streams.find(s => s.id.toLowerCase() === selectedStream.toLowerCase()); + this.setState({ stream: stream }); + }, + + render() { + if (this.props.streams.length === 0) { + return ( +
    + + + + Pipelines operate on streams, but your system currently has no streams. Please{' '} + create a stream{' '} + and come back here later to test pipelines processing messages in your new stream. + + + +
    + ); + } + + const streamHelp = ( + + Select a stream to use during simulation, the {this.defaultStream.title} stream is used by default. + + ); + + return ( +
    + + +

    Load a message

    +

    + Build an example message that will be used in the simulation.{' '} + No real messages stored in Graylog will be changed. All actions are purely simulated on the + temporary input you provide below. +

    + + + + Stream +