From a966c1bf1be5a308e7f947529ff8d9a536206238 Mon Sep 17 00:00:00 2001 From: Jochen Schalanda Date: Mon, 7 May 2018 17:29:17 +0200 Subject: [PATCH] Add support for exporting pipelines and rules Closes #2507 Closes #4222 --- .../pipelineprocessor/db/PipelineService.java | 2 + .../db/PipelineStreamConnectionsService.java | 2 + .../pipelineprocessor/db/RuleService.java | 2 + .../db/memory/InMemoryPipelineService.java | 9 + ...emoryPipelineStreamConnectionsService.java | 12 +- .../db/memory/InMemoryRuleService.java | 9 + .../db/mongodb/MongoDbPipelineService.java | 11 ++ ...ngoDbPipelineStreamConnectionsService.java | 15 +- .../db/mongodb/MongoDbRuleService.java | 10 ++ .../contentpacks/ContentPacksModule.java | 4 + .../catalogs/PipelineCatalog.java | 124 ++++++++++++++ .../catalogs/PipelineRuleCatalog.java | 70 ++++++++ .../contentpacks/codecs/PipelineCodec.java | 109 ++++++++++++ .../codecs/PipelineRuleCodec.java | 90 ++++++++++ .../contentpacks/model/ModelTypes.java | 2 + .../model/entities/PipelineEntity.java | 53 ++++++ .../model/entities/PipelineRuleEntity.java | 47 ++++++ .../db/memory/InMemoryRuleServiceTest.java | 16 ++ .../catalogs/PipelineCatalogTest.java | 155 ++++++++++++++++++ .../catalogs/PipelineRuleCatalogTest.java | 100 +++++++++++ .../codecs/PipelineCodecTest.java | 99 +++++++++++ .../codecs/PipelineRuleCodecTest.java | 88 ++++++++++ .../pipeline_processor_pipelines.json | 15 +- .../pipeline_processor_pipelines_streams.json | 13 -- .../pipeline_processor_rules.json | 14 ++ 25 files changed, 1053 insertions(+), 18 deletions(-) create mode 100644 graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineCatalog.java create mode 100644 graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalog.java create mode 100644 graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineCodec.java create mode 100644 graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineRuleCodec.java create mode 100644 graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineEntity.java create mode 100644 graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineRuleEntity.java create mode 100644 graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineCatalogTest.java create mode 100644 graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalogTest.java create mode 100644 graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineCodecTest.java create mode 100644 graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineRuleCodecTest.java delete mode 100644 graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines_streams.json diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java index e632691bf46f..437d6094901d 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java @@ -25,6 +25,8 @@ public interface PipelineService { PipelineDao load(String id) throws NotFoundException; + PipelineDao loadByName(String name) throws NotFoundException; + Collection loadAll(); void delete(String id); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java index b65502434b75..938f136c695e 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java @@ -28,5 +28,7 @@ public interface PipelineStreamConnectionsService { Set loadAll(); + Set loadByPipelineId(String pipelineId); + void delete(String streamId); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java index 168286de229a..f441bd947326 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java @@ -25,6 +25,8 @@ public interface RuleService { RuleDao load(String id) throws NotFoundException; + RuleDao loadByName(String name) throws NotFoundException; + Collection loadAll(); void delete(String id); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java index 9b2c9b26c25f..6eed39725516 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineService.java @@ -64,6 +64,15 @@ public PipelineDao load(String id) throws NotFoundException { return pipeline; } + @Override + public PipelineDao loadByName(String name) throws NotFoundException { + final String id = titleToId.get(name); + if (id == null) { + throw new NotFoundException("No pipeline with name " + name); + } + return load(id); + } + @Override public Collection loadAll() { return ImmutableSet.copyOf(store.values()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java index 524d023e2927..6444e714bc26 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java @@ -17,21 +17,22 @@ package org.graylog.plugins.pipelineprocessor.db.memory; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.MapMaker; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; import org.graylog2.database.NotFoundException; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; public class InMemoryPipelineStreamConnectionsService implements PipelineStreamConnectionsService { // poor man's id generator private AtomicLong idGen = new AtomicLong(0); - private Map store = new MapMaker().makeMap(); + private Map store = new ConcurrentHashMap<>(); @Override public PipelineConnections save(PipelineConnections connections) { @@ -57,6 +58,13 @@ public Set loadAll() { return ImmutableSet.copyOf(store.values()); } + @Override + public Set loadByPipelineId(String pipelineId) { + return store.values().stream() + .filter(connection -> connection.pipelineIds().contains(pipelineId)) + .collect(Collectors.toSet()); + } + @Override public void delete(String streamId) { try { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java index 4ca7b41436d8..a828515615e7 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleService.java @@ -67,6 +67,15 @@ public RuleDao load(String id) throws NotFoundException { return rule; } + @Override + public RuleDao loadByName(String name) throws NotFoundException { + final String id = titleToId.get(name); + if (id == null) { + throw new NotFoundException("No rule with name " + name); + } + return load(id); + } + @Override public Collection loadAll() { return ImmutableSet.copyOf(store.values()); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java index 7c29eb853d8a..89f97e75d925 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineService.java @@ -25,6 +25,7 @@ import org.graylog2.database.MongoConnection; import org.graylog2.database.NotFoundException; import org.mongojack.DBCursor; +import org.mongojack.DBQuery; import org.mongojack.DBSort; import org.mongojack.JacksonDBCollection; import org.mongojack.WriteResult; @@ -67,6 +68,16 @@ public PipelineDao load(String id) throws NotFoundException { return pipeline; } + @Override + public PipelineDao loadByName(String name) throws NotFoundException { + final DBQuery.Query query = DBQuery.is("title", name); + final PipelineDao pipeline = dbCollection.findOne(query); + if (pipeline == null) { + throw new NotFoundException("No pipeline with name " + name); + } + return pipeline; + } + @Override public Collection loadAll() { try (DBCursor daos = dbCollection.find()) { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java index cfcee7130db5..ae6b4f866561 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbPipelineStreamConnectionsService.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableSet; import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; import com.mongodb.MongoException; import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; @@ -53,7 +54,6 @@ public MongoDbPipelineStreamConnectionsService(MongoConnection mongoConnection, dbCollection.createIndex(DBSort.asc("stream_id"), new BasicDBObject("unique", true)); } - @Override public PipelineConnections save(PipelineConnections connections) { PipelineConnections existingConnections = dbCollection.findOne(DBQuery.is("stream_id", connections.streamId())); @@ -86,6 +86,19 @@ public Set loadAll() { } } + @Override + public Set loadByPipelineId(String pipelineId) { + // Thanks, MongoJack! + // https://github.com/mongojack/mongojack/issues/12 + final DBObject query = new BasicDBObject("pipeline_ids", new BasicDBObject("$in", Collections.singleton(pipelineId))); + try (DBCursor pipelineConnections = dbCollection.find(query)) { + return ImmutableSet.copyOf((Iterable) pipelineConnections); + } catch (MongoException e) { + log.error("Unable to load pipeline connections for pipeline ID " + pipelineId, e); + return Collections.emptySet(); + } + } + @Override public void delete(String streamId) { try { diff --git a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java index 8cc1d5234984..6d11142cd072 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbRuleService.java @@ -71,6 +71,16 @@ public RuleDao load(String id) throws NotFoundException { return rule; } + @Override + public RuleDao loadByName(String name) throws NotFoundException { + final DBQuery.Query query = DBQuery.is("title", name); + final RuleDao rule = dbCollection.findOne(query); + if (rule == null) { + throw new NotFoundException("No rule with name " + name); + } + return rule; + } + @Override public Collection loadAll() { try(DBCursor ruleDaos = dbCollection.find().sort(DBSort.asc("title"))) { diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java index ffb76c3e0bd9..e46f1a290478 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/ContentPacksModule.java @@ -24,6 +24,8 @@ import org.graylog2.contentpacks.catalogs.LookupDataAdapterCatalog; import org.graylog2.contentpacks.catalogs.LookupTableCatalog; import org.graylog2.contentpacks.catalogs.OutputCatalog; +import org.graylog2.contentpacks.catalogs.PipelineCatalog; +import org.graylog2.contentpacks.catalogs.PipelineRuleCatalog; import org.graylog2.contentpacks.catalogs.StreamCatalog; import org.graylog2.contentpacks.jersey.ModelIdParamConverter; import org.graylog2.plugin.PluginModule; @@ -44,6 +46,8 @@ protected void configure() { addEntityCatalog(LookupDataAdapterCatalog.TYPE, LookupDataAdapterCatalog.class); addEntityCatalog(LookupTableCatalog.TYPE, LookupTableCatalog.class); addEntityCatalog(OutputCatalog.TYPE, OutputCatalog.class); + addEntityCatalog(PipelineCatalog.TYPE, PipelineCatalog.class); + addEntityCatalog(PipelineRuleCatalog.TYPE, PipelineRuleCatalog.class); addEntityCatalog(StreamCatalog.TYPE, StreamCatalog.class); } } diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineCatalog.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineCatalog.java new file mode 100644 index 000000000000..f9890ac90f9a --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineCatalog.java @@ -0,0 +1,124 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.catalogs; + +import com.google.common.graph.Graph; +import com.google.common.graph.GraphBuilder; +import com.google.common.graph.ImmutableGraph; +import com.google.common.graph.MutableGraph; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Rule; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.contentpacks.codecs.PipelineCodec; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelType; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityDescriptor; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.database.NotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class PipelineCatalog implements EntityCatalog { + private static final Logger LOG = LoggerFactory.getLogger(PipelineCatalog.class); + + public static final ModelType TYPE = ModelTypes.PIPELINE; + + private final PipelineService pipelineService; + private final PipelineStreamConnectionsService streamConnectionsService; + private final PipelineRuleParser pipelineRuleParser; + private final PipelineCodec codec; + + @Inject + public PipelineCatalog(PipelineService pipelineService, + PipelineStreamConnectionsService streamConnectionsService, + PipelineRuleParser pipelineRuleParser, + PipelineCodec codec) { + this.pipelineService = pipelineService; + this.streamConnectionsService = streamConnectionsService; + this.pipelineRuleParser = pipelineRuleParser; + this.codec = codec; + } + + @Override + public Set listEntityExcerpts() { + return pipelineService.loadAll().stream() + .map(codec::createExcerpt) + .collect(Collectors.toSet()); + } + + @Override + public Optional collectEntity(EntityDescriptor entityDescriptor) { + final ModelId modelId = entityDescriptor.id(); + try { + final PipelineDao pipelineDao = pipelineService.loadByName(modelId.id()); + return Optional.of(codec.encode(pipelineDao)); + } catch (NotFoundException e) { + LOG.debug("Couldn't find pipeline {}", entityDescriptor, e); + return Optional.empty(); + } + } + + @Override + public Graph resolve(EntityDescriptor entityDescriptor) { + final MutableGraph mutableGraph = GraphBuilder.directed().build(); + mutableGraph.addNode(entityDescriptor); + + final ModelId modelId = entityDescriptor.id(); + try { + final PipelineDao pipelineDao = pipelineService.loadByName(modelId.id()); + final String pipelineSource = pipelineDao.source(); + final Collection referencedRules = referencedRules(pipelineSource); + referencedRules.stream() + .map(ModelId::of) + .map(id -> EntityDescriptor.create(id, ModelTypes.PIPELINE_RULE)) + .forEach(output -> mutableGraph.putEdge(entityDescriptor, output)); + + final Set pipelineConnections = streamConnectionsService.loadByPipelineId(pipelineDao.id()); + pipelineConnections.stream() + .map(PipelineConnections::streamId) + .map(ModelId::of) + .map(id -> EntityDescriptor.create(id, ModelTypes.STREAM)) + .forEach(output -> mutableGraph.putEdge(entityDescriptor, output)); + } catch (NotFoundException e) { + LOG.debug("Couldn't find pipeline {}", entityDescriptor, e); + } + + return ImmutableGraph.copyOf(mutableGraph); + } + + private Collection referencedRules(String pipelineSource) { + final Pipeline pipeline = pipelineRuleParser.parsePipeline("dummy", pipelineSource); + return pipeline.stages().stream() + .map(Stage::getRules) + .flatMap(Collection::stream) + .map(Rule::name) + .collect(Collectors.toSet()); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalog.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalog.java new file mode 100644 index 000000000000..d4df7358e599 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalog.java @@ -0,0 +1,70 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.catalogs; + +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.contentpacks.codecs.PipelineRuleCodec; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelType; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityDescriptor; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.database.NotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class PipelineRuleCatalog implements EntityCatalog { + private static final Logger LOG = LoggerFactory.getLogger(PipelineRuleCatalog.class); + + public static final ModelType TYPE = ModelTypes.PIPELINE_RULE; + + private final RuleService ruleService; + private final PipelineRuleCodec codec; + + @Inject + public PipelineRuleCatalog(RuleService ruleService, + PipelineRuleCodec codec) { + this.ruleService = ruleService; + this.codec = codec; + } + + @Override + public Set listEntityExcerpts() { + return ruleService.loadAll().stream() + .map(codec::createExcerpt) + .collect(Collectors.toSet()); + } + + @Override + public Optional collectEntity(EntityDescriptor entityDescriptor) { + final ModelId modelId = entityDescriptor.id(); + try { + final RuleDao ruleDao = ruleService.loadByName(modelId.id()); + return Optional.of(codec.encode(ruleDao)); + } catch (NotFoundException e) { + LOG.debug("Couldn't find pipeline rule {}", entityDescriptor, e); + return Optional.empty(); + } + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineCodec.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineCodec.java new file mode 100644 index 000000000000..1d495fed3741 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineCodec.java @@ -0,0 +1,109 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.codecs; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.PipelineEntity; +import org.graylog2.plugin.Tools; +import org.joda.time.DateTime; + +import javax.inject.Inject; +import java.util.Set; +import java.util.stream.Collectors; + +public class PipelineCodec implements EntityCodec { + private final ObjectMapper objectMapper; + private final PipelineService pipelineService; + private final PipelineStreamConnectionsService connectionsService; + + @Inject + public PipelineCodec(ObjectMapper objectMapper, + PipelineService pipelineService, + PipelineStreamConnectionsService connectionsService) { + this.objectMapper = objectMapper; + this.pipelineService = pipelineService; + this.connectionsService = connectionsService; + } + + @Override + public Entity encode(PipelineDao pipelineDao) { + final Set connectedStreams = connectedStreams(pipelineDao.id()); + final PipelineEntity pipelineEntity = PipelineEntity.create( + pipelineDao.title(), + pipelineDao.description(), + pipelineDao.source(), + connectedStreams); + final JsonNode data = objectMapper.convertValue(pipelineEntity, JsonNode.class); + + return EntityV1.builder() + .id(ModelId.of(pipelineDao.title())) + .type(ModelTypes.PIPELINE) + .data(data) + .build(); + } + + private Set connectedStreams(String pipelineId) { + final Set connections = connectionsService.loadByPipelineId(pipelineId); + return connections.stream() + .map(PipelineConnections::streamId) + .collect(Collectors.toSet()); + } + + @Override + public PipelineDao decode(Entity entity) { + if (entity instanceof EntityV1) { + return decodeEntityV1((EntityV1) entity); + } else { + throw new IllegalArgumentException("Unsupported entity version: " + entity.getClass()); + } + } + + private PipelineDao decodeEntityV1(EntityV1 entity) { + final DateTime now = Tools.nowUTC(); + final PipelineEntity pipelineEntity = objectMapper.convertValue(entity.data(), PipelineEntity.class); + final PipelineDao pipelineDao = PipelineDao.builder() + .title(pipelineEntity.title()) + .description(pipelineEntity.description()) + .source(pipelineEntity.source()) + .createdAt(now) + .modifiedAt(now) + .build(); + + // TODO: Create pipeline-stream connections + + return pipelineService.save(pipelineDao); + } + + @Override + public EntityExcerpt createExcerpt(PipelineDao pipeline) { + return EntityExcerpt.builder() + .id(ModelId.of(pipeline.title())) + .type(ModelTypes.PIPELINE) + .title(pipeline.title()) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineRuleCodec.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineRuleCodec.java new file mode 100644 index 000000000000..d6b005f865d1 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/codecs/PipelineRuleCodec.java @@ -0,0 +1,90 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.codecs; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.PipelineRuleEntity; +import org.graylog2.plugin.Tools; +import org.joda.time.DateTime; + +import javax.inject.Inject; + +public class PipelineRuleCodec implements EntityCodec { + private final ObjectMapper objectMapper; + private final RuleService ruleService; + + @Inject + public PipelineRuleCodec(ObjectMapper objectMapper, RuleService ruleService) { + this.objectMapper = objectMapper; + this.ruleService = ruleService; + } + + @Override + public Entity encode(RuleDao ruleDao) { + final PipelineRuleEntity ruleEntity = PipelineRuleEntity.create( + ruleDao.title(), + ruleDao.description(), + ruleDao.source()); + final JsonNode data = objectMapper.convertValue(ruleEntity, JsonNode.class); + + return EntityV1.builder() + .id(ModelId.of(ruleDao.title())) + .type(ModelTypes.PIPELINE_RULE) + .data(data) + .build(); + } + + @Override + public RuleDao decode(Entity entity) { + if (entity instanceof EntityV1) { + return decodeEntityV1((EntityV1) entity); + } else { + throw new IllegalArgumentException("Unsupported entity version: " + entity.getClass()); + } + } + + private RuleDao decodeEntityV1(EntityV1 entity) { + final DateTime now = Tools.nowUTC(); + final PipelineRuleEntity ruleEntity = objectMapper.convertValue(entity.data(), PipelineRuleEntity.class); + final RuleDao ruleDao = RuleDao.builder() + .title(ruleEntity.title()) + .description(ruleEntity.description()) + .source(ruleEntity.source()) + .createdAt(now) + .modifiedAt(now) + .build(); + + return ruleService.save(ruleDao); + } + + @Override + public EntityExcerpt createExcerpt(RuleDao ruleDao) { + return EntityExcerpt.builder() + .id(ModelId.of(ruleDao.title())) + .type(ModelTypes.PIPELINE_RULE) + .title(ruleDao.title()) + .build(); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java index d8cedee49acc..8f05bc2980ce 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/ModelTypes.java @@ -24,5 +24,7 @@ public interface ModelTypes { ModelType LOOKUP_TABLE = ModelType.of("lookup_table"); ModelType INPUT = ModelType.of("input"); ModelType OUTPUT = ModelType.of("output"); + ModelType PIPELINE = ModelType.of("pipeline"); + ModelType PIPELINE_RULE = ModelType.of("pipeline_rule"); ModelType STREAM = ModelType.of("stream"); } diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineEntity.java new file mode 100644 index 000000000000..92bd1e7620cc --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineEntity.java @@ -0,0 +1,53 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.model.entities; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog.autovalue.WithBeanGetter; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.Set; + +@JsonAutoDetect +@AutoValue +@WithBeanGetter +public abstract class PipelineEntity { + @JsonProperty("title") + public abstract String title(); + + @JsonProperty("description") + @Nullable + public abstract String description(); + + @JsonProperty("source") + public abstract String source(); + + @JsonProperty("connected_streams") + public abstract Set connectedStreams(); + + @JsonCreator + public static PipelineEntity create(@JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source, + @JsonProperty("connected_streams") Set connectedStreams) { + return new AutoValue_PipelineEntity(title, description, source, connectedStreams == null ? Collections.emptySet() : connectedStreams); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineRuleEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineRuleEntity.java new file mode 100644 index 000000000000..6e7c2211d7fa --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/PipelineRuleEntity.java @@ -0,0 +1,47 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.model.entities; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import org.graylog.autovalue.WithBeanGetter; + +import javax.annotation.Nullable; + +@JsonAutoDetect +@AutoValue +@WithBeanGetter +public abstract class PipelineRuleEntity { + @JsonProperty("title") + public abstract String title(); + + @JsonProperty("description") + @Nullable + public abstract String description(); + + @JsonProperty("source") + public abstract String source(); + + @JsonCreator + public static PipelineRuleEntity create(@JsonProperty("title") String title, + @JsonProperty("description") @Nullable String description, + @JsonProperty("source") String source) { + return new AutoValue_PipelineRuleEntity(title, description, source); + } +} diff --git a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java index 072e622a98e8..d420b1a1364a 100644 --- a/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java +++ b/graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryRuleServiceTest.java @@ -25,6 +25,7 @@ import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.fail; public class InMemoryRuleServiceTest { @@ -73,6 +74,21 @@ public void storeRetrieve() { } } + @Test + public void loadByName() throws NotFoundException { + RuleDao rule = RuleDao.create(null, "test", "description", "rule \"test\" when true then end", null, null); + final RuleDao savedRule = service.save(rule); + final RuleDao loadedRule = service.loadByName(savedRule.title()); + assertThat(loadedRule).isEqualTo(savedRule); + } + + @Test + public void loadByNameNotFound() { + assertThatThrownBy(() -> service.loadByName("Foobar")) + .isInstanceOf(NotFoundException.class) + .hasMessage("No rule with name Foobar"); + } + @Test public void uniqueTitles() { RuleDao rule = RuleDao.create(null, "test", "description", "rule \"test\" when true then end", null, null); diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineCatalogTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineCatalogTest.java new file mode 100644 index 000000000000..cd78b7109c4d --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineCatalogTest.java @@ -0,0 +1,155 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.catalogs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.graph.Graph; +import com.lordofthejars.nosqlunit.annotation.UsingDataSet; +import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; +import com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb; +import org.graylog.plugins.pipelineprocessor.ast.Pipeline; +import org.graylog.plugins.pipelineprocessor.ast.Stage; +import org.graylog.plugins.pipelineprocessor.ast.expressions.LogicalExpression; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineService; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbPipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.parser.PipelineRuleParser; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.contentpacks.codecs.PipelineCodec; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityDescriptor; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.PipelineEntity; +import org.graylog2.database.MongoConnectionRule; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import static com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb.InMemoryMongoRuleBuilder.newInMemoryMongoDbRule; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PipelineCatalogTest { + @ClassRule + public static final InMemoryMongoDb IN_MEMORY_MONGO_DB = newInMemoryMongoDbRule().build(); + + @Rule + public final MongoConnectionRule mongoRule = MongoConnectionRule.build("test"); + + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + @Mock + private PipelineRuleParser pipelineRuleParser; + private PipelineCatalog catalog; + + @Before + public void setUp() throws Exception { + final PipelineService pipelineService = new MongoDbPipelineService(mongoRule.getMongoConnection(), new MongoJackObjectMapperProvider(objectMapper)); + final PipelineStreamConnectionsService connectionsService = new MongoDbPipelineStreamConnectionsService(mongoRule.getMongoConnection(), new MongoJackObjectMapperProvider(objectMapper)); + final PipelineCodec codec = new PipelineCodec(objectMapper, pipelineService, connectionsService); + + catalog = new PipelineCatalog(pipelineService, connectionsService, pipelineRuleParser, codec); + } + + @Test + @UsingDataSet(locations = "/org/graylog2/contentpacks/pipeline_processor_pipelines.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void listEntityExcerpts() { + final EntityExcerpt expectedEntityExcerpt = EntityExcerpt.builder() + .id(ModelId.of("Test")) + .type(ModelTypes.PIPELINE) + .title("Test") + .build(); + + final Set entityExcerpts = catalog.listEntityExcerpts(); + assertThat(entityExcerpts).containsOnly(expectedEntityExcerpt); + } + + @Test + @UsingDataSet(locations = "/org/graylog2/contentpacks/pipeline_processor_pipelines.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void collectEntity() { + final Optional collectedEntity = catalog.collectEntity(EntityDescriptor.create(ModelId.of("Test"), ModelTypes.PIPELINE)); + assertThat(collectedEntity) + .isPresent() + .containsInstanceOf(EntityV1.class); + + final EntityV1 entity = (EntityV1) collectedEntity.orElseThrow(AssertionError::new); + assertThat(entity.id()).isEqualTo(ModelId.of("Test")); + assertThat(entity.type()).isEqualTo(ModelTypes.PIPELINE); + final PipelineEntity pipelineEntity = objectMapper.convertValue(entity.data(), PipelineEntity.class); + assertThat(pipelineEntity.title()).isEqualTo("Test"); + assertThat(pipelineEntity.description()).isEqualTo("Description"); + assertThat(pipelineEntity.source()).startsWith("pipeline \"Test\""); + } + + @Test + @UsingDataSet(locations = "/org/graylog2/contentpacks/pipeline_processor_pipelines.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void resolve() { + final Stage stage = Stage.builder() + .stage(0) + .matchAll(false) + .ruleReferences(ImmutableList.of("debug", "no-op")) + .build(); + org.graylog.plugins.pipelineprocessor.ast.Rule rule1 = org.graylog.plugins.pipelineprocessor.ast.Rule.builder() + .id("1") + .name("debug") + .when(mock(LogicalExpression.class)) + .then(Collections.emptyList()) + .build(); + org.graylog.plugins.pipelineprocessor.ast.Rule rule2 = org.graylog.plugins.pipelineprocessor.ast.Rule.builder() + .id("2") + .name("no-op") + .when(mock(LogicalExpression.class)) + .then(Collections.emptyList()) + .build(); + stage.setRules(ImmutableList.of(rule1, rule2)); + final Pipeline pipeline = Pipeline.builder() + .name("Test") + .stages(ImmutableSortedSet.of(stage)) + .build(); + when(pipelineRuleParser.parsePipeline(eq("dummy"), anyString())).thenReturn(pipeline); + final EntityDescriptor pipelineEntity = EntityDescriptor.create(ModelId.of("Test"), ModelTypes.PIPELINE); + + final Graph graph = catalog.resolve(pipelineEntity); + + final EntityDescriptor streamEntity = EntityDescriptor.create(ModelId.of("5adf23894b900a0fdb4e517d"), ModelTypes.STREAM); + final EntityDescriptor ruleEntity1 = EntityDescriptor.create(ModelId.of("debug"), ModelTypes.PIPELINE_RULE); + final EntityDescriptor ruleEntity2 = EntityDescriptor.create(ModelId.of("no-op"), ModelTypes.PIPELINE_RULE); + assertThat(graph.nodes()) + .containsOnly(pipelineEntity, streamEntity, ruleEntity1, ruleEntity2); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalogTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalogTest.java new file mode 100644 index 000000000000..596589e83922 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/catalogs/PipelineRuleCatalogTest.java @@ -0,0 +1,100 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.catalogs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.lordofthejars.nosqlunit.annotation.UsingDataSet; +import com.lordofthejars.nosqlunit.core.LoadStrategyEnum; +import com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbRuleService; +import org.graylog2.bindings.providers.MongoJackObjectMapperProvider; +import org.graylog2.contentpacks.codecs.PipelineRuleCodec; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityDescriptor; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.PipelineRuleEntity; +import org.graylog2.database.MongoConnectionRule; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Optional; +import java.util.Set; + +import static com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb.InMemoryMongoRuleBuilder.newInMemoryMongoDbRule; +import static org.assertj.core.api.Assertions.assertThat; + +public class PipelineRuleCatalogTest { + @ClassRule + public static final InMemoryMongoDb IN_MEMORY_MONGO_DB = newInMemoryMongoDbRule().build(); + + @Rule + public final MongoConnectionRule mongoRule = MongoConnectionRule.build("test"); + + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + private PipelineRuleCatalog catalog; + + @Before + public void setUp() throws Exception { + final RuleService ruleService = new MongoDbRuleService(mongoRule.getMongoConnection(), new MongoJackObjectMapperProvider(objectMapper)); + final PipelineRuleCodec codec = new PipelineRuleCodec(objectMapper, ruleService); + + catalog = new PipelineRuleCatalog(ruleService, codec); + } + + @Test + @UsingDataSet(locations = "/org/graylog2/contentpacks/pipeline_processor_rules.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void listEntityExcerpts() { + final EntityExcerpt expectedEntityExcerpt1 = EntityExcerpt.builder() + .id(ModelId.of("debug")) + .type(ModelTypes.PIPELINE_RULE) + .title("debug") + .build(); + final EntityExcerpt expectedEntityExcerpt2 = EntityExcerpt.builder() + .id(ModelId.of("no-op")) + .type(ModelTypes.PIPELINE_RULE) + .title("no-op") + .build(); + + final Set entityExcerpts = catalog.listEntityExcerpts(); + assertThat(entityExcerpts).containsOnly(expectedEntityExcerpt1, expectedEntityExcerpt2); + } + + @Test + @UsingDataSet(locations = "/org/graylog2/contentpacks/pipeline_processor_rules.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) + public void collectEntity() { + final Optional collectedEntity = catalog.collectEntity(EntityDescriptor.create(ModelId.of("debug"), ModelTypes.PIPELINE_RULE)); + assertThat(collectedEntity) + .isPresent() + .containsInstanceOf(EntityV1.class); + + final EntityV1 entity = (EntityV1) collectedEntity.orElseThrow(AssertionError::new); + assertThat(entity.id()).isEqualTo(ModelId.of("debug")); + assertThat(entity.type()).isEqualTo(ModelTypes.PIPELINE_RULE); + final PipelineRuleEntity pipelineRuleEntity = objectMapper.convertValue(entity.data(), PipelineRuleEntity.class); + assertThat(pipelineRuleEntity.title()).isEqualTo("debug"); + assertThat(pipelineRuleEntity.description()).isEqualTo("Debug"); + assertThat(pipelineRuleEntity.source()).startsWith("rule \"debug\"\n"); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineCodecTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineCodecTest.java new file mode 100644 index 000000000000..ee89defcc95b --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineCodecTest.java @@ -0,0 +1,99 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.codecs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog.plugins.pipelineprocessor.db.PipelineDao; +import org.graylog.plugins.pipelineprocessor.db.PipelineService; +import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.db.memory.InMemoryPipelineStreamConnectionsService; +import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelType; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.PipelineEntity; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PipelineCodecTest { + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + @Mock + private PipelineService pipelineService; + private PipelineStreamConnectionsService connectionsService; + private PipelineCodec codec; + + @Before + public void setUp() { + connectionsService = new InMemoryPipelineStreamConnectionsService(); + codec = new PipelineCodec(objectMapper, pipelineService, connectionsService); + } + + @Test + public void encode() { + final PipelineDao pipeline = PipelineDao.builder() + .id("pipeline-1234") + .title("title") + .description("description") + .source("pipeline \"Test\"\nstage 0 match either\nrule \"debug\"\nend") + .build(); + final PipelineConnections connections = PipelineConnections.create("id", "stream-1234", Collections.singleton("pipeline-1234")); + connectionsService.save(connections); + + final Entity entity = codec.encode(pipeline); + + assertThat(entity).isInstanceOf(EntityV1.class); + assertThat(entity.id()).isEqualTo(ModelId.of("title")); + assertThat(entity.type()).isEqualTo(ModelType.of("pipeline")); + + final EntityV1 entityV1 = (EntityV1) entity; + final PipelineEntity pipelineEntity = objectMapper.convertValue(entityV1.data(), PipelineEntity.class); + assertThat(pipelineEntity.title()).isEqualTo("title"); + assertThat(pipelineEntity.description()).isEqualTo("description"); + assertThat(pipelineEntity.source()).startsWith("pipeline \"Test\""); + assertThat(pipelineEntity.connectedStreams()).containsOnly("stream-1234"); + } + + @Test + public void createExcerpt() { + final PipelineDao pipeline = PipelineDao.builder() + .id("id") + .title("title") + .description("description") + .source("pipeline \"Test\"\nstage 0 match either\nrule \"debug\"\nend") + .build(); + final EntityExcerpt excerpt = codec.createExcerpt(pipeline); + + assertThat(excerpt.id()).isEqualTo(ModelId.of("title")); + assertThat(excerpt.type()).isEqualTo(ModelType.of("pipeline")); + assertThat(excerpt.title()).isEqualTo("title"); + } +} \ No newline at end of file diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineRuleCodecTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineRuleCodecTest.java new file mode 100644 index 000000000000..e945687fbfe8 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/codecs/PipelineRuleCodecTest.java @@ -0,0 +1,88 @@ +/** + * This file is part of Graylog. + * + * Graylog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Graylog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Graylog. If not, see . + */ +package org.graylog2.contentpacks.codecs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.graylog.plugins.pipelineprocessor.db.RuleDao; +import org.graylog.plugins.pipelineprocessor.db.RuleService; +import org.graylog2.contentpacks.model.ModelId; +import org.graylog2.contentpacks.model.ModelType; +import org.graylog2.contentpacks.model.entities.Entity; +import org.graylog2.contentpacks.model.entities.EntityExcerpt; +import org.graylog2.contentpacks.model.entities.EntityV1; +import org.graylog2.contentpacks.model.entities.PipelineEntity; +import org.graylog2.shared.bindings.providers.ObjectMapperProvider; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PipelineRuleCodecTest { + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + private final ObjectMapper objectMapper = new ObjectMapperProvider().get(); + + @Mock + private RuleService ruleService; + private PipelineRuleCodec codec; + + @Before + public void setUp() { + codec = new PipelineRuleCodec(objectMapper, ruleService); + } + + @Test + public void encode() { + final RuleDao pipelineRule = RuleDao.builder() + .id("id") + .title("title") + .description("description") + .source("rule \"debug\"\nwhen\n true\nthen\n debug($message.message);\nend") + .build(); + final Entity entity = codec.encode(pipelineRule); + + assertThat(entity).isInstanceOf(EntityV1.class); + assertThat(entity.id()).isEqualTo(ModelId.of("title")); + assertThat(entity.type()).isEqualTo(ModelType.of("pipeline_rule")); + + final EntityV1 entityV1 = (EntityV1) entity; + final PipelineEntity pipelineEntity = objectMapper.convertValue(entityV1.data(), PipelineEntity.class); + assertThat(pipelineEntity.title()).isEqualTo("title"); + assertThat(pipelineEntity.description()).isEqualTo("description"); + assertThat(pipelineEntity.source()).startsWith("rule \"debug\"\n"); + } + + @Test + public void createExcerpt() { + final RuleDao pipelineRule = RuleDao.builder() + .id("id") + .title("title") + .description("description") + .source("rule \"debug\"\nwhen\n true\nthen\n debug($message.message);\nend") + .build(); + final EntityExcerpt excerpt = codec.createExcerpt(pipelineRule); + + assertThat(excerpt.id()).isEqualTo(ModelId.of("title")); + assertThat(excerpt.type()).isEqualTo(ModelType.of("pipeline_rule")); + assertThat(excerpt.title()).isEqualTo("title"); + } +} \ No newline at end of file diff --git a/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines.json b/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines.json index 0ca232360cd9..fc1f89a6289f 100644 --- a/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines.json +++ b/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines.json @@ -5,8 +5,8 @@ "$oid": "5a85c4854b900afd5d662be3" }, "title": "Test", - "description": "test", - "source": "pipeline \"Test\"\nstage 0 match either\nrule \"debug\"\nend", + "description": "Description", + "source": "pipeline \"Test\"\nstage 0 match either\nrule \"debug\"\nrule \"no-op\"\nend", "created_at": { "$date": "2018-02-15T17:33:57.125Z" }, @@ -14,5 +14,16 @@ "$date": "2018-04-24T12:37:31.592Z" } } + ], + "pipeline_processor_pipelines_streams": [ + { + "_id": { + "$oid": "5adf28464b900a0fdb4e56f2" + }, + "stream_id": "5adf23894b900a0fdb4e517d", + "pipeline_ids": [ + "5a85c4854b900afd5d662be3" + ] + } ] } diff --git a/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines_streams.json b/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines_streams.json deleted file mode 100644 index 5daf8000e231..000000000000 --- a/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_pipelines_streams.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "pipeline_processor_pipelines_streams": [ - { - "_id": { - "$oid": "5adf28464b900a0fdb4e56f2" - }, - "stream_id": "5adf23894b900a0fdb4e517d", - "pipeline_ids": [ - "5a85c4854b900afd5d662be3" - ] - } - ] -} diff --git a/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_rules.json b/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_rules.json index c7e17ac99272..3ed22769d5a3 100644 --- a/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_rules.json +++ b/graylog2-server/src/test/resources/org/graylog2/contentpacks/pipeline_processor_rules.json @@ -13,6 +13,20 @@ "modified_at": { "$date": "2018-04-24T12:37:23.033Z" } + }, + { + "_id": { + "$oid": "5adf25034b900a0fdb4e5339" + }, + "title": "no-op", + "description": "Do nothing", + "source": "rule \"no-op\"\nwhen\n true\nthen\nend", + "created_at": { + "$date": "2018-04-24T12:37:23.033Z" + }, + "modified_at": { + "$date": "2018-04-24T12:37:23.033Z" + } } ] }