add optional to return search results as CSV

to be able to switch the serialization based on the accept header, we have to use the jersey objectmapper support which requires to have a separate csv body writer

also added support for multiple content types to the api browser

fixes #204

graylog2-server/pom.xml

@@ -31,12 +31,6 @@
     </properties>
 
     <dependencies>
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-            <version>1.8</version>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>log4j</groupId>
             <artifactId>log4j</artifactId>
@@ -48,12 +42,6 @@
             <version>2.5.1</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>com.twilio.sdk</groupId>
-            <artifactId>twilio-java-sdk</artifactId>
-            <version>3.3.15</version>
-            <classifier>jar-with-dependencies</classifier>
-        </dependency>
         <dependency>
             <groupId>com.lmax</groupId>
             <artifactId>disruptor</artifactId>
@@ -114,6 +102,16 @@
             <artifactId>opencsv</artifactId>
             <version>2.3</version>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.8</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.3.1</version>
+        </dependency>
     </dependencies>
 
     <build>

graylog2-server/src/main/java/org/graylog2/Core.java

@@ -21,6 +21,7 @@
 package org.graylog2;
 
 import com.codahale.metrics.MetricRegistry;
+import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -61,6 +62,7 @@
 import org.graylog2.plugin.outputs.MessageOutput;
 import org.graylog2.plugin.streams.Stream;
 import org.graylog2.plugins.PluginLoader;
+import org.graylog2.rest.ObjectMapperProvider;
 import org.graylog2.security.ShiroSecurityBinding;
 import org.graylog2.security.ShiroSecurityContextFactory;
 import org.graylog2.security.realm.LdapRealm;
@@ -76,6 +78,7 @@
 import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
 import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
 import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
+import org.jboss.netty.handler.stream.ChunkedWriteHandler;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
@@ -354,11 +357,14 @@ public void startRestApi() throws IOException {
                 bossExecutor,
                 workerExecutor
         ));
-        ResourceConfig rc = new ResourceConfig();
-        rc.property(NettyContainer.PROPERTY_BASE_URI, configuration.getRestListenUri());
-        rc.registerClasses(MetricsDynamicBinding.class, AnyExceptionClassMapper.class, ShiroSecurityBinding.class);
-        rc.register(new Graylog2Binder());
-        rc.registerFinder(new PackageNamesScanner(new String[] {"org.graylog2.rest.resources"}, true));
+
+        ResourceConfig rc = new ResourceConfig()
+                .property(NettyContainer.PROPERTY_BASE_URI, configuration.getRestListenUri())
+                .registerClasses(MetricsDynamicBinding.class, AnyExceptionClassMapper.class, ShiroSecurityBinding.class)
+                .register(new Graylog2Binder())
+                .register(ObjectMapperProvider.class)
+                .register(JacksonJsonProvider.class)
+                .registerFinder(new PackageNamesScanner(new String[]{"org.graylog2.rest.resources"}, true));
         final NettyContainer jerseyHandler = ContainerFactory.createContainer(NettyContainer.class, rc);
         jerseyHandler.setSecurityContextFactory(new ShiroSecurityContextFactory(this));
 
@@ -368,6 +374,7 @@ public ChannelPipeline getPipeline() throws Exception {
                 ChannelPipeline pipeline = Channels.pipeline();
                 pipeline.addLast("decoder", new HttpRequestDecoder());
                 pipeline.addLast("encoder", new HttpResponseEncoder());
+                pipeline.addLast("chunks", new ChunkedWriteHandler());
                 pipeline.addLast("jerseyHandler", jerseyHandler);
                 return pipeline;
             }

graylog2-server/src/main/java/org/graylog2/rest/ObjectMapperProvider.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013 TORCH GmbH
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+
+@Provider
+public class ObjectMapperProvider implements ContextResolver<ObjectMapper> {
+    private final ObjectMapper objectMapper;
+
+    public ObjectMapperProvider() {
+        objectMapper = new ObjectMapper();
+        objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+        objectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+    }
+
+    @Override
+    public ObjectMapper getContext(Class<?> type) {
+        return objectMapper;
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/documentation/generator/Generator.java

@@ -164,13 +164,24 @@ public Map<String, Object> generateForRoute(String route, String basePath) {
                         }
                     }
 
+                    Produces produces = null;
+                    if (clazz.isAnnotationPresent(Produces.class) || method.isAnnotationPresent(Produces.class)) {
+                        produces = clazz.getAnnotation(Produces.class);
+                        if (method.isAnnotationPresent(Produces.class)) {
+                            produces = method.getAnnotation(Produces.class);
+                        }
+                    }
                     api.put("path", methodPath);
 
                     Map<String, Object> operation = Maps.newHashMap();
                     operation.put("method", determineHttpMethod(method));
                     operation.put("summary", apiOperation.value());
                     operation.put("notes", apiOperation.notes());
                     operation.put("nickname", method.getName());
+                    if (produces != null) {
+                        operation.put("produces", produces.value());
+                    }
+                    operation.put("type", method.getReturnType().getSimpleName());
 
                     List<Map<String, Object>> parameters = determineParameters(method);
                     if (parameters != null && !parameters.isEmpty()) {
@@ -236,6 +247,9 @@ private List<Map<String, Object>> determineParameters(Method method) {
                     paramType = "query";
                 } else if (annotation instanceof PathParam) {
                     paramType = "path";
+                } else if (annotation instanceof HeaderParam) {
+                    // TODO skip header params for now, we use them for Accept headers until we return proper objects
+                    continue;
                 } else {
                     paramType = "body";
                 }

graylog2-server/src/main/java/org/graylog2/rest/resources/RestResource.java

@@ -23,13 +23,18 @@
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.jaxrs.cfg.EndpointConfigBase;
+import com.fasterxml.jackson.jaxrs.cfg.ObjectWriterInjector;
+import com.fasterxml.jackson.jaxrs.cfg.ObjectWriterModifier;
 import com.google.common.collect.Maps;
 import org.apache.shiro.subject.Subject;
 import org.bson.types.ObjectId;
-import org.codehaus.jackson.map.SerializationConfig;
 import org.graylog2.Core;
 import org.graylog2.security.ShiroSecurityContext;
 import org.slf4j.Logger;
@@ -38,10 +43,7 @@
 import javax.inject.Inject;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.*;
 import java.security.Principal;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -58,8 +60,7 @@ public abstract class RestResource {
     @Inject
     protected Core core;
 
-    @QueryParam("pretty")
-    boolean prettyPrint;
+    private boolean prettyPrint;
 
     @Context
     SecurityContext securityContext;
@@ -70,6 +71,21 @@ protected RestResource() {
           * Make it write ISO8601 instead.
           */
         objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+        objectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+    }
+
+    @QueryParam("pretty")
+    public void setPrettyPrint(boolean prettyPrint) {
+        if (prettyPrint) {
+            /* sigh jersey, hooray @cowtowncoder : https://twitter.com/cowtowncoder/status/402226988603035648 */
+            ObjectWriterInjector.set(new ObjectWriterModifier() {
+                @Override
+                public ObjectWriter modify(EndpointConfigBase<?> endpoint, MultivaluedMap<String, Object> responseHeaders, Object valueToWrite, ObjectWriter w, JsonGenerator g) {
+                    return w.withDefaultPrettyPrinter();
+                }
+            });
+        }
+        this.prettyPrint = prettyPrint;
     }
 
     protected int page(int page) {

graylog2-server/src/main/java/org/graylog2/rest/resources/search/AbsoluteSearchResource.java

@@ -26,6 +26,7 @@
 import org.graylog2.indexer.searches.timeranges.InvalidRangeParametersException;
 import org.graylog2.indexer.searches.timeranges.TimeRange;
 import org.graylog2.rest.documentation.annotations.*;
+import org.graylog2.rest.resources.search.responses.SearchResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,11 +47,11 @@ public class AbsoluteSearchResource extends SearchResource {
     @ApiOperation(value = "Message search with absolute timerange.",
             notes = "Search for messages using an absolute timerange, specified as from/to " +
                     "with format yyyy-MM-dd HH-mm-ss.SSS or yyyy-MM-dd HH-mm-ss.")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces({ MediaType.APPLICATION_JSON, "text/csv" })
     @ApiResponses(value = {
             @ApiResponse(code = 400, message = "Invalid timerange parameters provided.")
     })
-    public String searchAbsolute(
+    public SearchResponse searchAbsolute(
             @ApiParam(title = "query", description = "Query (Lucene syntax)", required = true) @QueryParam("query") String query,
             @ApiParam(title = "from", description = "Timerange start. See description for date format", required = true) @QueryParam("from") String from,
             @ApiParam(title = "to", description = "Timerange end. See description for date format", required = true) @QueryParam("to") String to,
@@ -60,19 +61,19 @@ public String searchAbsolute(
         checkQuery(query);
 
         try {
-            Map<String, Object> searchResult;
+            SearchResponse searchResponse;
 
             if (filter == null) {
-                searchResult = buildSearchResult(
+                searchResponse = buildSearchResponse(
                         core.getIndexer().searches().search(query, buildAbsoluteTimeRange(from, to), limit, offset)
                 );
             } else {
-                searchResult = buildSearchResult(
+                searchResponse = buildSearchResponse(
                         core.getIndexer().searches().search(query, filter, buildAbsoluteTimeRange(from, to), limit, offset)
                 );
             }
 
-            return json(searchResult);
+            return searchResponse;
         } catch (IndexHelper.InvalidRangeFormatException e) {
             LOG.warn("Invalid timerange parameters provided. Returning HTTP 400.", e);
             throw new WebApplicationException(400);

graylog2-server/src/main/java/org/graylog2/rest/resources/search/KeywordSearchResource.java

@@ -26,6 +26,7 @@
 import org.graylog2.indexer.searches.timeranges.KeywordRange;
 import org.graylog2.indexer.searches.timeranges.TimeRange;
 import org.graylog2.rest.documentation.annotations.*;
+import org.graylog2.rest.resources.search.responses.SearchResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,11 +45,11 @@ public class KeywordSearchResource extends SearchResource {
     @GET @Timed
     @ApiOperation(value = "Message search with keyword as timerange.",
             notes = "Search for messages in a timerange defined by a keyword like \"yesterday\" or \"2 weeks ago to wednesday\".")
-    @Produces(MediaType.APPLICATION_JSON)
+    @Produces({ MediaType.APPLICATION_JSON, "text/csv" })
     @ApiResponses(value = {
             @ApiResponse(code = 400, message = "Invalid keyword provided.")
     })
-    public String searchKeyword(
+    public SearchResponse searchKeyword(
             @ApiParam(title = "query", description = "Query (Lucene syntax)", required = true) @QueryParam("query") String query,
             @ApiParam(title = "keyword", description = "Range keyword", required = true) @QueryParam("keyword") String keyword,
             @ApiParam(title = "limit", description = "Maximum number of messages to return.", required = false) @QueryParam("limit") int limit,
@@ -58,13 +59,13 @@ public String searchKeyword(
 
         try {
             if (filter == null) {
-                return json(buildSearchResult(
+                return buildSearchResponse(
                         core.getIndexer().searches().search(query, buildKeywordTimeRange(keyword), limit, offset)
-                ));
+                );
             } else {
-                return json(buildSearchResult(
+                return buildSearchResponse(
                         core.getIndexer().searches().search(query, filter, buildKeywordTimeRange(keyword), limit, offset)
-                ));
+                );
             }
         } catch (IndexHelper.InvalidRangeFormatException e) {
             LOG.warn("Invalid timerange parameters provided. Returning HTTP 400.", e);

graylog2-server/src/main/java/org/graylog2/rest/resources/search/RelativeSearchResource.java

@@ -22,17 +22,16 @@
 import com.codahale.metrics.annotation.Timed;
 import org.graylog2.indexer.IndexHelper;
 import org.graylog2.indexer.Indexer;
-import org.graylog2.indexer.searches.Searches;
 import org.graylog2.indexer.searches.timeranges.InvalidRangeParametersException;
 import org.graylog2.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.indexer.searches.timeranges.TimeRange;
 import org.graylog2.rest.documentation.annotations.*;
+import org.graylog2.rest.resources.search.responses.SearchResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
-import java.util.Map;
 
 /**
  * @author Lennart Koopmann <lennart@torch.sh>
@@ -45,13 +44,13 @@ public class RelativeSearchResource extends SearchResource {
 
     @GET @Timed
     @ApiOperation(value = "Message search with relative timerange.",
-            notes = "Search for messages in a relative timerange, specified as seconds from now. " +
-                    "Example: 300 means search from 5 minutes ago to now.")
-    @Produces(MediaType.APPLICATION_JSON)
+                  notes = "Search for messages in a relative timerange, specified as seconds from now. " +
+                          "Example: 300 means search from 5 minutes ago to now.")
     @ApiResponses(value = {
             @ApiResponse(code = 400, message = "Invalid timerange parameters provided.")
     })
-    public String searchRelative(
+    @Produces({ MediaType.APPLICATION_JSON, "text/csv" })
+    public SearchResponse searchRelative(
             @ApiParam(title = "query", description = "Query (Lucene syntax)", required = true) @QueryParam("query") String query,
             @ApiParam(title = "range", description = "Relative timeframe to search in. See method description.", required = true) @QueryParam("range") int range,
             @ApiParam(title = "limit", description = "Maximum number of messages to return.", required = false) @QueryParam("limit") int limit,
@@ -60,19 +59,19 @@ public String searchRelative(
         checkQuery(query);
 
         try {
-            Map<String, Object> searchResult;
+            SearchResponse searchResponse;
 
             if (filter == null) {
-                searchResult = buildSearchResult(
+                searchResponse = buildSearchResponse(
                         core.getIndexer().searches().search(query, buildRelativeTimeRange(range), limit, offset)
                 );
             } else {
-                searchResult = buildSearchResult(
+                searchResponse = buildSearchResponse(
                         core.getIndexer().searches().search(query, filter, buildRelativeTimeRange(range), limit, offset)
                 );
             }
 
-            return json(searchResult);
+            return searchResponse;
         } catch (IndexHelper.InvalidRangeFormatException e) {
             LOG.warn("Invalid timerange parameters provided. Returning HTTP 400.", e);
             throw new WebApplicationException(400);