From d2b25af3f4a21806efc3e38ebbe6e66b7dc5c0d4 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 30 Oct 2023 17:25:12 -0600 Subject: [PATCH] examples: add an example plugin of an eve filetype This is an example of what adding plugin examples to the Suricata repo could look like. This plugin is an example plugin for an EVE filetype. It could be extended to support outputs like Redis, syslog, etc. There is one issue with adding plugins like this to an autotools project, the project can't be built with --disable-shared, which is more of an autotools limitation, and not really a Suricata issue. Suricata built with --disable-shared will load plugins just fine. Note that the examples directory was added as DIST_SUBDIRS as we don't want normal builds to recurse into it and attempt to build the plugin, its just an example, but we still need to keep distcheck happy. --- .github/workflows/builds.yml | 12 + Makefile.am | 1 + configure.ac | 2 + examples/plugins/README.md | 6 + examples/plugins/c-json-filetype/.gitignore | 2 + examples/plugins/c-json-filetype/Makefile.am | 17 ++ .../plugins/c-json-filetype/Makefile.example | 18 ++ examples/plugins/c-json-filetype/README.md | 123 +++++++++ examples/plugins/c-json-filetype/filetype.c | 243 ++++++++++++++++++ 9 files changed, 424 insertions(+) create mode 100644 examples/plugins/README.md create mode 100644 examples/plugins/c-json-filetype/.gitignore create mode 100644 examples/plugins/c-json-filetype/Makefile.am create mode 100644 examples/plugins/c-json-filetype/Makefile.example create mode 100644 examples/plugins/c-json-filetype/README.md create mode 100644 examples/plugins/c-json-filetype/filetype.c diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index adb2a7f2330e..93708415294a 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -288,6 +288,18 @@ jobs: test -e /usr/local/lib/suricata/python/suricata/update/configs/modify.conf test -e /usr/local/lib/suricata/python/suricata/update/configs/threshold.in test -e /usr/local/lib/suricata/python/suricata/update/configs/update.yaml + - name: Build C json filetype plugin + working-directory: examples/plugins/c-json-filetype + run: make + - name: Check C json filetype plugin + run: test -e examples/plugins/c-json-filetype/.libs/json-filetype.so.0.0.0 + - name: Installing headers and library + run: | + make install-headers + make install-library + - name: Test plugin build with Makefile.example + working-directory: examples/plugins/c-json-filetype + run: PATH=/usr/local/bin:$PATH make -f Makefile.example almalinux-9-templates: name: AlmaLinux 9 Test Templates diff --git a/Makefile.am b/Makefile.am index 67963ed32fcf..b7a221f49299 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,6 +10,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \ scripts/generate-images.sh SUBDIRS = $(HTP_DIR) rust src qa rules doc contrib etc python ebpf \ $(SURICATA_UPDATE_DIR) +DIST_SUBDIRS = examples/plugins/c-json-filetype $(SUBDIRS) CLEANFILES = stamp-h[0-9]* diff --git a/configure.ac b/configure.ac index 1908c76ab76a..1eb054d98c1e 100644 --- a/configure.ac +++ b/configure.ac @@ -2613,6 +2613,8 @@ AC_CONFIG_FILES(suricata.yaml etc/Makefile etc/suricata.logrotate etc/suricata.s AC_CONFIG_FILES(python/Makefile python/suricata/config/defaults.py) AC_CONFIG_FILES(ebpf/Makefile) AC_CONFIG_FILES(libsuricata-config) +AC_CONFIG_FILES(examples/plugins/c-json-filetype/Makefile) + AC_OUTPUT SURICATA_BUILD_CONF="Suricata Configuration: diff --git a/examples/plugins/README.md b/examples/plugins/README.md new file mode 100644 index 000000000000..8e47b6e7ffc2 --- /dev/null +++ b/examples/plugins/README.md @@ -0,0 +1,6 @@ +# Example Plugins + +## c-json-filetype + +An example plugin of an EVE/JSON filetype plugin. This type of plugin +is useful if you want to send EVE output to custom destinations. diff --git a/examples/plugins/c-json-filetype/.gitignore b/examples/plugins/c-json-filetype/.gitignore new file mode 100644 index 000000000000..f5bb3af73f70 --- /dev/null +++ b/examples/plugins/c-json-filetype/.gitignore @@ -0,0 +1,2 @@ +*.so +*.la diff --git a/examples/plugins/c-json-filetype/Makefile.am b/examples/plugins/c-json-filetype/Makefile.am new file mode 100644 index 000000000000..d5e912b8a469 --- /dev/null +++ b/examples/plugins/c-json-filetype/Makefile.am @@ -0,0 +1,17 @@ +plugindir = ${libdir}/suricata/plugins + +if BUILD_SHARED_LIBRARY +plugin_LTLIBRARIES = json-filetype.la +json_filetype_la_LDFLAGS = -module -shared +json_filetype_la_SOURCES = filetype.c + +json_filetype_la_CPPFLAGS = -I$(abs_top_srcdir)/rust/gen -I$(abs_top_srcdir)/rust/dist + +else + +all-local: + @echo + @echo "Shared library support must be enabled to build plugins." + @echo + +endif diff --git a/examples/plugins/c-json-filetype/Makefile.example b/examples/plugins/c-json-filetype/Makefile.example new file mode 100644 index 000000000000..6d514aab1445 --- /dev/null +++ b/examples/plugins/c-json-filetype/Makefile.example @@ -0,0 +1,18 @@ +SRCS := filetype.c + +LIBSURICATA_CONFIG ?= libsuricata-config + +CPPFLAGS += `$(LIBSURICATA_CONFIG) --cflags` +CPPFLAGS += -DSURICATA_PLUGIN -I. +CPPFLAGS += "-D__SCFILENAME__=\"$(*F)\"" + +OBJS := $(SRCS:.c=.o) + +filetype.so: $(OBJS) + $(CC) -fPIC -shared -o $@ $(OBJS) + +%.o: %.c + $(CC) -fPIC $(CPPFLAGS) -c -o $@ $< + +clean: + rm -f *.o *.so *~ diff --git a/examples/plugins/c-json-filetype/README.md b/examples/plugins/c-json-filetype/README.md new file mode 100644 index 000000000000..2f7978977dbd --- /dev/null +++ b/examples/plugins/c-json-filetype/README.md @@ -0,0 +1,123 @@ +# Example EVE Filetype Plugin + +## Building + +If in the Suricata source directory, this plugin can be built by +running `make` and installed with `make install`. + +Note that Suricata must have been built without `--disable-shared`. + +## Building Standalone + +The file `Makefile.example` is an example of how you might build a +plugin that is distributed separately from the Suricata source code. + +It has the following dependencies: + +- Suricata is installed +- The Suricata library is installed: `make install-library` +- The Suricata development headers are installed: `make install-headers` +- The program `libsuricata-config` is in your path (installed with + `make install-library`) + +The run: `make -f Makefile.example` + +Before building this plugin you will need to build and install Suricata from the +git master branch and install the development tools and headers: + +- `make install-library` +- `make install-headers` + +then make sure the newly installed tool `libsuricata-config` can be +found in your path, for example: +``` +libsuricata-config --cflags +``` + +Then a simple `make` should build this plugin. + +Or if the Suricata installation is not in the path, a command like the following +can be used: + +``` +PATH=/opt/suricata/bin:$PATH make +``` + +## Usage + +To run the plugin, first add the path to the plugin you just compiled to +your `suricata.yaml`, for example: +``` +plugins: + - /usr/lib/suricata/plugins/json-filetype.so +``` + +Then add an output for the plugin: +``` +outputs: + - eve-log: + enabled: yes + filetype: json-filetype-plugin + threaded: true + types: + - dns + - tls + - http +``` + +In the example above we use the name specified in the plugin as the `filetype` +and specify that all `dns`, `tls` and `http` log entries should be sent to the +plugin. + +## Details + +This plugin demonstrates a Suricata JSON/EVE output plugin +(file-type). The idea of a Suricata EVE output plugin is to provide a +file like interface for the handling of rendered JSON logs. This is +useful for custom destinations not builtin to Suricata or if the +formatted JSON requires some post-processing. + +Note: EVE output plugins are not that useful just for reformatting the +JSON output as the plugin does need to handle writing to a file once +the file type has been delegated to the plugin. + +### Registering a Plugin + +All Suricata plugins make themselves known to Suricata by using a +function named `SCPluginRegister` which is called after Suricata loads +the plugin shared object file. This function must return a `SCPlugin` +struct which contains basic information about the plugin. For +example: + +```c +const SCPlugin PluginRegistration = { + .name = "eve-filetype", + .author = "Jason Ish", + .license = "GPLv2", + .Init = TemplateInit, +}; + +const SCPlugin *SCPluginRegister() { + return &PluginRegistration; +} +``` + +### Initializing a Plugin + +After the plugin has been registered, the `Init` callback will be called. This +is where the plugin will set itself up as a specific type of plugin such as an +EVE output, or a capture method. + +This plugins registers itself as an EVE file type using the +`SCRegisterEveFileType` struct. To register as an EVE file type the +following must be provided: + +* name: This is the name of the output which will be used in the eve filetype + field in `suricata.yaml` to enable this output. +* Init: The callback called when the output is "opened". +* Deinit: The callback called the output is "closed". +* ThreadInit: Callback called to initialize per thread data (if threaded). +* ThreadDeinit: Callback called to deinitialize per thread data (if threaded). +* Write: The callback called when an EVE record is to be "written". + +Please see the code in `filetype.c` for more details about this functions. diff --git a/examples/plugins/c-json-filetype/filetype.c b/examples/plugins/c-json-filetype/filetype.c new file mode 100644 index 000000000000..9c81d7f03267 --- /dev/null +++ b/examples/plugins/c-json-filetype/filetype.c @@ -0,0 +1,243 @@ +/* Copyright (C) 2020-2023 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "suricata-plugin.h" +#include "util-mem.h" +#include "util-debug.h" + +#define FILETYPE_NAME "json-filetype-plugin" + +static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data); +static int FiletypeThreadDeinit(void *ctx, void *thread_data); + +/** + * Per thread context data for each logging thread. + */ +typedef struct ThreadData_ { + /** The thread ID, for demonstration purposes only. */ + int thread_id; + + /** The number of records logged on this thread. */ + uint64_t count; +} ThreadData; + +/** + * A context object for each eve logger using this output. + */ +typedef struct Context_ { + /** Verbose, or print to stdout. */ + int verbose; + + /** A thread context to use when not running in threaded mode. */ + ThreadData *thread; +} Context; + +/** + * This function is called to initialize the output, it can be somewhat thought + * of like opening a file. + * + * \param conf The EVE configuration node using this output. + * + * \param threaded If true the EVE subsystem is running in threaded mode. + * + * \param data A pointer where context data can be stored relevant to this + * output. + * + * Eve output plugins need to be thread aware as the threading happens at lower + * level than the EVE output, so a flag is provided here to notify the plugin if + * threading is enabled or not. + * + * If the plugin does not work with threads disabled, or enabled, this function + * should return -1. + * + * Note for upgrading a plugin from 6.0 to 7.0: The ConfNode in 7.0 is the + * configuration for the eve instance, not just a node named after the plugin. + * This allows the plugin to get more context about what it is logging. + */ +static int FiletypeInit(ConfNode *conf, bool threaded, void **data) +{ + SCLogNotice("Initializing template eve output plugin: threaded=%d", threaded); + Context *context = SCCalloc(1, sizeof(Context)); + if (context == NULL) { + return -1; + } + + /* Verbose by default. */ + int verbose = 1; + + /* An example of how you can access configuration data from a + * plugin. */ + if (conf && (conf = ConfNodeLookupChild(conf, "eve-template")) != NULL) { + if (!ConfGetChildValueBool(conf, "verbose", &verbose)) { + verbose = 1; + } else { + SCLogNotice("Read verbose configuration value of %d", verbose); + } + } + context->verbose = verbose; + + if (!threaded) { + /* We're not running in threaded mode so allocate a thread context here + * to avoid duplication of context data such as file pointers, database + * connections, etc. */ + if (FiletypeThreadInit(context, 0, (void **)&context->thread) != 0) { + SCFree(context); + return -1; + } + } + *data = context; + return 0; +} + +/** + * This function is called when the output is closed. + * + * This will be called after ThreadDeinit is called for each thread. + * + * \param data The data allocated in FiletypeInit. It should be cleaned up and + * deallocated here. + */ +static void FiletypeDeinit(void *data) +{ + printf("TemplateClose\n"); + Context *ctx = data; + if (ctx != NULL) { + if (ctx->thread) { + FiletypeThreadDeinit(ctx, (void *)ctx->thread); + } + SCFree(ctx); + } +} + +/** + * Initialize per thread context. + * + * \param ctx The context created in TemplateInitOutput. + * + * \param thread_id An identifier for this thread. + * + * \param thread_data Pointer where thread specific context can be stored. + * + * When the EVE output is running in threaded mode this will be called once for + * each output thread with a unique thread_id. For regular file logging in + * threaded mode Suricata uses the thread_id to construct the files in the form + * of "eve..json". This plugin may want to do similar, or open + * multiple connections to whatever the final logging location might be. + * + * In the case of non-threaded EVE logging this function is NOT called by + * Suricata, but instead this plugin chooses to use this method to create a + * default (single) thread context. + */ +static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data) +{ + ThreadData *tdata = SCCalloc(1, sizeof(ThreadData)); + if (tdata == NULL) { + SCLogError("Failed to allocate thread data"); + return -1; + } + tdata->thread_id = thread_id; + *thread_data = tdata; + SCLogNotice( + "Initialized thread %03d (pthread_id=%" PRIuMAX ")", tdata->thread_id, pthread_self()); + return 0; +} + +/** + * Deinitialize a thread. + * + * This is where any cleanup per thread should be done including free'ing of the + * thread_data if needed. + */ +static int FiletypeThreadDeinit(void *ctx, void *thread_data) +{ + if (thread_data == NULL) { + // Nothing to do. + return 0; + } + + ThreadData *tdata = thread_data; + SCLogNotice( + "Deinitializing thread %d: records written: %" PRIu64, tdata->thread_id, tdata->count); + SCFree(tdata); + return 0; +} + +/** + * This method is called with formatted Eve JSON data. + * + * \param buffer Formatted JSON buffer \param buffer_len Length of formatted + * JSON buffer \param data Data set in Init callback \param thread_data Data set + * in ThreadInit callbacl + * + * Do not block in this thread, it will cause packet loss. Instead of outputting + * to any resource that may block it might be best to enqueue the buffers for + * further processing which will require copying of the provided buffer. + */ +static int FiletypeWrite(const char *buffer, int buffer_len, void *data, void *thread_data) +{ + Context *ctx = data; + ThreadData *thread = thread_data; + + /* The thread_data could be null which is valid, or it could be that we are + * in single threaded mode. */ + if (thread == NULL) { + thread = ctx->thread; + } + + thread->count++; + + if (ctx->verbose) { + SCLogNotice("Received write with thread_data %p: %s", thread_data, buffer); + } + return 0; +} + +/** + * Called by Suricata to initialize the module. This module registers + * new file type to the JSON logger. + */ +void PluginInit(void) +{ + SCEveFileType *my_output = SCCalloc(1, sizeof(SCEveFileType)); + my_output->name = FILETYPE_NAME; + my_output->Init = FiletypeInit; + my_output->Deinit = FiletypeDeinit; + my_output->ThreadInit = FiletypeThreadInit; + my_output->ThreadDeinit = FiletypeThreadDeinit; + my_output->Write = FiletypeWrite; + if (!SCRegisterEveFileType(my_output)) { + FatalError("Failed to register filetype plugin: %s", FILETYPE_NAME); + } +} + +const SCPlugin PluginRegistration = { + .name = FILETYPE_NAME, + .author = "FirstName LastName ", + .license = "GPL-2.0-only", + .Init = PluginInit, +}; + +/** + * The function called by Suricata after loading this plugin. + * + * A pointer to a populated SCPlugin struct must be returned. + */ +const SCPlugin *SCPluginRegister() +{ + return &PluginRegistration; +}