diff --git a/doc/userguide/output/eve/eve-json-format.rst b/doc/userguide/output/eve/eve-json-format.rst index 20ecc854a0fc..b9e13eff0f5e 100644 --- a/doc/userguide/output/eve/eve-json-format.rst +++ b/doc/userguide/output/eve/eve-json-format.rst @@ -417,6 +417,72 @@ Example of a old DNS answer with an IPv4 (resource record type 'A') return: "rdata": "199.16.156.6" } +Event type: FTP +--------------- + +Fields +~~~~~~ + +* "command": The FTP command. +* "command_data": The data accompanying the command. +* "reply": The command reply, which may contain multiple lines, in array format. +* "completion_code": The 3-digit completion code. The first digit indicates whether the response is good, bad or incomplete. +* "dynamic_port": The dynamic port established for subsequent data transfers, when applicable, with a "PORT" or "EPRT" command. +* "mode": The type of FTP connection. Most connections are "passive" but may be "active". + +Examples +~~~~~~~~ + +Example of regular FTP logging: + +:: + + "ftp": { + "command": "RETR", + "command_data": "index.html", + "reply": [ + "Opening BINARY mode data connection for index.html (6712 bytes)", + "Transfer complete" + ], + "completion_code": "150" + } + +Example showing all fields + +:: + + "ftp": { + "command": "EPRT", + "command_data": "|2|2a01:e34:ee97:b130:8c3e:45ea:5ac6:e301|41813|", + "reply": [ + "EPRT command successful. Consider using EPSV" + ], + "reply_code": "200", + "dynamic_port": 41813, + "mode": "active" + } + +Event type: FTP_DATA +-------------------- + +Fields +~~~~~~ + +* "command": The FTP command associated with the event. +* "filename": The name of the involved file. + +Examples +~~~~~~~~ + +Example of FTP_DATA logging: + +:: + + "ftp_data": { + "filename": "temp.txt", + "command": "RETR" + } + Event type: TLS --------------- diff --git a/src/Makefile.am b/src/Makefile.am index fa5c8b9ff1d9..55dcfcd589c9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -307,6 +307,7 @@ output-json-drop.c output-json-drop.h \ output-json-email-common.c output-json-email-common.h \ output-json-file.c output-json-file.h \ output-json-flow.c output-json-flow.h \ +output-json-ftp.c output-json-ftp.h \ output-json-netflow.c output-json-netflow.h \ output-json-http.c output-json-http.h \ output-json-smtp.c output-json-smtp.h \ diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index 782a9c6ec057..ddcc2a7379b6 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -20,6 +20,7 @@ * * \author Pablo Rincon Crespo * \author Eric Leblond + * \author Jeff Lucovsky * * App Layer Parser for FTP */ @@ -63,6 +64,60 @@ #include "output-json.h" +const FtpCommand FtpCommands[FTP_COMMAND_MAX + 1] = { + /* Parsed and handled */ + { FTP_COMMAND_PORT, "PORT", "port", 4}, + { FTP_COMMAND_EPRT, "EPRT", "eprt", 4}, + { FTP_COMMAND_AUTH_TLS, "AUTH TLS", "auth tls", 8}, + { FTP_COMMAND_PASV, "PASV", "pasv", 4}, + { FTP_COMMAND_RETR, "RETR", "retr", 4}, + { FTP_COMMAND_EPSV, "EPSV", "epsv", 4}, + { FTP_COMMAND_STOR, "STOR", "stor", 4}, + + /* Parsed, but not handled */ + { FTP_COMMAND_ABOR, "ABOR", "abor", 4}, + { FTP_COMMAND_ACCT, "ACCT", "acct", 4}, + { FTP_COMMAND_ALLO, "ALLO", "allo", 4}, + { FTP_COMMAND_APPE, "APPE", "appe", 4}, + { FTP_COMMAND_CDUP, "CDUP", "cdup", 4}, + { FTP_COMMAND_CHMOD, "CHMOD", "chmod", 5}, + { FTP_COMMAND_CWD, "CWD", "cwd", 3}, + { FTP_COMMAND_DELE, "DELE", "dele", 4}, + { FTP_COMMAND_HELP, "HELP", "help", 4}, + { FTP_COMMAND_IDLE, "IDLE", "idle", 4}, + { FTP_COMMAND_LIST, "LIST", "list", 4}, + { FTP_COMMAND_MAIL, "MAIL", "mail", 4}, + { FTP_COMMAND_MDTM, "MDTM", "mdtm", 4}, + { FTP_COMMAND_MKD, "MKD", "mkd", 3}, + { FTP_COMMAND_MLFL, "MLFL", "mlfl", 4}, + { FTP_COMMAND_MODE, "MODE", "mode", 4}, + { FTP_COMMAND_MRCP, "MRCP", "mrcp", 4}, + { FTP_COMMAND_MRSQ, "MRSQ", "mrsq", 4}, + { FTP_COMMAND_MSAM, "MSAM", "msam", 4}, + { FTP_COMMAND_MSND, "MSND", "msnd", 4}, + { FTP_COMMAND_MSOM, "MSOM", "msom", 4}, + { FTP_COMMAND_NLST, "NLST", "nlst", 4}, + { FTP_COMMAND_NOOP, "NOOP", "noop", 4}, + { FTP_COMMAND_PASS, "PASS", "pass", 4}, + { FTP_COMMAND_PWD, "PWD", "pwd", 3}, + { FTP_COMMAND_QUIT, "QUIT", "quit", 4}, + { FTP_COMMAND_REIN, "REIN", "rein", 4}, + { FTP_COMMAND_REST, "REST", "rest", 4}, + { FTP_COMMAND_RMD, "RMD", "rmd", 3}, + { FTP_COMMAND_RNFR, "RNFR", "rnfr", 4}, + { FTP_COMMAND_RNTO, "RNTO", "rnto", 4}, + { FTP_COMMAND_SITE, "SITE", "site", 4}, + { FTP_COMMAND_SIZE, "SIZE", "size", 4}, + { FTP_COMMAND_SMNT, "SMNT", "smnt", 4}, + { FTP_COMMAND_STAT, "STAT", "stat", 4}, + { FTP_COMMAND_STOU, "STOU", "stou", 4}, + { FTP_COMMAND_STRU, "STRU", "stru", 4}, + { FTP_COMMAND_SYST, "SYST", "syst", 4}, + { FTP_COMMAND_TYPE, "TYPE", "type", 4}, + { FTP_COMMAND_UMASK, "UMASK", "umask", 5}, + { FTP_COMMAND_USER, "USER", "user", 4}, + { FTP_COMMAND_UNKNOWN, NULL, NULL, 0} +}; uint64_t ftp_config_memcap = 0; SC_ATOMIC_DECLARE(uint64_t, ftp_memuse); @@ -190,6 +245,61 @@ static void FTPFree(void *ptr, size_t size) FTPDecrMemuse((uint64_t)size); } +static FTPString *FTPStringAlloc(void) +{ + return FTPCalloc(1, sizeof(FTPString)); +} + +static void FTPStringFree(FTPString *str) +{ + if (str->str) { + FTPFree(str->str, str->len); + } + + FTPFree(str, sizeof(FTPString)); +} + +static FTPTransaction *FTPTransactionCreate(FtpState *state) +{ + SCEnter(); + FTPTransaction *tx = FTPCalloc(1, sizeof(*tx)); + if (tx == NULL) { + return NULL; + } + + TAILQ_INSERT_TAIL(&state->tx_list, tx, next); + tx->tx_id = state->tx_cnt++; + + TAILQ_INIT(&tx->response_list); + + SCLogDebug("new transaction %p (state tx cnt %"PRIu64")", tx, state->tx_cnt); + return tx; +} + +static void FTPTransactionFree(FTPTransaction *tx) +{ + SCEnter(); + + if (tx->de_state != NULL) { + DetectEngineStateFree(tx->de_state); + } + + if (tx->request) { + FTPFree(tx->request, tx->request_length); + } + if (tx->response) { + FTPFree(tx->response, tx->response_length); + } + + FTPString *str = NULL; + while ((str = TAILQ_FIRST(&tx->response_list))) { + TAILQ_REMOVE(&tx->response_list, str, next); + FTPStringFree(str); + } + + SCFree(tx); +} + static int FTPGetLineForDirection(FtpState *state, FtpLineState *line_state) { void *ptmp; @@ -312,49 +422,32 @@ static int FTPGetLine(FtpState *state) /** * \brief This function is called to determine and set which command is being - * transfered to the ftp server - * \param ftp_state the ftp state structure for the parser + * transferred to the ftp server * \param input input line of the command * \param len of the command + * \param cmd_descriptor when the command has been parsed * * \retval 1 when the command is parsed, 0 otherwise */ -static int FTPParseRequestCommand(void *ftp_state, uint8_t *input, - uint32_t input_len) +static int FTPParseRequestCommand(uint8_t *input, uint32_t input_len, const FtpCommand **cmd_descriptor) { SCEnter(); - FtpState *fstate = (FtpState *)ftp_state; - fstate->command = FTP_COMMAND_UNKNOWN; - - if (input_len >= 4 && SCMemcmpLowercase("port", input, 4) == 0) { - fstate->command = FTP_COMMAND_PORT; - } - - if (input_len >= 4 && SCMemcmpLowercase("eprt", input, 4) == 0) { - fstate->command = FTP_COMMAND_EPRT; - } - if (input_len >= 8 && SCMemcmpLowercase("auth tls", input, 8) == 0) { - fstate->command = FTP_COMMAND_AUTH_TLS; - } - - if (input_len >= 4 && SCMemcmpLowercase("pasv", input, 4) == 0) { - fstate->command = FTP_COMMAND_PASV; - } + *cmd_descriptor = NULL; - if (input_len > 5 && SCMemcmpLowercase("retr", input, 4) == 0) { - fstate->command = FTP_COMMAND_RETR; - } - - if (input_len >= 4 && SCMemcmpLowercase("epsv", input, 4) == 0) { - fstate->command = FTP_COMMAND_EPSV; - } + for (int i = 0; i < FTP_COMMAND_MAX; i++) { + if (!FtpCommands[i].command_length) { + break; + } + if (input_len >= FtpCommands[i].command_length && + SCMemcmpLowercase(FtpCommands[i].command_name_lower, + input, FtpCommands[i].command_length) == 0) { - if (input_len > 5 && SCMemcmpLowercase("stor", input, 4) == 0) { - fstate->command = FTP_COMMAND_STOR; + *cmd_descriptor = &FtpCommands[i]; + return 1; + } } - - return 1; + return 0; } struct FtpTransferCmd { @@ -378,6 +471,26 @@ static void FtpTransferCmdFree(void *data) FTPFree(cmd, sizeof(struct FtpTransferCmd)); } +static uint32_t CopyCommandLine(uint8_t **dest, uint8_t *src, uint32_t length) +{ + if (likely(length)) { + uint8_t *where = FTPCalloc(length, sizeof(char)); + if (unlikely(where == NULL)) { + return 0; + } + memcpy(where, src, length); + + /* Remove trailing newlines/carriage returns */ + if (isspace((unsigned char)where[length - 1])) { + while(length && isspace((unsigned char)where[--length])); + where[length] = '\0'; + } + *dest = where; + } + /* either 0 or actual */ + return length; +} + static uint16_t ftp_validate_port(int computed_port_value) { unsigned int port_val = computed_port_value; @@ -470,9 +583,30 @@ static int FTPParseRequest(Flow *f, void *ftp_state, int direction = STREAM_TOSERVER; while (FTPGetLine(state) >= 0) { - FTPParseRequestCommand(state, - state->current_line, state->current_line_len); + const FtpCommand *cmd_descriptor; + + if (!FTPParseRequestCommand(state->current_line, state->current_line_len, &cmd_descriptor)) { + state->command = FTP_COMMAND_UNKNOWN; + continue; + } + + state->command = cmd_descriptor->command; + + FTPTransaction *tx = state->curr_tx; + if (tx == NULL || tx->done) { + tx = FTPTransactionCreate(state); + if (unlikely(tx == NULL)) + return -1; + state->curr_tx = tx; + } + + tx->command_descriptor = cmd_descriptor; + tx->request_length = CopyCommandLine(&tx->request, state->current_line, state->current_line_len); + switch (state->command) { + case FTP_COMMAND_QUIT: + tx->done = true; + break; case FTP_COMMAND_EPRT: // fallthrough case FTP_COMMAND_PORT: @@ -563,6 +697,8 @@ static int FTPParsePassiveResponse(Flow *f, FtpState *state, uint8_t *input, uin SCLogDebug("FTP passive mode (v4): dynamic port %"PRIu16"", dyn_port); state->active = false; state->dyn_port = dyn_port; + state->curr_tx->dyn_port = dyn_port; + state->curr_tx->active = false; return 0; } @@ -581,6 +717,8 @@ static int FTPParsePassiveResponseV6(Flow *f, FtpState *state, uint8_t *input, u SCLogDebug("FTP passive mode (v6): dynamic port %"PRIu16"", dyn_port); state->active = false; state->dyn_port = dyn_port; + state->curr_tx->dyn_port = dyn_port; + state->curr_tx->active = false; return 0; } @@ -598,6 +736,12 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat void *local_data, const uint8_t flags) { FtpState *state = (FtpState *)ftp_state; + FTPTransaction *tx = state->curr_tx; + int retcode = 1; + + if (state->command == FTP_COMMAND_UNKNOWN) { + return 1; + } if (state->command == FTP_COMMAND_AUTH_TLS) { if (input_len >= 4 && SCMemcmp("234 ", input, 4) == 0) { @@ -608,10 +752,13 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat if (state->command == FTP_COMMAND_EPRT) { uint16_t dyn_port = FTPGetV6PortNumber(state->port_line, state->port_line_len); if (dyn_port == 0) { - return 0; + retcode = 0; + goto tx_complete; } state->dyn_port = dyn_port; state->active = true; + tx->dyn_port = dyn_port; + tx->active = true; SCLogDebug("FTP active mode (v6): dynamic port %"PRIu16"", dyn_port); } @@ -619,13 +766,17 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat if ((flags & STREAM_TOCLIENT)) { uint16_t dyn_port = FTPGetV4PortNumber(state->port_line, state->port_line_len); if (dyn_port == 0) { - return 0; + retcode = 0; + goto tx_complete; } state->dyn_port = dyn_port; state->active = true; + tx->dyn_port = state->dyn_port; + tx->active = true; SCLogDebug("FTP active mode (v4): dynamic port %"PRIu16"", dyn_port); } } + if (state->command == FTP_COMMAND_PASV) { if (input_len >= 4 && SCMemcmp("227 ", input, 4) == 0) { FTPParsePassiveResponse(f, ftp_state, input, input_len); @@ -638,7 +789,18 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat } } - return 1; + if (likely(input_len)) { + FTPString *response = FTPStringAlloc(); + if (likely(response)) { + response->len = CopyCommandLine(&response->str, input, input_len); + if (response->str) + TAILQ_INSERT_TAIL(&tx->response_list, response, next); + } + } + +tx_complete: + tx->done = true; + return retcode; } #ifdef DEBUG @@ -649,11 +811,12 @@ static uint64_t ftp_state_memcnt = 0; static void *FTPStateAlloc(void) { - void *s = FTPMalloc(sizeof(FtpState)); + void *s = FTPCalloc(1, sizeof(FtpState)); if (unlikely(s == NULL)) return NULL; - memset(s, 0, sizeof(FtpState)); + FtpState *ftp_state = (FtpState *) s; + TAILQ_INIT(&ftp_state->tx_list); #ifdef DEBUG SCMutexLock(&ftp_state_mem_lock); @@ -676,8 +839,10 @@ static void FTPStateFree(void *s) //AppLayerDecoderEventsFreeEvents(&s->decoder_events); - if (fstate->de_state != NULL) { - DetectEngineStateFree(fstate->de_state); + FTPTransaction *tx = NULL; + while ((tx = TAILQ_FIRST(&fstate->tx_list))) { + TAILQ_REMOVE(&fstate->tx_list, tx, next); + FTPTransactionFree(tx); } FTPFree(s, sizeof(FtpState)); @@ -691,32 +856,85 @@ static void FTPStateFree(void *s) static int FTPSetTxDetectState(void *vtx, DetectEngineState *de_state) { - FtpState *ftp_state = (FtpState *)vtx; - ftp_state->de_state = de_state; + FTPTransaction *tx = (FTPTransaction *)vtx; + tx->de_state = de_state; return 0; } +static void *FTPGetTx(void *state, uint64_t tx_id) +{ + FtpState *ftp_state = (FtpState *)state; + if (ftp_state) { + FTPTransaction *tx = NULL; + + if (ftp_state->curr_tx == NULL) + return NULL; + if (ftp_state->curr_tx->tx_id == tx_id) + return ftp_state->curr_tx; + + TAILQ_FOREACH(tx, &ftp_state->tx_list, next) { + if (tx->tx_id == tx_id) + return tx; + } + } + return NULL; + +} + static DetectEngineState *FTPGetTxDetectState(void *vtx) { - FtpState *ftp_state = (FtpState *)vtx; - return ftp_state->de_state; + FTPTransaction *tx = (FTPTransaction *)vtx; + return tx->de_state; } -static void FTPStateTransactionFree(void *state, uint64_t tx_id) + +static uint64_t FTPGetTxDetectFlags(void *vtx, uint8_t dir) { - /* do nothing */ + FTPTransaction *tx = (FTPTransaction *)vtx; + if (dir & STREAM_TOSERVER) { + return tx->detect_flags_ts; + } else { + return tx->detect_flags_tc; + } } -static void *FTPGetTx(void *state, uint64_t tx_id) +static void FTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t flags) { - FtpState *ftp_state = (FtpState *)state; - return ftp_state; + FTPTransaction *tx = (FTPTransaction *)vtx; + if (dir & STREAM_TOSERVER) { + tx->detect_flags_ts = flags; + } else { + tx->detect_flags_tc = flags; + } +} + +static void FTPStateTransactionFree(void *state, uint64_t tx_id) +{ + FtpState *ftp_state = state; + FTPTransaction *tx = NULL; + TAILQ_FOREACH(tx, &ftp_state->tx_list, next) { + if (tx_id < tx->tx_id) + break; + else if (tx_id > tx->tx_id) + continue; + + if (tx == ftp_state->curr_tx) + ftp_state->curr_tx = NULL; + TAILQ_REMOVE(&ftp_state->tx_list, tx, next); + FTPTransactionFree(tx); + break; + } } static uint64_t FTPGetTxCnt(void *state) { - /* single tx */ - return 1; + uint64_t cnt = 0; + FtpState *ftp_state = state; + if (ftp_state) { + cnt = ftp_state->tx_cnt; + } + SCLogDebug("returning state %p %"PRIu64, state, cnt); + return cnt; } static int FTPGetAlstateProgressCompletionStatus(uint8_t direction) @@ -724,12 +942,13 @@ static int FTPGetAlstateProgressCompletionStatus(uint8_t direction) return FTP_STATE_FINISHED; } -static int FTPGetAlstateProgress(void *tx, uint8_t direction) +static int FTPGetAlstateProgress(void *vtx, uint8_t direction) { - FtpState *ftp_state = (FtpState *)tx; + SCLogDebug("tx %p", vtx); + FTPTransaction *tx = vtx; if (direction == STREAM_TOSERVER && - ftp_state->command == FTP_COMMAND_PORT) { + tx->command_descriptor->command == FTP_COMMAND_PORT) { return FTP_STATE_PORT_DONE; } @@ -860,6 +1079,17 @@ static int FTPDataParse(Flow *f, FtpDataState *ftpdata_state, return ret; } +static void FTPStateSetTxLogged(void *state, void *vtx, LoggerId logged) +{ + FTPTransaction *tx = vtx; + tx->logged = logged; +} + +static LoggerId FTPStateGetTxLogged(void *state, void *vtx) +{ + FTPTransaction *tx = vtx; + return tx->logged; +} static int FTPDataParseRequest(Flow *f, void *ftp_state, AppLayerParserState *pstate, uint8_t *input, uint32_t input_len, @@ -886,12 +1116,12 @@ static uint64_t ftpdata_state_memcnt = 0; static void *FTPDataStateAlloc(void) { - void *s = FTPMalloc(sizeof(FtpDataState)); + void *s = FTPCalloc(1, sizeof(FtpDataState)); if (unlikely(s == NULL)) return NULL; - memset(s, 0, sizeof(FtpDataState)); - ((FtpDataState *)s)->state = FTPDATA_STATE_IN_PROGRESS; + FtpDataState *state = (FtpDataState *) s; + state->state = FTPDATA_STATE_IN_PROGRESS; #ifdef DEBUG SCMutexLock(&ftpdata_state_mem_lock); @@ -935,6 +1165,7 @@ static DetectEngineState *FTPDataGetTxDetectState(void *vtx) { FtpDataState *ftp_state = (FtpDataState *)vtx; return ftp_state->de_state; + } static void FTPDataStateTransactionFree(void *state, uint64_t tx_id) @@ -1001,7 +1232,12 @@ void RegisterFTPParsers(void) AppLayerParserRegisterDetectStateFuncs(IPPROTO_TCP, ALPROTO_FTP, FTPGetTxDetectState, FTPSetTxDetectState); + AppLayerParserRegisterDetectFlagsFuncs(IPPROTO_TCP, ALPROTO_FTP, + FTPGetTxDetectFlags, FTPSetTxDetectFlags); + AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_FTP, FTPGetTx); + AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_FTP, FTPStateGetTxLogged, + FTPStateSetTxLogged); AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_FTP, FTPGetTxCnt); @@ -1145,7 +1381,7 @@ static int FTPParserTest01(void) return result; } -/** \test Send a splitted get request. */ +/** \test Send a split get request. */ static int FTPParserTest03(void) { int result = 1; diff --git a/src/app-layer-ftp.h b/src/app-layer-ftp.h index 144937065cf8..cd379186ef89 100644 --- a/src/app-layer-ftp.h +++ b/src/app-layer-ftp.h @@ -79,9 +79,21 @@ typedef enum { FTP_COMMAND_TYPE, FTP_COMMAND_UMASK, FTP_COMMAND_USER, - FTP_COMMAND_EPRT + FTP_COMMAND_EPRT, + + /* must be last */ + FTP_COMMAND_MAX /** \todo more if missing.. */ } FtpRequestCommand; + +typedef struct FtpCommand_ { + FtpRequestCommand command; + const char *command_name_upper; + const char *command_name_lower; + const uint32_t command_length; +} FtpCommand; +extern const FtpCommand FtpCommands[FTP_COMMAND_MAX + 1]; + typedef uint32_t FtpRequestCommandArgOfs; typedef uint16_t FtpResponseCode; @@ -111,6 +123,42 @@ typedef struct FtpLineState_ { uint8_t current_line_lf_seen; } FtpLineState; +typedef struct FTPString_ { + uint8_t *str; + uint16_t len; + TAILQ_ENTRY(FTPString_) next; +} FTPString; + +typedef struct FTPTransaction_ { + /** id of this tx, starting at 0 */ + uint64_t tx_id; + + uint64_t detect_flags_ts; + uint64_t detect_flags_tc; + + /** indicates loggers done logging */ + uint32_t logged; + bool done; + + const FtpCommand *command_descriptor; + + uint8_t direction; + uint16_t dyn_port; + bool active; + + uint8_t *request; + uint32_t request_length; + + /* Handle multiple responses */ + TAILQ_HEAD(, FTPString_) response_list; + uint8_t *response; + uint32_t response_length; + + DetectEngineState *de_state; + + TAILQ_ENTRY(FTPTransaction_) next; +} FTPTransaction; + /** FTP State for app layer parser */ typedef struct FtpState_ { uint8_t *input; @@ -118,6 +166,10 @@ typedef struct FtpState_ { uint8_t direction; bool active; + FTPTransaction *curr_tx; + TAILQ_HEAD(, FTPTransaction_) tx_list; /**< transaction list */ + uint64_t tx_cnt; + /* --parser details-- */ /** current line extracted by the parser from the call to FTPGetline() */ uint8_t *current_line; @@ -138,7 +190,6 @@ typedef struct FtpState_ { /* specifies which loggers are done logging */ uint32_t logged; - DetectEngineState *de_state; } FtpState; enum { diff --git a/src/output-json-ftp.c b/src/output-json-ftp.c new file mode 100644 index 000000000000..a257cc11b6be --- /dev/null +++ b/src/output-json-ftp.c @@ -0,0 +1,247 @@ +/* Copyright (C) 2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jeff Lucovsky + * + * Implement JSON/eve logging app-layer FTP. + */ + + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "pkt-var.h" +#include "conf.h" + +#include "threads.h" +#include "threadvars.h" +#include "tm-threads.h" + +#include "util-unittest.h" +#include "util-buffer.h" +#include "util-debug.h" +#include "util-byte.h" + +#include "output.h" +#include "output-json.h" + +#include "app-layer.h" +#include "app-layer-parser.h" + +#include "app-layer-ftp.h" +#include "output-json-ftp.h" + +#ifdef HAVE_LIBJANSSON + +typedef struct LogFTPFileCtx_ { + LogFileCtx *file_ctx; + OutputJsonCommonSettings cfg; +} LogFTPFileCtx; + +typedef struct LogFTPLogThread_ { + LogFTPFileCtx *ftplog_ctx; + uint32_t count; + MemBuffer *buffer; +} LogFTPLogThread; + +static void JsonFTPLogJSON(json_t *tjs, Flow *f, FTPTransaction *tx) +{ + json_t *cjs = NULL; + if (f->alproto == ALPROTO_FTPDATA) { + cjs = JsonFTPDataAddMetadata(f); + } else if (tx->command_descriptor->command != FTP_COMMAND_UNKNOWN) { + cjs = json_object(); + if (cjs) { + FTPString *response; + json_object_set_new(cjs, "command", json_string(tx->command_descriptor->command_name_upper)); + uint32_t min_length = tx->command_descriptor->command_length + 1; /* command + space */ + if (tx->request_length >= min_length) { + json_object_set_new(cjs, "command_data", + JsonAddStringN((const char *)tx->request + min_length, + tx->request_length - min_length)); + } + if (!TAILQ_EMPTY(&tx->response_list)) { + json_t *js_resplist = json_array(); + if (likely(js_resplist != NULL)) { + json_t *resp_code = NULL; + TAILQ_FOREACH(response, &tx->response_list, next) { + if (!resp_code) { + /* the first completion codes with multiple response lines is definitive */ + resp_code = JsonAddStringN((const char *)response->str, 3); + } + /* move past 3 character completion code */ + if (response->len >= 4) { + json_array_append_new(js_resplist, + JsonAddStringN((const char *)response->str + 4, + response->len - 4)); + } + } + + json_object_set_new(cjs, "reply", js_resplist); + + if (resp_code) { + json_object_set_new(cjs, "completion_code", resp_code); + } + } + } + if (tx->dyn_port) { + json_object_set_new(cjs, "dynamic_port", json_integer(tx->dyn_port)); + } + if (tx->command_descriptor->command == FTP_COMMAND_PORT || + tx->command_descriptor->command == FTP_COMMAND_EPRT) { + json_object_set_new(cjs, "mode", + json_string((char*)(tx->active ? "active" : "passive"))); + } + } + } + + if (cjs) { + json_object_set_new(tjs, f->alproto == ALPROTO_FTP ? "ftp" : "ftp_data", cjs); + } +} + +static int JsonFTPLogger(ThreadVars *tv, void *thread_data, + const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id) +{ + SCEnter(); + + FTPTransaction *tx = vtx; + LogFTPLogThread *thread = thread_data; + LogFTPFileCtx *ftp_ctx = thread->ftplog_ctx; + + json_t *js = CreateJSONHeaderWithTxId(p, LOG_DIR_FLOW, + f->alproto == ALPROTO_FTP ? "ftp" : "ftp_data", + tx_id); + if (likely(js)) { + JsonAddCommonOptions(&ftp_ctx->cfg, p, f, js); + JsonFTPLogJSON(js, f, tx); + + MemBufferReset(thread->buffer); + OutputJSONBuffer(js, thread->ftplog_ctx->file_ctx, &thread->buffer); + + json_object_clear(js); + json_decref(js); + } + return TM_ECODE_OK; +} + +static void OutputFTPLogDeInitCtxSub(OutputCtx *output_ctx) +{ + LogFTPFileCtx *ftplog_ctx = (LogFTPFileCtx *)output_ctx->data; + SCFree(ftplog_ctx); + SCFree(output_ctx); +} + + +static OutputInitResult OutputFTPLogInitSub(ConfNode *conf, + OutputCtx *parent_ctx) +{ + OutputInitResult result = { NULL, false }; + OutputJsonCtx *ajt = parent_ctx->data; + + LogFTPFileCtx *ftplog_ctx = SCCalloc(1, sizeof(*ftplog_ctx)); + if (unlikely(ftplog_ctx == NULL)) { + return result; + } + ftplog_ctx->file_ctx = ajt->file_ctx; + ftplog_ctx->cfg = ajt->cfg; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx)); + if (unlikely(output_ctx == NULL)) { + SCFree(ftplog_ctx); + return result; + } + output_ctx->data = ftplog_ctx; + output_ctx->DeInit = OutputFTPLogDeInitCtxSub; + + SCLogDebug("FTP log sub-module initialized."); + + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTP); + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTPDATA); + + result.ctx = output_ctx; + result.ok = true; + return result; +} + +#define OUTPUT_BUFFER_SIZE 65535 +static TmEcode JsonFTPLogThreadInit(ThreadVars *t, const void *initdata, void **data) +{ + LogFTPLogThread *thread = SCCalloc(1, sizeof(*thread)); + if (unlikely(thread == NULL)) { + return TM_ECODE_FAILED; + } + + if (initdata == NULL) { + SCLogDebug("Error getting context for EveLogFTP. \"initdata\" is NULL."); + SCFree(thread); + return TM_ECODE_FAILED; + } + + thread->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (unlikely(thread->buffer == NULL)) { + SCFree(thread); + return TM_ECODE_FAILED; + } + + thread->ftplog_ctx = ((OutputCtx *)initdata)->data; + *data = (void *)thread; + + return TM_ECODE_OK; +} + +static TmEcode JsonFTPLogThreadDeinit(ThreadVars *t, void *data) +{ + LogFTPLogThread *thread = (LogFTPLogThread *)data; + if (thread == NULL) { + return TM_ECODE_OK; + } + if (thread->buffer != NULL) { + MemBufferFree(thread->buffer); + } + /* clear memory */ + memset(thread, 0, sizeof(LogFTPLogThread)); + SCFree(thread); + return TM_ECODE_OK; +} + +void JsonFTPLogRegister(void) +{ + /* Register as an eve sub-module. */ + OutputRegisterTxSubModule(LOGGER_JSON_FTP, "eve-log", "JsonFTPLog", + "eve-log.ftp", OutputFTPLogInitSub, + ALPROTO_FTP, JsonFTPLogger, + JsonFTPLogThreadInit, JsonFTPLogThreadDeinit, + NULL); + OutputRegisterTxSubModule(LOGGER_JSON_FTP, "eve-log", "JsonFTPLog", + "eve-log.ftp", OutputFTPLogInitSub, + ALPROTO_FTPDATA, JsonFTPLogger, + JsonFTPLogThreadInit, JsonFTPLogThreadDeinit, + NULL); + + SCLogDebug("FTP JSON logger registered."); +} +#else /* HAVE_LIBJANSSON */ + +void JsonFTPLogRegister(void) +{ +} + +#endif /* HAVE_LIBJANSSON */ diff --git a/src/output-json-ftp.h b/src/output-json-ftp.h new file mode 100644 index 000000000000..acba5539e1c6 --- /dev/null +++ b/src/output-json-ftp.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2019 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jeff Lucovsky + */ + +#ifndef __OUTPUT_JSON_FTP_H__ +#define __OUTPUT_JSON_FTP_H__ + +void JsonFTPLogRegister(void); + +#endif /* __OUTPUT_JSON_FTP_H__ */ diff --git a/src/output-json.c b/src/output-json.c index b44ee79c727d..541b36813e54 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -148,6 +148,15 @@ json_t *SCJsonString(const char *val) /* Default Sensor ID value */ static int64_t sensor_id = -1; /* -1 = not defined */ +json_t *JsonAddStringN(const char *string, size_t size) +{ + char tmpbuf[size + 1]; + + memcpy(tmpbuf, string, size); + tmpbuf[size] = '\0'; + + return SCJsonString(tmpbuf); +} static void JsonAddPacketvars(const Packet *p, json_t *js_vars) { if (p == NULL || p->pktvar == NULL) { diff --git a/src/output-json.h b/src/output-json.h index 73eddf88ae0a..a7d996e4277a 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -88,6 +88,7 @@ typedef struct OutputJsonThreadCtx_ { json_t *SCJsonBool(int val); json_t *SCJsonString(const char *val); +json_t *JsonAddStringN(const char *string, size_t size); void SCJsonDecref(json_t *js); void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg, diff --git a/src/output.c b/src/output.c index 576183b35c6b..5b99a0414b67 100644 --- a/src/output.c +++ b/src/output.c @@ -68,6 +68,7 @@ #include "log-stats.h" #include "output-json.h" #include "output-json-nfs.h" +#include "output-json-ftp.h" #include "output-json-tftp.h" #include "output-json-smb.h" #include "output-json-ikev2.h" @@ -1095,6 +1096,8 @@ void OutputRegisterLoggers(void) JsonNFSLogRegister(); /* TFTP JSON logger. */ JsonTFTPLogRegister(); + /* FTP JSON logger. */ + JsonFTPLogRegister(); /* SMB JSON logger. */ JsonSMBLogRegister(); /* IKEv2 JSON logger. */ diff --git a/src/suricata-common.h b/src/suricata-common.h index 788e6037c246..4942e325b8f9 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -431,6 +431,7 @@ typedef enum { LOGGER_JSON_TLS, LOGGER_JSON_NFS, LOGGER_JSON_TFTP, + LOGGER_JSON_FTP, LOGGER_JSON_DNP3_TS, LOGGER_JSON_DNP3_TC, LOGGER_JSON_SSH, diff --git a/src/util-error.c b/src/util-error.c index a77519271be5..592659d9467b 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -106,6 +106,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_DEBUG_LOG_GENERIC); CASE_CODE (SC_ERR_UNIFIED_LOG_GENERIC); CASE_CODE (SC_ERR_HTTP_LOG_GENERIC); + CASE_CODE (SC_ERR_FTP_LOG_GENERIC); CASE_CODE (SC_ERR_UNIFIED_ALERT_GENERIC); CASE_CODE (SC_ERR_UNIFIED2_ALERT_GENERIC); CASE_CODE (SC_ERR_FWRITE); diff --git a/src/util-error.h b/src/util-error.h index 76debeca71ef..ee8363857e75 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -350,6 +350,7 @@ typedef enum { SC_WARN_DEFAULT_WILL_CHANGE, SC_WARN_EVE_MISSING_EVENTS, SC_ERR_PLEDGE_FAILED, + SC_ERR_FTP_LOG_GENERIC, SC_ERR_MAX, } SCError; diff --git a/src/util-profiling.c b/src/util-profiling.c index 91a6e6bd11f0..25fbef251bf4 100644 --- a/src/util-profiling.c +++ b/src/util-profiling.c @@ -1313,6 +1313,7 @@ const char * PacketProfileLoggertIdToString(LoggerId id) CASE_CODE (LOGGER_JSON_DHCP); CASE_CODE (LOGGER_JSON_KRB5); CASE_CODE (LOGGER_JSON_IKEV2); + CASE_CODE (LOGGER_JSON_FTP); CASE_CODE (LOGGER_JSON_TFTP); CASE_CODE (LOGGER_JSON_SMTP); CASE_CODE (LOGGER_JSON_TLS); diff --git a/suricata.yaml.in b/suricata.yaml.in index 091cbd96c1e6..e7022fc13fca 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -225,6 +225,7 @@ outputs: #md5: [body, subject] #- dnp3 + #- ftp - nfs - smb - tftp