diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 9555ae20797c..145401b09cdb 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1014,6 +1014,7 @@ jobs: libjansson-dev \ libpython2.7 \ make \ + mscgen \ parallel \ python3-yaml \ rustc \ diff --git a/configure.ac b/configure.ac index 30d2b4f09300..2e54f1fb229f 100644 --- a/configure.ac +++ b/configure.ac @@ -2322,6 +2322,18 @@ fi AC_DEFINE([CLS],[64],[L1 cache line size]) fi +# mscgen for devguide images + AC_PATH_PROG([HAVE_MSCGEN], mscgen, "no") + if test "$HAVE_MSCGEN" = "no"; then + enable_mscgen=no + echo "WARNING! mscgen package not installed." + echo " Devguide images won't be generated!" + echo " Get mscgen package:" + echo " https://www.mcternan.me.uk/mscgen/" + echo " or install it from your distribution" + fi + AM_CONDITIONAL([HAVE_MSCGEN], [test "x$enable_mscgen" != "xno" ]) + # sphinx for documentation AC_PATH_PROG(HAVE_SPHINXBUILD, sphinx-build, "no") if test "$HAVE_SPHINXBUILD" = "no"; then diff --git a/doc/devguide/.gitignore b/doc/devguide/.gitignore index e35d8850c968..cfc63d82f814 100644 --- a/doc/devguide/.gitignore +++ b/doc/devguide/.gitignore @@ -1 +1,2 @@ _build +extending/app-layer/img/*.png diff --git a/doc/devguide/Makefile.am b/doc/devguide/Makefile.am index 44317c606d93..cfc15ca0735d 100644 --- a/doc/devguide/Makefile.am +++ b/doc/devguide/Makefile.am @@ -7,6 +7,7 @@ EXTRA_DIST = \ extending/index.rst \ extending/app-layer/index.rst \ extending/app-layer/parser.rst \ + extending/app-layer/transactions.rst \ extending/capture/index.rst \ extending/output/index.rst \ internals/engines/index.rst \ @@ -22,6 +23,7 @@ EXTRA_DIST = \ codebase/fuzz-testing.rst if HAVE_SPHINXBUILD +if HAVE_MSCGEN if HAVE_PDFLATEX EXTRA_DIST += devguide.pdf @@ -30,6 +32,7 @@ endif SPHINX_BUILD = sphinx-build -q html: + $(top_srcdir)/doc/devguide/extending/app-layer/img/generate-images.sh sysconfdir=$(sysconfdir) \ localstatedir=$(localstatedir) \ version=$(PACKAGE_VERSION) \ @@ -37,6 +40,7 @@ html: $(top_srcdir)/doc/devguide _build/html _build/latex/Suricata.pdf: + $(top_srcdir)/doc/devguide/extending/app-layer/img/generate-images.sh sysconfdir=$(sysconfdir) \ localstatedir=$(localstatedir) \ version=$(PACKAGE_VERSION) \ @@ -62,4 +66,5 @@ clean-local: rm -f $(top_builddir)/doc/devguide/suricata.1 rm -f $(top_builddir)/doc/devguide/devguide.pdf +endif # HAVE_MSCGEN endif # HAVE_SPHINXBUILD diff --git a/doc/devguide/Makefile.sphinx b/doc/devguide/Makefile.sphinx index 549ff8c07584..9002b14cc944 100644 --- a/doc/devguide/Makefile.sphinx +++ b/doc/devguide/Makefile.sphinx @@ -18,6 +18,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +BUILD_IMG = ./extending/app-layer/img/generate-images.sh .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text changes linkcheck doctest coverage gettext @@ -51,37 +52,44 @@ clean: rm -rf $(BUILDDIR)/* html: + $(BUILD_IMG) $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: + $(BUILD_IMG) $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: + $(BUILD_IMG) $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: + $(BUILD_IMG) $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: + $(BUILD_IMG) $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: + $(BUILD_IMG) $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: + $(BUILD_IMG) $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -91,6 +99,7 @@ qthelp: @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Suricata.qhc" applehelp: + $(BUILD_IMG) $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @@ -99,6 +108,7 @@ applehelp: "bundle." devhelp: + $(BUILD_IMG) $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @@ -108,11 +118,13 @@ devhelp: @echo "# devhelp" epub: + $(BUILD_IMG) $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: + $(BUILD_IMG) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -120,23 +132,27 @@ latex: "(use \`make latexpdf' here to do that automatically)." latexpdf: + $(BUILD_IMG) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: + $(BUILD_IMG) $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: + $(BUILD_IMG) $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." texinfo: + $(BUILD_IMG) $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @@ -144,12 +160,14 @@ texinfo: "(use \`make info' here to do that automatically)." info: + $(BUILD_IMG) $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: + $(BUILD_IMG) $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." diff --git a/doc/devguide/extending/app-layer/img/DnsRequestUnidirectionalTransaction.msc b/doc/devguide/extending/app-layer/img/DnsRequestUnidirectionalTransaction.msc new file mode 100644 index 000000000000..301151c78679 --- /dev/null +++ b/doc/devguide/extending/app-layer/img/DnsRequestUnidirectionalTransaction.msc @@ -0,0 +1,13 @@ +# MSC Sequence Diagram Example: DNS Query Transaction + +msc { + # Chart Options + arcgradient = "10"; + + # Entities + a [ label = "Client" ], b [ label = "Server" ]; + + # Message Flow + a =>> b [ label = "DNS Request" ]; + --- [ label = "Transaction Completed" ]; +} diff --git a/doc/devguide/extending/app-layer/img/HTTP2BidirectionalTransaction.msc b/doc/devguide/extending/app-layer/img/HTTP2BidirectionalTransaction.msc new file mode 100644 index 000000000000..0ba93e6511ba --- /dev/null +++ b/doc/devguide/extending/app-layer/img/HTTP2BidirectionalTransaction.msc @@ -0,0 +1,18 @@ +# MSC Sequence Diagram for an HTTP2 Transaction, which is bidirectional in Suricata + +msc { + + # Chart options + arcgradient = "10"; + + # Entities + a [ label = "Client" ], b [ label = "Server" ]; + + # Message flow + a =>> b [ label = "Request" ]; + b =>> a [ label = "Response" ]; + |||; + --- [ label = "Transaction Completed" ]; +} + +# Reference: https://tools.ietf.org/html/rfc7540#section-8.1 diff --git a/doc/devguide/extending/app-layer/img/TemplateTransaction.msc b/doc/devguide/extending/app-layer/img/TemplateTransaction.msc new file mode 100644 index 000000000000..73a82d15347e --- /dev/null +++ b/doc/devguide/extending/app-layer/img/TemplateTransaction.msc @@ -0,0 +1,15 @@ +# MSC Sequence Diagram Example: Template transaction + +msc { + # Chart Options + arcgradient = "10"; + + # Entities + a [ label = "Client" ], b [ label = "Server" ]; + + # Message Flow + a =>> b [ label = "Request ('12:HelloWorld!')" ]; + b =>> a [ label = "Response ('3:Bye')" ]; + |||; + --- [ label = "Transaction Completed" ]; +} diff --git a/doc/devguide/extending/app-layer/img/TlsHandshake.msc b/doc/devguide/extending/app-layer/img/TlsHandshake.msc new file mode 100644 index 000000000000..7f13bc93dbac --- /dev/null +++ b/doc/devguide/extending/app-layer/img/TlsHandshake.msc @@ -0,0 +1,21 @@ +# MSC Sequence Diagram Example: TLS Handshake Transaction + +msc { + # Chart Options + arcgradient = "10"; + + # Entities + a [ label = "Client" ], b [ label = "Server"]; + + # Message Flow + a =>> b [ label = "ClientHello"]; + b =>> a [ label = "ServerHello"]; + b =>> a [ label = "ServerCertificate"]; + b =>> a [ label = "ServerHello Done"]; + a =>> b [ label = "ClientCertificate"]; + a =>> b [ label = "ClientKeyExchange"]; + a =>> b [ label = "Finished" ]; + b =>> a [ label = "Finished" ]; + + --- [ label = "Transaction Completed" ]; +} diff --git a/doc/devguide/extending/app-layer/img/generate-images.sh b/doc/devguide/extending/app-layer/img/generate-images.sh new file mode 100755 index 000000000000..e5f8de0d1464 --- /dev/null +++ b/doc/devguide/extending/app-layer/img/generate-images.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Script to generate Sequence Diagram images with mscgen +# + +cd extending/app-layer/img + +for FILE in *.msc ; do + # call mscgen and convert each file in images dir + mscgen -T png -F Arial $FILE + # if command fails, lets inform about that + if [ $? -ne 0 ]; then + echo "$FILE couldn't be converted in the devguide" + fi +done + +exit 0 diff --git a/doc/devguide/extending/app-layer/index.rst b/doc/devguide/extending/app-layer/index.rst index 783f14e20e22..d3086b7908a0 100644 --- a/doc/devguide/extending/app-layer/index.rst +++ b/doc/devguide/extending/app-layer/index.rst @@ -5,3 +5,4 @@ App-Layer :maxdepth: 2 parser.rst + transactions.rst diff --git a/doc/devguide/extending/app-layer/transactions.rst b/doc/devguide/extending/app-layer/transactions.rst new file mode 100644 index 000000000000..a3f32139b987 --- /dev/null +++ b/doc/devguide/extending/app-layer/transactions.rst @@ -0,0 +1,299 @@ +************ +Transactions +************ + +.. contents:: Table of Contents + +_`General Concepts` +=================== + +Transactions are abstractions that help detecting and logging in Suricata. They also help during the detection phase, +when dealing with protocols that can have large PDUs, like TCP, in controlling state for partial rule matching, in case of rules that mention more than one field. + +Transactions are implemented and stored in the per-flow state. The engine interacts with them using a set of callbacks the parser registers. + +_`How the engine uses transactions` +=================================== + +Logging +~~~~~~~ + +Suricata controls when logging should happen based on transaction completeness. For simpler protocols, such as ``dns`` +or ``ntp``, that will most +likely happen once per transaction, by the time of its completion. In other cases, like with HTTP, this may happen at intermediary states. + +In ``OutputTxLog``, the engine will compare current state with the value defined for the logging to happen, per flow +direction (``logger->tc_log_progress``, ``logger->ts_log_progress``). If state is less than that value, the engine skips to +the next logger. Code snippet from: suricata/src/output-tx.c: + +.. code-block:: c + + static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) + { + . + . + . + if ((ts_eof && tc_eof) || last_pseudo) { + SCLogDebug("EOF, so log now"); + } else { + if (logger->LogCondition) { + int r = logger->LogCondition(tv, p, alstate, tx, tx_id); + if (r == FALSE) { + SCLogDebug("conditions not met, not logging"); + goto next_logger; + } + } else { + if (tx_progress_tc < logger->tc_log_progress) { + SCLogDebug("progress not far enough, not logging"); + goto next_logger; + } + + if (tx_progress_ts < logger->ts_log_progress) { + SCLogDebug("progress not far enough, not logging"); + goto next_logger; + } + } + } + . + . + . + } + +Rule Matching +~~~~~~~~~~~~~ + +Transaction progress is also used for certain keywords to know what is the minimum state before we can expect a match: until that, Suricata won't even try to look for the patterns. + +As seen in ``DetectAppLayerMpmRegister2`` that has ``int progress`` as parameter, and ``DetectAppLayerInspectEngineRegister2``, which expects ``int tx_min_progress``, for instance. In the code snippet, +``HTTP2StateDataClient``, ``HTTP2StateDataServer`` and ``0`` are the values passed to the functions. + + +.. code-block:: c + + void DetectFiledataRegister(void) + { + . + . + DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2, + PrefilterMpmFiledataRegister, NULL, + ALPROTO_HTTP2, HTTP2StateDataClient); + DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2, + PrefilterMpmFiledataRegister, NULL, + ALPROTO_HTTP2, HTTP2StateDataServer); + . + . + DetectAppLayerInspectEngineRegister2("file_data", + ALPROTO_HTTP2, SIG_FLAG_TOCLIENT, HTTP2StateDataServer, + DetectEngineInspectFiledata, NULL); + DetectAppLayerInspectEngineRegister2( + "file_data", ALPROTO_FTPDATA, SIG_FLAG_TOSERVER, 0, DetectEngineInspectFiledata, NULL); + . + . + } + +_`Progress Tracking` +==================== + +As a rule of thumb, transactions will follow a request-response model: if a transaction has had a request and a response, it is complete. + +But if a protocol has situations where a request or response won’t expect or generate a message from its counterpart, +it is also possible to have uni-directional transactions. In such cases, transaction is set to complete at the moment of +creation. + +For example, DNS responses may be considered as completed transactions, because they also contain the request data, so +all information needed for logging and detection can be found in the response. + +In addition, for file transfer protocols, or similar ones where there may be several messages before the file exchange +is completed (NFS, SMB), it is possible to create a level of abstraction to handle such complexity. This could be achieved by adding phases to the protocol implemented model (e.g., protocol negotiation phase (SMB), request parsed (HTTP), and so on). + +This is controlled by implementing states. In Suricata, those will be enums that are incremented as the parsing +progresses. A state will start at 0. The higher its value, the closer the transaction would be to completion. + +The engine interacts with transactions state using a set of callbacks the parser registers. State is defined per flow direction (``STREAM_TOSERVER`` / ``STREAM_TOCLIENT``). + +In Summary - Transactions and State +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Initial state value: ``0``. +- Simpler scenarios: state is simply an int. ``1`` represents transaction completion, per direction. +- Complex Transaction State in Suricata: ``enum`` (Rust: ``i32``). Completion is indicated by the highest enum value (some examples are: SSH, HTTP, DNS, SMB). + +_`Examples` +=========== + +This section shares some examples from Suricata codebase, to help visualize how Transactions work and are handled by the engine. + +Enums +~~~~~ + +Code snippet from: rust/src/ssh/ssh.rs: + +.. code-block:: rust + + pub enum SSHConnectionState { + SshStateInProgress = 0, + SshStateBannerWaitEol = 1, + SshStateBannerDone = 2, + SshStateFinished = 3, + } + +From src/app-layer-ftp.h: + +.. code-block:: c + + enum { + FTP_STATE_IN_PROGRESS, + FTP_STATE_PORT_DONE, + FTP_STATE_FINISHED, + }; + +API Callbacks +~~~~~~~~~~~~~ + +In Rust, this is done via the RustParser struct. As seen in rust/src/applayer.rs: + +.. code-block:: rust + + /// Rust parser declaration + pub struct RustParser { + . + . + . + /// Progress values at which the tx is considered complete in a direction + pub tx_comp_st_ts: c_int, + pub tx_comp_st_tc: c_int, + . + . + . + } + +In C, the callback API is: + +.. code-block:: c + + void AppLayerParserRegisterStateProgressCompletionStatus( + AppProto alproto, const int ts, const int tc) + +Simple scenario described, in Rust: + +rust/src/dhcp/dhcp.rs: + +.. code-block:: rust + + tx_comp_st_ts: 1 + tx_comp_st_tc: 1 + +For SSH, this looks like this: + +rust/src/ssh/ssh.rs: + +.. code-block:: rust + + tx_comp_st_ts: SSHConnectionState::SshStateFinished as i32, + tx_comp_st_tc: SSHConnectionState::SshStateFinished as i32, + +In C, callback usage would be as follows: + +src/app-layer-dcerpc.c: + +.. code-block:: c + + AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_DCERPC, 1, 1); + +src/app-layer-ftp.c: + +.. code-block:: c + + AppLayerParserRegisterStateProgressCompletionStatus( + ALPROTO_FTP, FTP_STATE_FINISHED, FTP_STATE_FINISHED); + +Sequence Diagrams +~~~~~~~~~~~~~~~~~ + +A DNS transaction in Suricata can be considered unidirectional: + +.. image:: img/DnsRequestUnidirectionalTransaction.png + :width: 600 + :alt: A sequence diagram with two entities, Client and Server, with an arrow going from the Client to the Server, labeled "DNS Request". After that, there is a dotted line labeled "Transaction Completed". + +An HTTP2 transaction is an example of a bidirectional transaction, in Suricata (note that transactions in HTTP2 may +overlap, scenario not shown in this Sequence Diagram): + +.. TODO add another example for overlapping HTTP2 transaction + +.. image:: img/HTTP2BidirectionalTransaction.png + :width: 600 + :alt: A sequence diagram with two entities, Client and Server, with an arrow going from the Client to the Server labeled "Request" and below that an arrow going from Server to Client labeled "Response". Below those arrows, a dotted line indicates that the transaction is completed. + +A TLS Handshake is a more complex example, where several messages are exchanged before the transaction is considered completed: + +.. image:: img/TlsHandshake.png + :width: 600 + :alt: A sequence diagram with two entities, Client and Server, with an arrow going from the Client to the Server labeled "ClientHello" and below that an arrow going from Server to Client labeled "ServerHello". Below those arrows, several more follow from Server to Client and vice-versa, before a dotted line indicates that the transaction is finally completed. + +Template Protocol +~~~~~~~~~~~~~~~~~ + +Suricata has a template protocol for educational purposes, which has simple bidirectional transactions. + +A completed transaction for the template looks like this: + +.. image:: img/TemplateTransaction.png + :width: 600 + :alt: A sequence diagram with two entities, Client and Server, with an arrow going from the Client to the Server, labeled "Request". An arrow below that first one goes from Server to Client. + +Following are the functions that check whether a transaction is considered completed, for the Template Protocol. Those are called by the Suricata API. Similar functions exist for each protocol, and may present implementation differences, based on what is considered a transaction for that given protocol. + +In C: + +.. code-block:: c + + static int TemplateGetStateProgress(void *txv, uint8_t direction) + { + TemplateTransaction *tx = txv; + + SCLogNotice("Transaction progress requested for tx ID %"PRIu64 + ", direction=0x%02x", tx->tx_id, direction); + + if (direction & STREAM_TOCLIENT && tx->response_done) { + return 1; + } + else if (direction & STREAM_TOSERVER) { + /* For the template, just the existence of the transaction means the + * request is done. */ + return 1; + } + + return 0; + } + +And in Rust: + +.. code-block:: rust + + pub extern "C" fn rs_template_tx_get_alstate_progress( + tx: *mut std::os::raw::c_void, + _direction: u8, + ) -> std::os::raw::c_int { + let tx = cast_pointer!(tx, TemplateTransaction); + + // Transaction is done if we have a response. + if tx.response.is_some() { + return 1; + } + return 0; + } + +_`Common words and abbreviations` +================================= + +- al, applayer: application layer +- alproto: application layer protocol +- alstate: application layer state +- engine: refers to Suricata core detection logic +- flow: a bidirectional flow of packets with the same 5-tuple elements (protocol, source ip, destination ip, source port, destination port. Vlans can be added as well) +- PDU: Protocol Data Unit +- rs: rust +- tc: to client +- ts: to server +- tx: transaction