From 777d7745bf134255c95101178c3c6b084141691a Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Sun, 13 Jun 2021 17:26:50 -0400 Subject: [PATCH] minor release 3.2.1 (#101) * fix #98 remove 300 message. in addition, fix erroneous pcpp::DnsLayer parsing, sometimes causing multiple parses * issue #100 add tcp reassembly error tracking * pcap stats handler implementation * add pcap stream handler to default pktvisord capture * add pcap handler metrics to cli, make time out 30s * bump pcapplusplus to 21.05 release * issue #94 add TLS support to web server * address issue #84, improve flushing and error messages. change to DAEMON syslog facility * address #83, improved daemonize and logging. * address #83, more improved daemonize and logging. * better error handling and documentation, esp. TLS config * update download link in read me * add TLS support to CLI * handle pcap stats monotonic counter at each new bucket. --- .dockerignore | 1 + .gitignore | 3 +- CMakeLists.txt | 4 +- README.md | 25 ++- cmd/pktvisor-pcap/CMakeLists.txt | 4 +- cmd/pktvisord/main.cpp | 128 +++++++----- conanfile.txt | 3 +- golang/cmd/pktvisor-cli/main.go | 26 ++- golang/pkg/client/types.go | 8 + src/CMakeLists.txt | 1 + src/CoreServer.cpp | 6 +- src/CoreServer.h | 2 +- src/HttpServer.h | 79 ++++++-- src/handlers/CMakeLists.txt | 1 + src/handlers/dns/DnsLayer.cpp | 88 ++++----- src/handlers/dns/DnsLayer.h | 2 +- src/handlers/dns/DnsStreamHandler.cpp | 8 +- src/handlers/dns/tests/test_dns_layer.cpp | 41 ++++ src/handlers/pcap/CMakeLists.txt | 24 +++ src/handlers/pcap/PcapHandler.conf | 2 + src/handlers/pcap/PcapHandlerModulePlugin.cpp | 184 ++++++++++++++++++ src/handlers/pcap/PcapHandlerModulePlugin.h | 30 +++ src/handlers/pcap/PcapStreamHandler.cpp | 173 ++++++++++++++++ src/handlers/pcap/PcapStreamHandler.h | 112 +++++++++++ src/handlers/pcap/README.md | 7 + src/handlers/pcap/tests/CMakeLists.txt | 17 ++ src/handlers/pcap/tests/main.cpp | 17 ++ src/handlers/pcap/tests/test_json_schema.cpp | 62 ++++++ src/handlers/pcap/tests/test_pcap_layer.cpp | 35 ++++ src/handlers/pcap/tests/window-schema.json | 152 +++++++++++++++ src/handlers/static_plugins.h | 1 + src/inputs/pcap/PcapInputStream.cpp | 53 +++-- src/inputs/pcap/PcapInputStream.h | 7 +- 33 files changed, 1148 insertions(+), 158 deletions(-) create mode 100644 src/handlers/pcap/CMakeLists.txt create mode 100644 src/handlers/pcap/PcapHandler.conf create mode 100644 src/handlers/pcap/PcapHandlerModulePlugin.cpp create mode 100644 src/handlers/pcap/PcapHandlerModulePlugin.h create mode 100644 src/handlers/pcap/PcapStreamHandler.cpp create mode 100644 src/handlers/pcap/PcapStreamHandler.h create mode 100644 src/handlers/pcap/README.md create mode 100644 src/handlers/pcap/tests/CMakeLists.txt create mode 100644 src/handlers/pcap/tests/main.cpp create mode 100644 src/handlers/pcap/tests/test_json_schema.cpp create mode 100644 src/handlers/pcap/tests/test_pcap_layer.cpp create mode 100644 src/handlers/pcap/tests/window-schema.json diff --git a/.dockerignore b/.dockerignore index 919a15e1c..3c69a9eb7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ Dockerfile appimage/Dockerfile.part appimage/export.sh appimage/Makefile +localconfig/* diff --git a/.gitignore b/.gitignore index 82dd8837c..0636b85a8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ docs/html-documentation-generated* integration_tests/external golang/pkg/client/version.go docs/internals/html -appimage/*.AppImage \ No newline at end of file +appimage/*.AppImage +localconfig/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c7d1ac37..e3bbeb6d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,12 +4,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") # VERSION # this is the source of truth for semver version -project(visor VERSION 3.2.0) +project(visor VERSION 3.2.1) # for main line release, this is empty # for development release, this is "-develop" # for release candidate, this is "-rc" -set(VISOR_PRERELEASE "") +set(VISOR_PRERELEASE "-develop") # these are computed set(VISOR_VERSION_NUM "${PROJECT_VERSION}${VISOR_PRERELEASE}") diff --git a/README.md b/README.md index 909f18202..86f4c44c6 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ download [on the Releases page](https://github.com/ns1labs/pktvisor/releases). I Linux distributions and does not require installation or any other dependencies. ```shell -curl https://github.com/ns1labs/pktvisor/releases/download/v3.2.0/pktvisor-x86_64-3.2.0.AppImage --output pktvisor-x86_64.AppImage +curl -L http://pktvisor.com/download -o pktvisor-x86_64.AppImage chmod +x pktvisor-x86_64.AppImage ./pktvisor-x86_64.AppImage pktvisord -h ``` @@ -153,19 +153,24 @@ or pktvisord summarizes data streams and exposes a REST API control plane for configuration and metrics. IFACE, if specified, is either a network interface or an IP address (4 or 6). If this is specified, - a "pcap" input stream will be automatically created, with "net" and "dns" handler modules attached. + a "pcap" input stream will be automatically created, with "net", "dns", and "pcap" handler modules attached. Base Options: - -l HOST Run webserver on the given host or IP [default: localhost] - -p PORT Run webserver on the given port [default: 10853] - --admin-api Enable admin REST API giving complete control plane functionality [default: false] - When not specified, the exposed API is read-only access to summarized metrics. - When specified, write access is enabled for all modules. -d Daemonize; fork and continue running in the background [default: false] -h --help Show this screen -v Verbose log output - --no-track Don't send lightweight, anonymous usage metrics. + --no-track Don't send lightweight, anonymous usage metrics --version Show version + Web Server Options: + -l HOST Run web server on the given host or IP [default: localhost] + -p PORT Run web server on the given port [default: 10853] + --tls Enable TLS on the web server + --tls-cert FILE Use given TLS cert. Required if --tls is enabled. + --tls-key FILE Use given TLS private key. Required if --tls is enabled. + --admin-api Enable admin REST API giving complete control plane functionality [default: false] + When not specified, the exposed API is read-only access to summarized metrics. + When specified, write access is enabled for all modules. + Geo Options: --geo-city FILE GeoLite2 City database to use for IP to Geo mapping --geo-asn FILE GeoLite2 ASN database to use for IP to ASN mapping Logging Options: @@ -173,12 +178,12 @@ or --syslog Log to syslog Prometheus Options: --prometheus Enable native Prometheus metrics at path /metrics - --prom-instance ID Optionally set the 'instance' label to ID + --prom-instance ID Optionally set the 'instance' label to given ID Handler Module Defaults: --max-deep-sample N Never deep sample more than N% of streams (an int between 0 and 100) [default: 100] --periods P Hold this many 60 second time periods of history in memory [default: 5] pcap Input Module Options: - -b BPF Filter packets using the given BPF string + -b BPF Filter packets using the given tcpdump compatible filter expression. Example: "port 53" -H HOSTSPEC Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64" Specifying this for live capture will append to any automatic detection. diff --git a/cmd/pktvisor-pcap/CMakeLists.txt b/cmd/pktvisor-pcap/CMakeLists.txt index 79fba690a..d327ff843 100644 --- a/cmd/pktvisor-pcap/CMakeLists.txt +++ b/cmd/pktvisor-pcap/CMakeLists.txt @@ -9,4 +9,6 @@ target_link_libraries(pktvisor-pcap PRIVATE ${CONAN_LIBS_DOCOPT.CPP} Visor::Handler::Net - Visor::Handler::Dns) + Visor::Handler::Dns + Visor::Handler::Pcap + ) diff --git a/cmd/pktvisord/main.cpp b/cmd/pktvisord/main.cpp index f8c53680c..c03c7e6e6 100644 --- a/cmd/pktvisord/main.cpp +++ b/cmd/pktvisord/main.cpp @@ -19,6 +19,7 @@ #include "GeoDB.h" #include "handlers/dns/DnsStreamHandler.h" #include "handlers/net/NetStreamHandler.h" +#include "handlers/pcap/PcapStreamHandler.h" #include "inputs/pcap/PcapInputStream.h" #include "timer.hpp" @@ -32,19 +33,24 @@ static const char USAGE[] = pktvisord summarizes data streams and exposes a REST API control plane for configuration and metrics. IFACE, if specified, is either a network interface or an IP address (4 or 6). If this is specified, - a "pcap" input stream will be automatically created, with "net" and "dns" handler modules attached. + a "pcap" input stream will be automatically created, with "net", "dns", and "pcap" handler modules attached. Base Options: - -l HOST Run webserver on the given host or IP [default: localhost] - -p PORT Run webserver on the given port [default: 10853] - --admin-api Enable admin REST API giving complete control plane functionality [default: false] - When not specified, the exposed API is read-only access to summarized metrics. - When specified, write access is enabled for all modules. -d Daemonize; fork and continue running in the background [default: false] -h --help Show this screen -v Verbose log output - --no-track Don't send lightweight, anonymous usage metrics. + --no-track Don't send lightweight, anonymous usage metrics --version Show version + Web Server Options: + -l HOST Run web server on the given host or IP [default: localhost] + -p PORT Run web server on the given port [default: 10853] + --tls Enable TLS on the web server + --tls-cert FILE Use given TLS cert. Required if --tls is enabled. + --tls-key FILE Use given TLS private key. Required if --tls is enabled. + --admin-api Enable admin REST API giving complete control plane functionality [default: false] + When not specified, the exposed API is read-only access to summarized metrics. + When specified, write access is enabled for all modules. + Geo Options: --geo-city FILE GeoLite2 City database to use for IP to Geo mapping --geo-asn FILE GeoLite2 ASN database to use for IP to ASN mapping Logging Options: @@ -52,12 +58,12 @@ static const char USAGE[] = --syslog Log to syslog Prometheus Options: --prometheus Enable native Prometheus metrics at path /metrics - --prom-instance ID Optionally set the 'instance' label to ID + --prom-instance ID Optionally set the 'instance' label to given ID Handler Module Defaults: --max-deep-sample N Never deep sample more than N% of streams (an int between 0 and 100) [default: 100] --periods P Hold this many 60 second time periods of history in memory [default: 5] pcap Input Module Options: - -b BPF Filter packets using the given BPF string + -b BPF Filter packets using the given tcpdump compatible filter expression. Example: "port 53" -H HOSTSPEC Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64" Specifying this for live capture will append to any automatic detection. @@ -86,7 +92,8 @@ void initialize_geo(const docopt::value &city, const docopt::value &asn) // adapted from LPI becomeDaemon() int daemonize() { - switch (fork()) { + + switch (auto pid = fork()) { case -1: return -1; case 0: @@ -94,53 +101,34 @@ int daemonize() break; default: // while parent terminates + spdlog::get("pktvisor-daemon")->info("daemonized to PID {}", pid); _exit(EXIT_SUCCESS); } // Become leader of new session if (setsid() == -1) { + spdlog::get("pktvisor-daemon")->error("setsid() fail"); return -1; } - // Ensure we are not session leader - switch (auto pid = fork()) { - case -1: - return -1; - case 0: - break; - default: - std::cerr << "pktvisord running at PID " << pid << std::endl; - _exit(EXIT_SUCCESS); - } - // Clear file mode creation mask umask(0); - // Change to root directory - chdir("/"); - int maxfd, fd; - maxfd = sysconf(_SC_OPEN_MAX); - // Limit is indeterminate... - if (maxfd == -1) { - maxfd = 8192; // so take a guess - } - - for (fd = 0; fd < maxfd; fd++) { - close(fd); - } - // Reopen standard fd's to /dev/null close(STDIN_FILENO); - fd = open("/dev/null", O_RDWR); + int fd = open("/dev/null", O_RDWR); if (fd != STDIN_FILENO) { + spdlog::get("pktvisor-daemon")->error("open() fail"); return -1; } if (dup2(STDIN_FILENO, STDOUT_FILENO) != STDOUT_FILENO) { + spdlog::get("pktvisor-daemon")->error("dup2 fail (STDOUT)"); return -1; } if (dup2(STDIN_FILENO, STDERR_FILENO) != STDERR_FILENO) { + spdlog::get("pktvisor-daemon")->error("dup2 fail (STDERR)"); return -1; } @@ -155,23 +143,38 @@ int main(int argc, char *argv[]) true, // show help if requested VISOR_VERSION); // version string - if (args["-d"].asBool()) { + bool daemon{args["-d"].asBool()}; + if (daemon) { + // before we daemonize, if they are using a log file, ensure it can be opened + if (args["--log-file"]) { + try { + auto logger_probe = spdlog::basic_logger_mt("pktvisor-log-probe", args["--log-file"].asString()); + } catch (const spdlog::spdlog_ex &ex) { + // note in daemon mode, this may get swallowed because stdout is already closed + std::cerr << "Log init failed: " << ex.what() << std::endl; + exit(EXIT_FAILURE); + } + } + auto dlogger = spdlog::stderr_color_st("pktvisor-daemon"); + dlogger->flush_on(spdlog::level::info); if (daemonize()) { - std::cerr << "failed to daemonize" << std::endl; + dlogger->error("failed to daemonize"); exit(EXIT_FAILURE); } } std::shared_ptr logger; + spdlog::flush_on(spdlog::level::err); if (args["--log-file"]) { try { logger = spdlog::basic_logger_mt("pktvisor", args["--log-file"].asString()); + spdlog::flush_every(std::chrono::seconds(3)); } catch (const spdlog::spdlog_ex &ex) { std::cerr << "Log init failed: " << ex.what() << std::endl; exit(EXIT_FAILURE); } } else if (args["--syslog"].asBool()) { - logger = spdlog::syslog_logger_mt("pktvisor", "pktvisord", LOG_PID); + logger = spdlog::syslog_logger_mt("pktvisor", "pktvisord", LOG_PID, LOG_DAEMON); } else { logger = spdlog::stdout_color_mt("pktvisor"); } @@ -179,6 +182,13 @@ int main(int argc, char *argv[]) logger->set_level(spdlog::level::debug); } + logger->info("{} starting up", VISOR_VERSION); + + // if we are demonized, change to root directory now that (potentially) logs are open + if (daemon) { + chdir("/"); + } + PrometheusConfig prom_config; if (args["--prometheus"].asBool()) { prom_config.path = "/metrics"; @@ -186,8 +196,29 @@ int main(int argc, char *argv[]) prom_config.instance = args["--prom-instance"].asString(); } } - CoreServer svr(!args["--admin-api"].asBool(), logger, prom_config); - svr.set_http_logger([&logger](const auto &req, const auto &res) { + + HttpConfig http_config; + http_config.read_only = !args["--admin-api"].asBool(); + if (args["--tls"].asBool()) { + http_config.tls_enabled = true; + if (!args["--tls-key"] || !args["--tls-cert"]) { + logger->error("you must specify --tls-key and --tls-cert to use --tls"); + exit(EXIT_FAILURE); + } + http_config.key = args["--tls-key"].asString(); + http_config.cert = args["--tls-cert"].asString(); + logger->info("Enabling TLS with cert {} and key {}", http_config.key, http_config.cert); + } + + std::unique_ptr svr; + try { + svr = std::make_unique(logger, http_config, prom_config); + } catch (const std::exception &e) { + logger->error(e.what()); + logger->info("exit with failure"); + exit(EXIT_FAILURE); + } + svr->set_http_logger([&logger](const auto &req, const auto &res) { logger->info("REQUEST: {} {} {}", req.method, req.path, res.status); if (res.status == 500) { logger->error(res.body); @@ -196,7 +227,9 @@ int main(int argc, char *argv[]) shutdown_handler = [&]([[maybe_unused]] int signal) { logger->info("Shutting down"); - svr.stop(); + logger->flush(); + svr->stop(); + logger->flush(); }; std::signal(SIGINT, signal_handler); std::signal(SIGTERM, signal_handler); @@ -261,14 +294,18 @@ int main(int argc, char *argv[]) input_stream->config_set("bpf", bpf); input_stream->config_set("host_spec", host_spec); - auto input_manager = svr.input_manager(); - auto handler_manager = svr.handler_manager(); + auto input_manager = svr->input_manager(); + auto handler_manager = svr->handler_manager(); input_manager->module_add(std::move(input_stream)); auto [input_stream_, stream_mgr_lock] = input_manager->module_get_locked("pcap"); stream_mgr_lock.unlock(); auto pcap_stream = dynamic_cast(input_stream_); + { + auto pcap_module = std::make_unique("pcap", pcap_stream, periods, sample_rate); + handler_manager->module_add(std::move(pcap_module)); + } { auto handler_module = std::make_unique("net", pcap_stream, periods, sample_rate); handler_manager->module_add(std::move(handler_module)); @@ -284,6 +321,7 @@ int main(int argc, char *argv[]) } catch (const std::exception &e) { logger->error(e.what()); + logger->info("exit with failure"); exit(EXIT_FAILURE); } } else if (!args["--admin-api"].asBool()) { @@ -294,11 +332,13 @@ int main(int argc, char *argv[]) } try { - svr.start(host.c_str(), port); + svr->start(host.c_str(), port); } catch (const std::exception &e) { logger->error(e.what()); + logger->info("exit with failure"); exit(EXIT_FAILURE); } + logger->info("exit with success"); exit(EXIT_SUCCESS); } diff --git a/conanfile.txt b/conanfile.txt index dc69654b9..8b300b464 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -5,8 +5,9 @@ docopt.cpp/0.6.3 nlohmann_json/3.9.1 cpp-httplib/0.8.0 corrade/2020.06 -pcapplusplus/ns1-dev +pcapplusplus/21.05 json-schema-validator/2.1.0 +openssl/1.1.1k [build_requires] benchmark/1.5.2 diff --git a/golang/cmd/pktvisor-cli/main.go b/golang/cmd/pktvisor-cli/main.go index 0c6615524..6c3830452 100644 --- a/golang/cmd/pktvisor-cli/main.go +++ b/golang/cmd/pktvisor-cli/main.go @@ -6,6 +6,7 @@ package main import ( "context" + "crypto/tls" "encoding/json" "fmt" "log" @@ -27,6 +28,7 @@ var ( refreshPeriod = 1 // seconds currentView = "main" serverVersion = "" + protocol = "http" ) func main() { @@ -40,9 +42,13 @@ Usage: Options: -p PORT Query pktvisord metrics webserver on the given port [default: 10853] -H HOST Query pktvisord metrics webserver on the given host [default: localhost] + --tls Use TLS to communicate with pktvisord metrics webserver + --tls-noverify Do not verify TLS certificate -h Show this screen --version Show client version` + wantTLS := flag.Bool("tls", false, "Use TLS to communicate with pktvisord metrics webserver") + wantTLSNoVerify := flag.Bool("tls-noverify", false, "Use TLS to communicate with pktvisord metrics webserver, do not verify TLS certificate") wantVersion := flag.Bool("version", false, "Show client version") wantHelp := flag.Bool("h", false, "Show help") fPort := flag.Int("p", 10853, "Query pktvisord metrics webserver on the given port") @@ -61,7 +67,15 @@ Options: statHost = *fHost var appMetrics client.AppMetrics - err := getMetrics(fmt.Sprintf("http://%s:%d/api/v1/metrics/app", statHost, statPort), &appMetrics) + + if *wantTLS || *wantTLSNoVerify { + protocol = "https" + if *wantTLSNoVerify { + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + } + URL := fmt.Sprintf("%s://%s:%d/api/v1/metrics/app", protocol, statHost, statPort) + err := getMetrics(URL, &appMetrics) if err != nil { log.Panicln(err) } @@ -114,7 +128,7 @@ func updateHeader(v *gocui.View, window5m *client.StatSnapshot) { pcounts.DeepSamples, (float64(pcounts.DeepSamples)/float64(pcounts.Total))*100, ) - _, _ = fmt.Fprintf(v, "Pkt Rates Total %d/s %d/%d/%d/%d pps | In %d/s %d/%d/%d/%d pps | Out %d/s %d/%d/%d/%d pps | IP Card. In: %d | Out: %d\n\n", + _, _ = fmt.Fprintf(v, "Pkt Rates Total %d/s %d/%d/%d/%d pps | In %d/s %d/%d/%d/%d pps | Out %d/s %d/%d/%d/%d pps | IP Card. In: %d | Out: %d | TCP Errors %d | OS Drops %d | IF Drops %d\n\n", pcounts.Rates.Pps_total.Live, pcounts.Rates.Pps_total.P50, pcounts.Rates.Pps_total.P90, @@ -132,6 +146,10 @@ func updateHeader(v *gocui.View, window5m *client.StatSnapshot) { pcounts.Rates.Pps_out.P99, pcounts.Cardinality.SrcIpsIn, pcounts.Cardinality.DstIpsOut, + window5m.Pcap.TcpReassemblyErrors, + window5m.Pcap.OsDrops, + window5m.Pcap.IfDrops, + ) dnsc := window5m.DNS.WirePackets _, _ = fmt.Fprintf(v, "DNS Wire Pkts %d (%3.1f%%) | Rates Total %d/s %d/%d/%d/%d | UDP %d (%3.1f%%) | TCP %d (%3.1f%%) | IPv4 %d (%3.1f%%) | IPv6 %d (%3.1f%%) | Query %d (%3.1f%%) | Response %d (%3.1f%%)\n", @@ -452,7 +470,7 @@ func quit(g *gocui.Gui, v *gocui.View) error { func getMetrics(url string, payload interface{}) error { spaceClient := http.Client{ - Timeout: time.Second * 2, // Maximum of 2 secs + Timeout: time.Second * 30, } req, err := http.NewRequest(http.MethodGet, url, nil) @@ -477,7 +495,7 @@ func getMetrics(url string, payload interface{}) error { func getStats() (*client.StatSnapshot, error) { var rawStats map[string]client.StatSnapshot - err := getMetrics(fmt.Sprintf("http://%s:%d/api/v1/metrics/window/5", statHost, statPort), &rawStats) + err := getMetrics(fmt.Sprintf("%s://%s:%d/api/v1/metrics/window/5", protocol, statHost, statPort), &rawStats) if err != nil { return nil, err } diff --git a/golang/pkg/client/types.go b/golang/pkg/client/types.go index 22458011d..8bfac7c27 100644 --- a/golang/pkg/client/types.go +++ b/golang/pkg/client/types.go @@ -127,6 +127,13 @@ type PacketPayload struct { Period PeriodPayload `json:"period"` } +// PcapPayload contains information about pcap input stream +type PcapPayload struct { + TcpReassemblyErrors int64 `json:"tcp_reassembly_errors"` + IfDrops int64 `json:"if_drops"` + OsDrops int64 `json:"os_drops"` +} + // PeriodPayload indicates the period of time for which a snapshot refers to type PeriodPayload struct { StartTS int64 `json:"start_ts"` @@ -137,4 +144,5 @@ type PeriodPayload struct { type StatSnapshot struct { DNS DNSPayload `json:"dns"` Packets PacketPayload `json:"packets"` + Pcap PcapPayload `json:"pcap"` } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b5fce816d..1b11edcff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,6 +30,7 @@ target_link_libraries(visor-core ${CONAN_LIBS_CORRADE} ${CONAN_LIBS_SPDLOG} ${CONAN_LIBS_FMT} + ${CONAN_LIBS_OPENSSL} ${VISOR_STATIC_PLUGINS} ) diff --git a/src/CoreServer.cpp b/src/CoreServer.cpp index 8490ff101..75b280a9f 100644 --- a/src/CoreServer.cpp +++ b/src/CoreServer.cpp @@ -9,8 +9,8 @@ #include #include -visor::CoreServer::CoreServer(bool read_only, std::shared_ptr logger, const PrometheusConfig &prom_config) - : _svr(read_only) +visor::CoreServer::CoreServer(std::shared_ptr logger, const HttpConfig &http_config, const PrometheusConfig &prom_config) + : _svr(http_config) , _logger(logger) , _start_time(std::chrono::system_clock::now()) { @@ -45,7 +45,7 @@ visor::CoreServer::CoreServer(bool read_only, std::shared_ptr lo void visor::CoreServer::start(const std::string &host, int port) { if (!_svr.bind_to_port(host.c_str(), port)) { - throw std::runtime_error("unable to bind host/port"); + throw std::runtime_error("unable to bind to " + host + ":" + std::to_string(port)); } _logger->info("web server listening on {}:{}", host, port); if (!_svr.listen_after_bind()) { diff --git a/src/CoreServer.h b/src/CoreServer.h index 66c34ce20..ba0090818 100644 --- a/src/CoreServer.h +++ b/src/CoreServer.h @@ -47,7 +47,7 @@ class CoreServer void _setup_routes(const PrometheusConfig &prom_config); public: - CoreServer(bool read_only, std::shared_ptr logger, const PrometheusConfig &prom_config); + CoreServer(std::shared_ptr logger, const HttpConfig &http_config, const PrometheusConfig &prom_config); ~CoreServer(); void start(const std::string &host, int port); diff --git a/src/HttpServer.h b/src/HttpServer.h index b890818da..aeba45d9d 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -4,48 +4,91 @@ #pragma once +#define CPPHTTPLIB_OPENSSL_SUPPORT +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" #include +#pragma GCC diagnostic pop #include namespace visor { -class HttpServer : public httplib::Server + +using namespace httplib; + +struct HttpConfig { + bool read_only{true}; + bool tls_enabled{false}; + std::string cert; + std::string key; +}; + +class HttpServer { - bool _read_only = true; + HttpConfig _config; + std::unique_ptr _svr; public: - HttpServer(bool read_only) - : _read_only(read_only) + HttpServer(const HttpConfig &config) + : _config(config) + { + if (config.tls_enabled) { + _svr = std::make_unique(config.cert.c_str(), config.key.c_str()); + if (!_svr->is_valid()) { + throw std::runtime_error("invalid TLS configuration"); + } + } else { + _svr = std::make_unique(); + } + } + + void set_logger(Logger logger) + { + _svr->set_logger(std::move(logger)); + } + + bool bind_to_port(const char *host, int port, int socket_flags = 0) + { + return _svr->bind_to_port(host, port, socket_flags); + } + + bool listen_after_bind() + { + return _svr->listen_after_bind(); + } + + void stop() { + _svr->stop(); } - Server &Get(const char *pattern, Handler handler) + Server &Get(const char *pattern, Server::Handler handler) { spdlog::get("pktvisor")->info("Registering GET {}", pattern); - return httplib::Server::Get(pattern, handler); + return _svr->Get(pattern, handler); } - Server &Post(const char *pattern, Handler handler) + Server &Post(const char *pattern, Server::Handler handler) { - if (_read_only) { - return *this; + if (_config.read_only) { + return *_svr; } spdlog::get("pktvisor")->info("Registering POST {}", pattern); - return httplib::Server::Post(pattern, handler); + return _svr->Post(pattern, handler); } - Server &Put(const char *pattern, Handler handler) + Server &Put(const char *pattern, Server::Handler handler) { - if (_read_only) { - return *this; + if (_config.read_only) { + return *_svr; } spdlog::get("pktvisor")->info("Registering PUT {}", pattern); - return httplib::Server::Put(pattern, handler); + return _svr->Put(pattern, handler); } - Server &Delete(const char *pattern, Handler handler) + Server &Delete(const char *pattern, Server::Handler handler) { - if (_read_only) { - return *this; + if (_config.read_only) { + return *_svr; } spdlog::get("pktvisor")->info("Registering DELETE {}", pattern); - return httplib::Server::Delete(pattern, handler); + return _svr->Delete(pattern, handler); } }; } diff --git a/src/handlers/CMakeLists.txt b/src/handlers/CMakeLists.txt index 97ffdf919..3a5e3cf35 100644 --- a/src/handlers/CMakeLists.txt +++ b/src/handlers/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(net) add_subdirectory(dns) +add_subdirectory(pcap) set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} PARENT_SCOPE) diff --git a/src/handlers/dns/DnsLayer.cpp b/src/handlers/dns/DnsLayer.cpp index ae475ff8b..2f9d567a2 100644 --- a/src/handlers/dns/DnsLayer.cpp +++ b/src/handlers/dns/DnsLayer.cpp @@ -118,38 +118,31 @@ bool DnsLayer::shortenLayer(int offsetInLayer, size_t numOfBytesToShorten, IDnsR return true; } - -void DnsLayer::parseResources(bool queryOnly) -{ - size_t offsetInPacket = sizeof(dnshdr); - IDnsResource* curResource = m_ResourceList; - - uint16_t numOfQuestions = be16toh(getDnsHeader()->numberOfQuestions); - uint16_t numOfAnswers = be16toh(getDnsHeader()->numberOfAnswers); - uint16_t numOfAuthority = be16toh(getDnsHeader()->numberOfAuthority); - uint16_t numOfAdditional = be16toh(getDnsHeader()->numberOfAdditional); - - uint32_t numOfOtherResources = numOfQuestions + numOfAnswers + numOfAuthority + numOfAdditional; - - if (numOfOtherResources > 300) - { - LOG_ERROR("DNS layer contains more than 300 resources, probably a bad packet. " - "Skipping parsing DNS resources"); - return; - } - - for (uint32_t i = 0; i < numOfOtherResources; i++) - { - DnsResourceType resType; - if (numOfQuestions > 0) - { - resType = DnsQueryType; - numOfQuestions--; - } - else if (numOfAnswers > 0) - { - resType = DnsAnswerType; - numOfAnswers--; +bool DnsLayer::parseResources(bool queryOnly) +{ + size_t offsetInPacket = sizeof(dnshdr); + IDnsResource *curResource = m_ResourceList; + + uint16_t numOfQuestions = be16toh(getDnsHeader()->numberOfQuestions); + uint16_t numOfAnswers = be16toh(getDnsHeader()->numberOfAnswers); + uint16_t numOfAuthority = be16toh(getDnsHeader()->numberOfAuthority); + uint16_t numOfAdditional = be16toh(getDnsHeader()->numberOfAdditional); + + uint32_t numOfOtherResources = numOfQuestions + numOfAnswers + numOfAuthority + numOfAdditional; + + if (numOfOtherResources > 100) { + // probably bad packet + return false; + } + + for (uint32_t i = 0; i < numOfOtherResources; i++) { + DnsResourceType resType; + if (numOfQuestions > 0) { + resType = DnsQueryType; + numOfQuestions--; + } else if (numOfAnswers > 0) { + resType = DnsAnswerType; + numOfAnswers--; } else if (numOfAuthority > 0) { @@ -178,14 +171,13 @@ void DnsLayer::parseResources(bool queryOnly) offsetInPacket += newResource->getSize(); } - if (offsetInPacket > m_DataLen) - { - //Parse packet failed, DNS resource is out of bounds. Probably a bad packet - delete newGenResource; - return; - } + if (offsetInPacket > m_DataLen) { + //Parse packet failed, DNS resource is out of bounds. Probably a bad packet + delete newGenResource; + return false; + } - // this resource is the first resource + // this resource is the first resource if (m_ResourceList == NULL) { m_ResourceList = newGenResource; @@ -201,15 +193,15 @@ void DnsLayer::parseResources(bool queryOnly) m_FirstQuery = newQuery; if (queryOnly) break; - } - else if (resType == DnsAnswerType && m_FirstAnswer == NULL) - m_FirstAnswer = newResource; - else if (resType == DnsAuthorityType && m_FirstAuthority == NULL) - m_FirstAuthority = newResource; - else if (resType == DnsAdditionalType && m_FirstAdditional == NULL) - m_FirstAdditional = newResource; - } - + } else if (resType == DnsAnswerType && m_FirstAnswer == NULL) + m_FirstAnswer = newResource; + else if (resType == DnsAuthorityType && m_FirstAuthority == NULL) + m_FirstAuthority = newResource; + else if (resType == DnsAdditionalType && m_FirstAdditional == NULL) + m_FirstAdditional = newResource; + } + + return true; } IDnsResource* DnsLayer::getResourceByName(IDnsResource* startFrom, size_t resourceCount, const std::string& name, bool exactMatch) const diff --git a/src/handlers/dns/DnsLayer.h b/src/handlers/dns/DnsLayer.h index 73d774de2..e6466c9cb 100644 --- a/src/handlers/dns/DnsLayer.h +++ b/src/handlers/dns/DnsLayer.h @@ -433,7 +433,7 @@ struct dnshdr { std::string toString() const; - void parseResources(bool queryOnly); + bool parseResources(bool queryOnly); pcpp::OsiModelLayer getOsiModelLayer() const { return pcpp::OsiModelApplicationLayer; } diff --git a/src/handlers/dns/DnsStreamHandler.cpp b/src/handlers/dns/DnsStreamHandler.cpp index 1869ce6db..245382f15 100644 --- a/src/handlers/dns/DnsStreamHandler.cpp +++ b/src/handlers/dns/DnsStreamHandler.cpp @@ -374,7 +374,12 @@ void DnsMetricsBucket::process_dns_layer(bool deep, DnsLayer &payload, pcpp::Pro return; } - payload.parseResources(true); + _dns_topUDPPort.update(port); + + auto success = payload.parseResources(true); + if (!success) { + return; + } if (payload.getDnsHeader()->queryOrResponse == response) { _dns_topRCode.update(payload.getDnsHeader()->responseCode); @@ -409,7 +414,6 @@ void DnsMetricsBucket::process_dns_layer(bool deep, DnsLayer &payload, pcpp::Pro } } - _dns_topUDPPort.update(port); } void DnsMetricsBucket::new_dns_transaction(bool deep, float to90th, float from90th, DnsLayer &dns, PacketDirection dir, DnsTransaction xact) diff --git a/src/handlers/dns/tests/test_dns_layer.cpp b/src/handlers/dns/tests/test_dns_layer.cpp index 690a49865..2ccd8e36e 100644 --- a/src/handlers/dns/tests/test_dns_layer.cpp +++ b/src/handlers/dns/tests/test_dns_layer.cpp @@ -3,10 +3,51 @@ #include "DnsStreamHandler.h" #include "PcapInputStream.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma clang diagnostic ignored "-Wrange-loop-analysis" +#include +#include +#include +#include +#include +#include +#include +#pragma GCC diagnostic pop +#pragma GCC diagnostic ignored "-Wold-style-cast" + using namespace visor::handler::dns; using namespace visor::input::pcap; using namespace nlohmann; +TEST_CASE("Ensure we use only pktvisor DnsLayer", "[pcap][ipv4][dns]") +{ + + pcpp::IFileReaderDevice *reader = pcpp::IFileReaderDevice::getReader("tests/fixtures/dns_udp_tcp_random.pcap"); + + CHECK(reader->open()); + + pcpp::RawPacket rawPacket; + + while (reader->getNextPacket(rawPacket)) { + pcpp::Packet dnsRequest(&rawPacket, pcpp::TCP | pcpp::UDP); + if (dnsRequest.isPacketOfType(pcpp::UDP)) { + CHECK(dnsRequest.getLayerOfType() != nullptr); + } else { + CHECK(dnsRequest.getLayerOfType() != nullptr); + } + // we do NOT expect to see pcpp::DnsLayer or DNS protocol yet + CHECK(dnsRequest.getLayerOfType() == nullptr); + CHECK(dnsRequest.getLayerOfType() == nullptr); + CHECK(dnsRequest.isPacketOfType(pcpp::DNS) == false); + } + + reader->close(); + delete reader; +} + TEST_CASE("Parse DNS UDP IPv4 tests", "[pcap][ipv4][udp][dns]") { diff --git a/src/handlers/pcap/CMakeLists.txt b/src/handlers/pcap/CMakeLists.txt new file mode 100644 index 000000000..38c91ca7d --- /dev/null +++ b/src/handlers/pcap/CMakeLists.txt @@ -0,0 +1,24 @@ +message(STATUS "Handler Module: Pcap") + +set_directory_properties(PROPERTIES CORRADE_USE_PEDANTIC_FLAGS ON) + +corrade_add_static_plugin(VisorHandlerPcap + ${CMAKE_CURRENT_BINARY_DIR} + PcapHandler.conf + PcapHandlerModulePlugin.cpp + PcapStreamHandler.cpp) +add_library(Visor::Handler::Pcap ALIAS VisorHandlerPcap) + +target_include_directories(VisorHandlerPcap + INTERFACE + $ + ) + +target_link_libraries(VisorHandlerPcap + PUBLIC + Visor::Input::Pcap + ) + +set(VISOR_STATIC_PLUGINS ${VISOR_STATIC_PLUGINS} Visor::Handler::Pcap PARENT_SCOPE) + +add_subdirectory(tests) \ No newline at end of file diff --git a/src/handlers/pcap/PcapHandler.conf b/src/handlers/pcap/PcapHandler.conf new file mode 100644 index 000000000..4216fe72b --- /dev/null +++ b/src/handlers/pcap/PcapHandler.conf @@ -0,0 +1,2 @@ +[data] +desc=Pcap specific operational metrics diff --git a/src/handlers/pcap/PcapHandlerModulePlugin.cpp b/src/handlers/pcap/PcapHandlerModulePlugin.cpp new file mode 100644 index 000000000..541a616e6 --- /dev/null +++ b/src/handlers/pcap/PcapHandlerModulePlugin.cpp @@ -0,0 +1,184 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "PcapHandlerModulePlugin.h" +#include "PcapInputStream.h" +#include "PcapStreamHandler.h" +#include +#include + +CORRADE_PLUGIN_REGISTER(VisorHandlerPcap, visor::handler::pcap::PcapHandlerModulePlugin, + "dev.visor.module.handler/1.0") + +namespace visor::handler::pcap { + +using namespace visor::input::pcap; +using json = nlohmann::json; + +void PcapHandlerModulePlugin::_setup_routes(HttpServer &svr) +{ + // CREATE + svr.Post("/api/v1/inputs/pcap/(\\w+)/handlers/pcap", [this](const httplib::Request &req, httplib::Response &res) { + json result; + try { + auto body = json::parse(req.body); + SchemaMap req_schema = {{"name", "\\w+"}}; + SchemaMap opt_schema = {{"periods", "\\d{1,3}"}, {"deep_sample_rate", "\\d{1,3}"}}; + try { + _check_schema(body, req_schema, opt_schema); + } catch (const SchemaException &e) { + res.status = 400; + result["error"] = e.what(); + res.set_content(result.dump(), "text/json"); + return; + } + auto input_name = req.matches[1]; + if (!_input_manager->module_exists(input_name)) { + res.status = 404; + result["error"] = "input name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + if (_handler_manager->module_exists(body["name"])) { + res.status = 400; + result["error"] = "handler name already exists"; + res.set_content(result.dump(), "text/json"); + return; + } + // note, may be a race on exists() above, this may fail. if so we will catch and 500. + auto [input_stream, stream_mgr_lock] = _input_manager->module_get_locked(input_name); + assert(input_stream); + auto pcap_stream = dynamic_cast(input_stream); + if (!pcap_stream) { + res.status = 400; + result["error"] = "input stream is not pcap"; + res.set_content(result.dump(), "text/json"); + return; + } + if (!input_stream->running()) { + res.status = 400; + result["error"] = "input stream is not running"; + res.set_content(result.dump(), "text/json"); + return; + } + // TODO use global default from command line + uint periods{5}; + uint deep_sample_rate{100}; + if (body.contains("periods")) { + periods = body["periods"]; + } + if (body.contains("deep_sample_rate")) { + deep_sample_rate = body["deep_sample_rate"]; + } + auto handler_module = std::make_unique(body["name"], pcap_stream, periods, deep_sample_rate); + _handler_manager->module_add(std::move(handler_module)); + result["name"] = body["name"]; + result["periods"] = periods; + result["deep_sample_rate"] = deep_sample_rate; + res.set_content(result.dump(), "text/json"); + } catch (const std::exception &e) { + res.status = 500; + result.clear(); + result["error"] = e.what(); + res.set_content(result.dump(), "text/json"); + } + }); + svr.Get("/api/v1/inputs/pcap/(\\w+)/handlers/pcap/(\\w+)", [this](const httplib::Request &req, httplib::Response &res) { + json result; + try { + auto input_name = req.matches[1]; + if (!_input_manager->module_exists(input_name)) { + res.status = 404; + result["error"] = "input name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + auto handler_name = req.matches[2]; + if (!_handler_manager->module_exists(handler_name)) { + res.status = 404; + result["error"] = "handler name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + auto [handler, handler_mgr_lock] = _handler_manager->module_get_locked(handler_name); + auto pcap_handler = dynamic_cast(handler); + if (!pcap_handler) { + res.status = 400; + result["error"] = "handler stream is not pcap"; + res.set_content(result.dump(), "text/json"); + return; + } + pcap_handler->info_json(result); + res.set_content(result.dump(), "text/json"); + } catch (const std::exception &e) { + res.status = 500; + result.clear(); + result["error"] = e.what(); + res.set_content(result.dump(), "text/json"); + } + }); + svr.Get("/api/v1/inputs/pcap/(\\w+)/handlers/pcap/(\\w+)/bucket/(\\d+)", [this](const httplib::Request &req, httplib::Response &res) { + json result; + try { + auto input_name = req.matches[1]; + if (!_input_manager->module_exists(input_name)) { + res.status = 404; + result["error"] = "input name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + auto handler_name = req.matches[2]; + if (!_handler_manager->module_exists(handler_name)) { + res.status = 404; + result["error"] = "handler name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + auto [handler, handler_mgr_lock] = _handler_manager->module_get_locked(handler_name); + auto pcap_handler = dynamic_cast(handler); + if (!pcap_handler) { + res.status = 400; + result["error"] = "handler stream is not pcap"; + res.set_content(result.dump(), "text/json"); + return; + } + pcap_handler->window_json(result, std::stoi(req.matches[3]), false); + res.set_content(result.dump(), "text/json"); + } catch (const std::exception &e) { + res.status = 500; + result.clear(); + result["error"] = e.what(); + res.set_content(result.dump(), "text/json"); + } + }); + // DELETE + svr.Delete("/api/v1/inputs/pcap/(\\w+)/handlers/pcap/(\\w+)", [this](const httplib::Request &req, httplib::Response &res) { + json result; + try { + auto input_name = req.matches[1]; + if (!_input_manager->module_exists(input_name)) { + res.status = 404; + result["error"] = "input name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + auto handler_name = req.matches[2]; + if (!_handler_manager->module_exists(handler_name)) { + res.status = 404; + result["error"] = "handler name does not exist"; + res.set_content(result.dump(), "text/json"); + return; + } + _handler_manager->module_remove(handler_name); + res.set_content(result.dump(), "text/json"); + } catch (const std::exception &e) { + res.status = 500; + result.clear(); + result["error"] = e.what(); + res.set_content(result.dump(), "text/json"); + } + }); +} + +} \ No newline at end of file diff --git a/src/handlers/pcap/PcapHandlerModulePlugin.h b/src/handlers/pcap/PcapHandlerModulePlugin.h new file mode 100644 index 000000000..d9ce7af59 --- /dev/null +++ b/src/handlers/pcap/PcapHandlerModulePlugin.h @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#pragma once + + +#include "HandlerModulePlugin.h" + +namespace visor::handler::pcap { + +class PcapHandlerModulePlugin : public HandlerModulePlugin +{ + +protected: + void _setup_routes(HttpServer &svr) override; + +public: + explicit PcapHandlerModulePlugin(Corrade::PluginManager::AbstractManager &manager, const std::string &plugin) + : visor::HandlerModulePlugin{manager, plugin} + { + } + + std::string name() const override + { + return "PcapHandler"; + } +}; +} + diff --git a/src/handlers/pcap/PcapStreamHandler.cpp b/src/handlers/pcap/PcapStreamHandler.cpp new file mode 100644 index 000000000..595dd60cd --- /dev/null +++ b/src/handlers/pcap/PcapStreamHandler.cpp @@ -0,0 +1,173 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "PcapStreamHandler.h" +#include "GeoDB.h" +#include "utils.h" +#include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wc99-extensions" +#pragma GCC diagnostic ignored "-Wpedantic" +#include +#include +#pragma GCC diagnostic pop +#include +#include + +namespace visor::handler::pcap { + +PcapStreamHandler::PcapStreamHandler(const std::string &name, PcapInputStream *stream, uint periods, uint deepSampleRate) + : visor::StreamMetricsHandler(name, periods, deepSampleRate) + , _stream(stream) +{ + assert(stream); +} + +void PcapStreamHandler::start() +{ + if (_running) { + return; + } + + if (config_exists("recorded_stream")) { + _metrics->set_recorded_stream(); + } + + _start_tstamp_connection = _stream->start_tstamp_signal.connect(&PcapStreamHandler::set_start_tstamp, this); + _end_tstamp_connection = _stream->end_tstamp_signal.connect(&PcapStreamHandler::set_end_tstamp, this); + + _pcap_tcp_reassembly_errors_connection = _stream->tcp_reassembly_error_signal.connect(&PcapStreamHandler::process_pcap_tcp_reassembly_error, this); + _pcap_stats_connection = _stream->pcap_stats_signal.connect(&PcapStreamHandler::process_pcap_stats, this); + + _running = true; +} + +void PcapStreamHandler::stop() +{ + if (!_running) { + return; + } + + _start_tstamp_connection.disconnect(); + _end_tstamp_connection.disconnect(); + _pcap_tcp_reassembly_errors_connection.disconnect(); + + _running = false; +} + +PcapStreamHandler::~PcapStreamHandler() +{ +} + +// callback from input module +void PcapStreamHandler::process_pcap_tcp_reassembly_error(pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3, timespec stamp) +{ + _metrics->process_pcap_tcp_reassembly_error(payload, dir, l3, stamp); +} +void PcapStreamHandler::process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats) +{ + _metrics->process_pcap_stats(stats); +} + +void PcapStreamHandler::window_json(json &j, uint64_t period, bool merged) +{ + if (merged) { + _metrics->window_merged_json(j, schema_key(), period); + } else { + _metrics->window_single_json(j, schema_key(), period); + } +} +void PcapStreamHandler::set_start_tstamp(timespec stamp) +{ + _metrics->set_start_tstamp(stamp); +} +void PcapStreamHandler::set_end_tstamp(timespec stamp) +{ + _metrics->set_end_tstamp(stamp); +} +void PcapStreamHandler::info_json(json &j) const +{ + _common_info_json(j); +} +void PcapStreamHandler::window_prometheus(std::stringstream &out) +{ + if (_metrics->current_periods() > 1) { + _metrics->window_single_prometheus(out, 1); + } else { + _metrics->window_single_prometheus(out, 0); + } +} + +void PcapMetricsBucket::specialized_merge(const AbstractMetricsBucket &o) +{ + // static because caller guarantees only our own bucket type + const auto &other = static_cast(o); + + std::shared_lock r_lock(other._mutex); + std::unique_lock w_lock(_mutex); + + _counters.pcap_TCP_reassembly_errors += other._counters.pcap_TCP_reassembly_errors; + _counters.pcap_os_drop += other._counters.pcap_os_drop; + _counters.pcap_if_drop += other._counters.pcap_if_drop; +} + +void PcapMetricsBucket::to_prometheus(std::stringstream &out) const +{ + std::shared_lock r_lock(_mutex); + + _counters.pcap_TCP_reassembly_errors.to_prometheus(out); + _counters.pcap_os_drop.to_prometheus(out); + _counters.pcap_if_drop.to_prometheus(out); +} + +void PcapMetricsBucket::to_json(json &j) const +{ + std::shared_lock r_lock(_mutex); + + _counters.pcap_TCP_reassembly_errors.to_json(j); + _counters.pcap_os_drop.to_json(j); + _counters.pcap_if_drop.to_json(j); +} + +void PcapMetricsBucket::process_pcap_tcp_reassembly_error([[maybe_unused]] bool deep, [[maybe_unused]] pcpp::Packet &payload, [[maybe_unused]] PacketDirection dir, [[maybe_unused]] pcpp::ProtocolType l3) +{ + std::unique_lock lock(_mutex); + ++_counters.pcap_TCP_reassembly_errors; +} +void PcapMetricsBucket::process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats) +{ + std::unique_lock lock(_mutex); + + // pcap keeps monotonic counters, so at the start of every new bucket we have to record + // the current pcap value and then keep track of differences. + if (_counters.pcap_last_os_drop == std::numeric_limits::max() || _counters.pcap_last_if_drop == std::numeric_limits::max()) { + _counters.pcap_last_os_drop = stats.packetsDrop; + _counters.pcap_last_if_drop = stats.packetsDropByInterface; + return; + } + if (stats.packetsDrop > _counters.pcap_last_os_drop) { + _counters.pcap_os_drop += stats.packetsDrop - _counters.pcap_last_os_drop; + _counters.pcap_last_os_drop = stats.packetsDrop; + } + if (stats.packetsDropByInterface > _counters.pcap_last_if_drop) { + _counters.pcap_if_drop += stats.packetsDropByInterface - _counters.pcap_last_if_drop; + _counters.pcap_last_if_drop = stats.packetsDropByInterface; + } +} + +// the general metrics manager entry point +void PcapMetricsManager::process_pcap_tcp_reassembly_error(pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3, [[maybe_unused]] timespec stamp) +{ + // process in the "live" bucket + live_bucket()->process_pcap_tcp_reassembly_error(_deep_sampling_now, payload, dir, l3); +} +void PcapMetricsManager::process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats) +{ + // process in the "live" bucket + live_bucket()->process_pcap_stats(stats); +} + +} \ No newline at end of file diff --git a/src/handlers/pcap/PcapStreamHandler.h b/src/handlers/pcap/PcapStreamHandler.h new file mode 100644 index 000000000..a9e8aae16 --- /dev/null +++ b/src/handlers/pcap/PcapStreamHandler.h @@ -0,0 +1,112 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#pragma once + +#include "AbstractMetricsManager.h" +#include "PcapInputStream.h" +#include "StreamHandler.h" +#include +#include +#include + +namespace visor::handler::pcap { + +using namespace visor::input::pcap; + +class PcapMetricsBucket final : public visor::AbstractMetricsBucket +{ + +protected: + mutable std::shared_mutex _mutex; + + // total numPackets is tracked in base class num_events + struct counters { + + Counter pcap_TCP_reassembly_errors; + + Counter pcap_os_drop; + uint64_t pcap_last_os_drop{std::numeric_limits::max()}; + + Counter pcap_if_drop; + uint64_t pcap_last_if_drop{std::numeric_limits::max()}; + + counters() + : pcap_TCP_reassembly_errors("pcap", {"tcp_reassembly_errors"}, "Count of TCP reassembly errors") + , pcap_os_drop("pcap", {"os_drops"}, "Count of packets dropped by the operating system (if supported)") + , pcap_if_drop("pcap", {"if_drops"}, "Count of packets dropped by the interface (if supported)") + { + } + }; + counters _counters; + +public: + PcapMetricsBucket() + { + } + + // get a copy of the counters + counters counters() const + { + std::shared_lock lock(_mutex); + return _counters; + } + + // visor::AbstractMetricsBucket + void specialized_merge(const AbstractMetricsBucket &other) override; + void to_json(json &j) const override; + void to_prometheus(std::stringstream &out) const override; + + void process_pcap_tcp_reassembly_error(bool deep, pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3); + void process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats); +}; + +class PcapMetricsManager final : public visor::AbstractMetricsManager +{ +public: + PcapMetricsManager(uint periods, int deepSampleRate) + : visor::AbstractMetricsManager(periods, deepSampleRate) + { + } + + void process_pcap_tcp_reassembly_error(pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3, timespec stamp); + void process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats); +}; + +class PcapStreamHandler final : public visor::StreamMetricsHandler +{ + + PcapInputStream *_stream; + + sigslot::connection _start_tstamp_connection; + sigslot::connection _end_tstamp_connection; + + sigslot::connection _pcap_tcp_reassembly_errors_connection; + sigslot::connection _pcap_stats_connection; + + void process_pcap_tcp_reassembly_error(pcpp::Packet &payload, PacketDirection dir, pcpp::ProtocolType l3, timespec stamp); + void process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats); + + void set_start_tstamp(timespec stamp); + void set_end_tstamp(timespec stamp); + +public: + PcapStreamHandler(const std::string &name, PcapInputStream *stream, uint periods, uint deepSampleRate); + ~PcapStreamHandler() override; + + // visor::AbstractModule + std::string schema_key() const override + { + return "pcap"; + } + void start() override; + void stop() override; + void info_json(json &j) const override; + + // visor::StreamHandler + void window_json(json &j, uint64_t period, bool merged) override; + void window_prometheus(std::stringstream &out) override; +}; + +} diff --git a/src/handlers/pcap/README.md b/src/handlers/pcap/README.md new file mode 100644 index 000000000..fd8557e41 --- /dev/null +++ b/src/handlers/pcap/README.md @@ -0,0 +1,7 @@ +# Pcap Application Metrics Stream Handler + +This directory contains the Pcap application metrics stream handler + +It can attach to pcap input streams and expose pcap application specific operational metrics + +[PcapStreamHandler.h](PcapStreamHandler.h) contains the list of metrics. diff --git a/src/handlers/pcap/tests/CMakeLists.txt b/src/handlers/pcap/tests/CMakeLists.txt new file mode 100644 index 000000000..b208b197c --- /dev/null +++ b/src/handlers/pcap/tests/CMakeLists.txt @@ -0,0 +1,17 @@ + +## TEST SUITE +add_executable(unit-tests-handler-pcap + main.cpp + test_pcap_layer.cpp + test_json_schema.cpp + ) + +target_link_libraries(unit-tests-handler-pcap + PRIVATE + ${CONAN_LIBS_JSON-SCHEMA-VALIDATOR} + Visor::Handler::Pcap) + +add_test(NAME unit-tests-handler-pcap + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src + COMMAND unit-tests-handler-pcap + ) diff --git a/src/handlers/pcap/tests/main.cpp b/src/handlers/pcap/tests/main.cpp new file mode 100644 index 000000000..32179b931 --- /dev/null +++ b/src/handlers/pcap/tests/main.cpp @@ -0,0 +1,17 @@ +#define CATCH_CONFIG_RUNNER +#include +#include + +int main(int argc, char *argv[]) +{ + Catch::Session session; + + int result = session.applyCommandLine(argc, argv); + if (result != 0) { + return result; + } + + result = session.run(); + + return (result == 0 ? EXIT_SUCCESS : EXIT_FAILURE); +} diff --git a/src/handlers/pcap/tests/test_json_schema.cpp b/src/handlers/pcap/tests/test_json_schema.cpp new file mode 100644 index 000000000..9a91bab6d --- /dev/null +++ b/src/handlers/pcap/tests/test_json_schema.cpp @@ -0,0 +1,62 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include +#include +#include +#include +#include + +#include "PcapInputStream.h" +#include "PcapStreamHandler.h" + +using namespace visor::handler::pcap; +using namespace visor::input::pcap; +using namespace nlohmann; +using nlohmann::json_schema::json_validator; + +TEST_CASE("Pcap JSON Schema", "[pcap][iface][json]") +{ + + SECTION("json iface") + { + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_tcp_random.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + PcapStreamHandler pcap_handler{"pcap-test", &stream, 5, 100}; + pcap_handler.config_set("recorded_stream", true); + + pcap_handler.start(); + stream.start(); + stream.stop(); + pcap_handler.stop(); + + json pcap_json; + pcap_handler.metrics()->window_merged_json(pcap_json, pcap_handler.schema_key(), 5); + WARN(pcap_json); + std::ifstream sfile("handlers/pcap/tests/window-schema.json"); + CHECK(sfile.is_open()); + std::string schema; + + sfile.seekg(0, std::ios::end); + schema.reserve(sfile.tellg()); + sfile.seekg(0, std::ios::beg); + + schema.assign((std::istreambuf_iterator(sfile)), std::istreambuf_iterator()); + json_validator validator; + + auto schema_json = json::parse(schema); + + try { + validator.set_root_schema(schema_json); + validator.validate(pcap_json); + } catch (const std::exception &e) { + FAIL(e.what()); + } + } +} diff --git a/src/handlers/pcap/tests/test_pcap_layer.cpp b/src/handlers/pcap/tests/test_pcap_layer.cpp new file mode 100644 index 000000000..8cc8102db --- /dev/null +++ b/src/handlers/pcap/tests/test_pcap_layer.cpp @@ -0,0 +1,35 @@ +#include + +#include "GeoDB.h" +#include "PcapInputStream.h" +#include "PcapStreamHandler.h" + +using namespace visor::handler::pcap; +using namespace visor::input::pcap; + +TEST_CASE("Parse net (dns) random UDP/TCP tests", "[pcap][net]") +{ + + PcapInputStream stream{"pcap-test"}; + stream.config_set("pcap_file", "tests/fixtures/dns_udp_tcp_random.pcap"); + stream.config_set("bpf", ""); + stream.config_set("host_spec", "192.168.0.0/24"); + stream.parse_host_spec(); + + PcapStreamHandler pcap_handler{"pcap-handler-test", &stream, 1, 100}; + + pcap_handler.start(); + stream.start(); + stream.stop(); + pcap_handler.stop(); + + auto counters = pcap_handler.metrics()->bucket(0)->counters(); + + CHECK(pcap_handler.metrics()->start_tstamp().tv_sec == 1614874231); + CHECK(pcap_handler.metrics()->start_tstamp().tv_nsec == 565771000); + + // confirmed with wireshark + CHECK(counters.pcap_TCP_reassembly_errors.value() == 0); + CHECK(counters.pcap_os_drop.value() == 0); + CHECK(counters.pcap_if_drop.value() == 0); +} diff --git a/src/handlers/pcap/tests/window-schema.json b/src/handlers/pcap/tests/window-schema.json new file mode 100644 index 000000000..0b17e09ee --- /dev/null +++ b/src/handlers/pcap/tests/window-schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "The root schema", + "description": "The root schema comprises the entire JSON document.", + "default": {}, + "examples": [ + { + "5m": { + "pcap": { + "if_drops": 0, + "os_drops": 0, + "period": { + "length": 31, + "start_ts": 1614874231 + }, + "tcp_reassembly_errors": 0 + } + } + } + ], + "required": [ + "5m" + ], + "properties": { + "5m": { + "$id": "#/properties/5m", + "type": "object", + "title": "The 5m schema", + "description": "An explanation about the purpose of this instance.", + "default": {}, + "examples": [ + { + "pcap": { + "if_drops": 0, + "os_drops": 0, + "period": { + "length": 31, + "start_ts": 1614874231 + }, + "tcp_reassembly_errors": 0 + } + } + ], + "required": [ + "pcap" + ], + "properties": { + "pcap": { + "$id": "#/properties/5m/properties/pcap", + "type": "object", + "title": "The pcap schema", + "description": "An explanation about the purpose of this instance.", + "default": {}, + "examples": [ + { + "if_drops": 0, + "os_drops": 0, + "period": { + "length": 31, + "start_ts": 1614874231 + }, + "tcp_reassembly_errors": 0 + } + ], + "required": [ + "if_drops", + "os_drops", + "period", + "tcp_reassembly_errors" + ], + "properties": { + "if_drops": { + "$id": "#/properties/5m/properties/pcap/properties/if_drops", + "type": "integer", + "title": "The if_drops schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 0 + ] + }, + "os_drops": { + "$id": "#/properties/5m/properties/pcap/properties/os_drops", + "type": "integer", + "title": "The os_drops schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 0 + ] + }, + "period": { + "$id": "#/properties/5m/properties/pcap/properties/period", + "type": "object", + "title": "The period schema", + "description": "An explanation about the purpose of this instance.", + "default": {}, + "examples": [ + { + "length": 31, + "start_ts": 1614874231 + } + ], + "required": [ + "length", + "start_ts" + ], + "properties": { + "length": { + "$id": "#/properties/5m/properties/pcap/properties/period/properties/length", + "type": "integer", + "title": "The length schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 31 + ] + }, + "start_ts": { + "$id": "#/properties/5m/properties/pcap/properties/period/properties/start_ts", + "type": "integer", + "title": "The start_ts schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 1614874231 + ] + } + }, + "additionalProperties": true + }, + "tcp_reassembly_errors": { + "$id": "#/properties/5m/properties/pcap/properties/tcp_reassembly_errors", + "type": "integer", + "title": "The tcp_reassembly_errors schema", + "description": "An explanation about the purpose of this instance.", + "default": 0, + "examples": [ + 0 + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} \ No newline at end of file diff --git a/src/handlers/static_plugins.h b/src/handlers/static_plugins.h index 617da13ab..a2edf5fb9 100644 --- a/src/handlers/static_plugins.h +++ b/src/handlers/static_plugins.h @@ -3,6 +3,7 @@ int import_handler_plugins() { CORRADE_PLUGIN_IMPORT(VisorHandlerNet); CORRADE_PLUGIN_IMPORT(VisorHandlerDns); + CORRADE_PLUGIN_IMPORT(VisorHandlerPcap); return 0; } diff --git a/src/inputs/pcap/PcapInputStream.cpp b/src/inputs/pcap/PcapInputStream.cpp index 5a7a01def..640a5fbe5 100644 --- a/src/inputs/pcap/PcapInputStream.cpp +++ b/src/inputs/pcap/PcapInputStream.cpp @@ -3,8 +3,8 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ #include "PcapInputStream.h" -#include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -12,10 +12,10 @@ #pragma GCC diagnostic ignored "-Wpedantic" #include #include +#include #include #include #include -#include #pragma GCC diagnostic pop #include #include @@ -55,10 +55,10 @@ static void _packet_arrives_cb(pcpp::RawPacket *rawPacket, [[maybe_unused]] pcpp stream->process_raw_packet(rawPacket); } -static void _pcap_stats_update([[maybe_unused]] pcpp::IPcapDevice::PcapStats& stats, [[maybe_unused]] void *cookie) +static void _pcap_stats_update(pcpp::IPcapDevice::PcapStats &stats, void *cookie) { - // auto stream = static_cast(cookie); - // TODO expose this + auto stream = static_cast(cookie); + stream->process_pcap_stats(stats); } PcapInputStream::PcapInputStream(const std::string &name) @@ -70,6 +70,7 @@ PcapInputStream::PcapInputStream(const std::string &name) _tcp_connection_end_cb, {true, 5, 500, 50}) { + pcpp::LoggerPP::getInstance().suppressErrors(); } PcapInputStream::~PcapInputStream() @@ -111,15 +112,13 @@ void PcapInputStream::start() auto req_source = config_get("pcap_source"); if (req_source == "libpcap") { _cur_pcap_source = PcapSource::libpcap; - } - else if (req_source == "af_packet") { + } else if (req_source == "af_packet") { #ifndef __linux__ throw PcapException("af_packet is only available on linux"); #else _cur_pcap_source = PcapSource::af_packet; #endif - } - else { + } else { throw PcapException("unknown pcap source"); } } @@ -143,15 +142,13 @@ void PcapInputStream::start() } _get_hosts_from_libpcap_iface(); _open_libpcap_iface(config_get("bpf")); - } - else if (_cur_pcap_source == PcapSource::af_packet) { + } else if (_cur_pcap_source == PcapSource::af_packet) { #ifndef __linux__ assert(true); #else _open_af_packet_iface(TARGET, config_get("bpf")); #endif - } - else { + } else { assert(true); } @@ -197,11 +194,16 @@ void PcapInputStream::tcp_connection_end(const pcpp::ConnectionData &connectionD tcp_connection_end_signal(connectionData, reason); } +void PcapInputStream::process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats) +{ + pcap_stats_signal(stats); +} + void PcapInputStream::process_raw_packet(pcpp::RawPacket *rawPacket) { pcpp::ProtocolType l3(pcpp::UnknownProtocol), l4(pcpp::UnknownProtocol); - pcpp::Packet packet(rawPacket, pcpp::OsiModelTransportLayer); + pcpp::Packet packet(rawPacket, pcpp::TCP | pcpp::UDP); if (packet.isPacketOfType(pcpp::IPv4)) { l3 = pcpp::IPv4; } else if (packet.isPacketOfType(pcpp::IPv6)) { @@ -245,7 +247,20 @@ void PcapInputStream::process_raw_packet(pcpp::RawPacket *rawPacket) if (l4 == pcpp::UDP) { udp_signal(packet, dir, l3, pcpp::hash5Tuple(&packet), rawPacket->getPacketTimeStamp()); } else if (l4 == pcpp::TCP) { - _tcp_reassembly.reassemblePacket(rawPacket); + auto result = _tcp_reassembly.reassemblePacket(packet); + switch (result) { + case pcpp::TcpReassembly::Error_PacketDoesNotMatchFlow: + case pcpp::TcpReassembly::NonTcpPacket: + case pcpp::TcpReassembly::NonIpPacket: + tcp_reassembly_error_signal(packet, dir, l3, rawPacket->getPacketTimeStamp()); + case pcpp::TcpReassembly::TcpMessageHandled: + case pcpp::TcpReassembly::OutOfOrderTcpMessageBuffered: + case pcpp::TcpReassembly::FIN_RSTWithNoData: + case pcpp::TcpReassembly::Ignore_PacketWithNoData: + case pcpp::TcpReassembly::Ignore_PacketOfClosedFlow: + case pcpp::TcpReassembly::Ignore_Retransimission: + break; + } } else { // unsupported layer3 protocol } @@ -304,11 +319,11 @@ void PcapInputStream::_open_pcap(const std::string &fileName, const std::string } #ifdef __linux__ -void PcapInputStream::_open_af_packet_iface(const std::string &iface, const std::string &bpfFilter) { +void PcapInputStream::_open_af_packet_iface(const std::string &iface, const std::string &bpfFilter) +{ _af_device = std::make_unique(this, _packet_arrives_cb, bpfFilter, iface); _af_device->start_capture(); - } #endif @@ -337,7 +352,7 @@ void PcapInputStream::_open_libpcap_iface(const std::string &bpfFilter) // try to open device if (!_pcapDevice->open(config)) { - throw PcapException("Cannot open interface for packing capture"); + throw PcapException("Cannot open interface for packet capture"); } // set BPF filter if set by the user @@ -387,7 +402,7 @@ void PcapInputStream::_get_hosts_from_libpcap_iface() } } -void PcapInputStream::info_json(json& j) const +void PcapInputStream::info_json(json &j) const { _common_info_json(j); json info; diff --git a/src/inputs/pcap/PcapInputStream.h b/src/inputs/pcap/PcapInputStream.h index 53d9bef0c..1ad153d0a 100644 --- a/src/inputs/pcap/PcapInputStream.h +++ b/src/inputs/pcap/PcapInputStream.h @@ -4,7 +4,6 @@ #pragma once - #include "InputStream.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" @@ -82,7 +81,7 @@ class PcapInputStream : public visor::InputStream void info_json(json &j) const override; size_t consumer_count() const override { - return packet_signal.slot_count() + udp_signal.slot_count() + start_tstamp_signal.slot_count() + tcp_message_ready_signal.slot_count() + tcp_connection_start_signal.slot_count() + tcp_connection_end_signal.slot_count(); + return packet_signal.slot_count() + udp_signal.slot_count() + start_tstamp_signal.slot_count() + tcp_message_ready_signal.slot_count() + tcp_connection_start_signal.slot_count() + tcp_connection_end_signal.slot_count() + tcp_reassembly_error_signal.slot_count() + pcap_stats_signal.slot_count(); } // utilities @@ -90,6 +89,7 @@ class PcapInputStream : public visor::InputStream // public methods that can be called from a static callback method via cookie, required by PcapPlusPlus void process_raw_packet(pcpp::RawPacket *rawPacket); + void process_pcap_stats(const pcpp::IPcapDevice::PcapStats &stats); void tcp_message_ready(int8_t side, const pcpp::TcpStreamData &tcpData); void tcp_connection_start(const pcpp::ConnectionData &connectionData); void tcp_connection_end(const pcpp::ConnectionData &connectionData, pcpp::TcpReassembly::ConnectionEndReason reason); @@ -104,7 +104,8 @@ class PcapInputStream : public visor::InputStream mutable sigslot::signal tcp_message_ready_signal; mutable sigslot::signal tcp_connection_start_signal; mutable sigslot::signal tcp_connection_end_signal; + mutable sigslot::signal tcp_reassembly_error_signal; + mutable sigslot::signal pcap_stats_signal; }; } -