minor release 3.2.1 (#101)

* fix #98 remove 300 message. in addition, fix erroneous pcpp::DnsLayer parsing, sometimes causing multiple parses
* issue #100 add tcp reassembly error tracking
* pcap stats handler implementation
* add pcap stream handler to default pktvisord capture
* add pcap handler metrics to cli, make time out 30s
* bump pcapplusplus to 21.05 release
* issue #94 add TLS support to web server
* address issue #84, improve flushing and error messages. change to DAEMON syslog facility
* address #83, improved daemonize and logging.
* address #83, more improved daemonize and logging.
* better error handling and documentation, esp. TLS config
* update download link in read me
* add TLS support to CLI
* handle pcap stats monotonic counter at each new bucket.

.dockerignore

@@ -7,3 +7,4 @@ Dockerfile
 appimage/Dockerfile.part
 appimage/export.sh
 appimage/Makefile
+localconfig/*

.gitignore

@@ -5,4 +5,5 @@ docs/html-documentation-generated*
 integration_tests/external
 golang/pkg/client/version.go
 docs/internals/html
-appimage/*.AppImage
+appimage/*.AppImage
+localconfig/

CMakeLists.txt

@@ -4,12 +4,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 
 # VERSION
 # this is the source of truth for semver version
-project(visor VERSION 3.2.0)
+project(visor VERSION 3.2.1)
 
 # for main line release, this is empty
 # for development release, this is "-develop"
 # for release candidate, this is "-rc"
-set(VISOR_PRERELEASE "")
+set(VISOR_PRERELEASE "-develop")
 
 # these are computed
 set(VISOR_VERSION_NUM "${PROJECT_VERSION}${VISOR_PRERELEASE}")

README.md

@@ -96,7 +96,7 @@ download [on the Releases page](https://github.com/ns1labs/pktvisor/releases). I
 Linux distributions and does not require installation or any other dependencies.
 
 ```shell
-curl https://github.com/ns1labs/pktvisor/releases/download/v3.2.0/pktvisor-x86_64-3.2.0.AppImage --output pktvisor-x86_64.AppImage
+curl -L http://pktvisor.com/download -o pktvisor-x86_64.AppImage
 chmod +x pktvisor-x86_64.AppImage
 ./pktvisor-x86_64.AppImage pktvisord -h
 ```
@@ -153,32 +153,37 @@ or
     pktvisord summarizes data streams and exposes a REST API control plane for configuration and metrics.
 
     IFACE, if specified, is either a network interface or an IP address (4 or 6). If this is specified,
-    a "pcap" input stream will be automatically created, with "net" and "dns" handler modules attached.
+    a "pcap" input stream will be automatically created, with "net", "dns", and "pcap" handler modules attached.
 
     Base Options:
-      -l HOST               Run webserver on the given host or IP [default: localhost]
-      -p PORT               Run webserver on the given port [default: 10853]
-      --admin-api           Enable admin REST API giving complete control plane functionality [default: false]
-                            When not specified, the exposed API is read-only access to summarized metrics.
-                            When specified, write access is enabled for all modules.
       -d                    Daemonize; fork and continue running in the background [default: false]
       -h --help             Show this screen
       -v                    Verbose log output
-      --no-track            Don't send lightweight, anonymous usage metrics.
+      --no-track            Don't send lightweight, anonymous usage metrics
       --version             Show version
+    Web Server Options:
+      -l HOST               Run web server on the given host or IP [default: localhost]
+      -p PORT               Run web server on the given port [default: 10853]
+      --tls                 Enable TLS on the web server
+      --tls-cert FILE       Use given TLS cert. Required if --tls is enabled.
+      --tls-key FILE        Use given TLS private key. Required if --tls is enabled.
+      --admin-api           Enable admin REST API giving complete control plane functionality [default: false]
+                            When not specified, the exposed API is read-only access to summarized metrics.
+                            When specified, write access is enabled for all modules.
+    Geo Options:
       --geo-city FILE       GeoLite2 City database to use for IP to Geo mapping
       --geo-asn FILE        GeoLite2 ASN database to use for IP to ASN mapping
     Logging Options:
       --log-file FILE       Log to the given output file name
       --syslog              Log to syslog
     Prometheus Options:
       --prometheus          Enable native Prometheus metrics at path /metrics
-      --prom-instance ID    Optionally set the 'instance' label to ID
+      --prom-instance ID    Optionally set the 'instance' label to given ID
     Handler Module Defaults:
       --max-deep-sample N   Never deep sample more than N% of streams (an int between 0 and 100) [default: 100]
       --periods P           Hold this many 60 second time periods of history in memory [default: 5]
     pcap Input Module Options:
-      -b BPF                Filter packets using the given BPF string
+      -b BPF                Filter packets using the given tcpdump compatible filter expression. Example: "port 53"
       -H HOSTSPEC           Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically
                             from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64"
                             Specifying this for live capture will append to any automatic detection.

cmd/pktvisor-pcap/CMakeLists.txt

@@ -9,4 +9,6 @@ target_link_libraries(pktvisor-pcap
         PRIVATE
         ${CONAN_LIBS_DOCOPT.CPP}
         Visor::Handler::Net
-        Visor::Handler::Dns)
+        Visor::Handler::Dns
+        Visor::Handler::Pcap
+        )

cmd/pktvisord/main.cpp

@@ -19,6 +19,7 @@
 #include "GeoDB.h"
 #include "handlers/dns/DnsStreamHandler.h"
 #include "handlers/net/NetStreamHandler.h"
+#include "handlers/pcap/PcapStreamHandler.h"
 #include "inputs/pcap/PcapInputStream.h"
 #include "timer.hpp"
 
@@ -32,32 +33,37 @@ static const char USAGE[] =
     pktvisord summarizes data streams and exposes a REST API control plane for configuration and metrics.
 
     IFACE, if specified, is either a network interface or an IP address (4 or 6). If this is specified,
-    a "pcap" input stream will be automatically created, with "net" and "dns" handler modules attached.
+    a "pcap" input stream will be automatically created, with "net", "dns", and "pcap" handler modules attached.
 
     Base Options:
-      -l HOST               Run webserver on the given host or IP [default: localhost]
-      -p PORT               Run webserver on the given port [default: 10853]
-      --admin-api           Enable admin REST API giving complete control plane functionality [default: false]
-                            When not specified, the exposed API is read-only access to summarized metrics.
-                            When specified, write access is enabled for all modules.
       -d                    Daemonize; fork and continue running in the background [default: false]
       -h --help             Show this screen
       -v                    Verbose log output
-      --no-track            Don't send lightweight, anonymous usage metrics.
+      --no-track            Don't send lightweight, anonymous usage metrics
       --version             Show version
+    Web Server Options:
+      -l HOST               Run web server on the given host or IP [default: localhost]
+      -p PORT               Run web server on the given port [default: 10853]
+      --tls                 Enable TLS on the web server
+      --tls-cert FILE       Use given TLS cert. Required if --tls is enabled.
+      --tls-key FILE        Use given TLS private key. Required if --tls is enabled.
+      --admin-api           Enable admin REST API giving complete control plane functionality [default: false]
+                            When not specified, the exposed API is read-only access to summarized metrics.
+                            When specified, write access is enabled for all modules.
+    Geo Options:
       --geo-city FILE       GeoLite2 City database to use for IP to Geo mapping
       --geo-asn FILE        GeoLite2 ASN database to use for IP to ASN mapping
     Logging Options:
       --log-file FILE       Log to the given output file name
       --syslog              Log to syslog
     Prometheus Options:
       --prometheus          Enable native Prometheus metrics at path /metrics
-      --prom-instance ID    Optionally set the 'instance' label to ID
+      --prom-instance ID    Optionally set the 'instance' label to given ID
     Handler Module Defaults:
       --max-deep-sample N   Never deep sample more than N% of streams (an int between 0 and 100) [default: 100]
       --periods P           Hold this many 60 second time periods of history in memory [default: 5]
     pcap Input Module Options:
-      -b BPF                Filter packets using the given BPF string
+      -b BPF                Filter packets using the given tcpdump compatible filter expression. Example: "port 53"
       -H HOSTSPEC           Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically
                             from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64"
                             Specifying this for live capture will append to any automatic detection.
@@ -86,61 +92,43 @@ void initialize_geo(const docopt::value &city, const docopt::value &asn)
 // adapted from LPI becomeDaemon()
 int daemonize()
 {
-    switch (fork()) {
+
+    switch (auto pid = fork()) {
     case -1:
         return -1;
     case 0:
         // Child falls through...
         break;
     default:
         // while parent terminates
+        spdlog::get("pktvisor-daemon")->info("daemonized to PID {}", pid);
         _exit(EXIT_SUCCESS);
     }
 
     // Become leader of new session
     if (setsid() == -1) {
+        spdlog::get("pktvisor-daemon")->error("setsid() fail");
         return -1;
     }
 
-    // Ensure we are not session leader
-    switch (auto pid = fork()) {
-    case -1:
-        return -1;
-    case 0:
-        break;
-    default:
-        std::cerr << "pktvisord running at PID " << pid << std::endl;
-        _exit(EXIT_SUCCESS);
-    }
-
     // Clear file mode creation mask
     umask(0);
 
-    // Change to root directory
-    chdir("/");
-    int maxfd, fd;
-    maxfd = sysconf(_SC_OPEN_MAX);
-    // Limit is indeterminate...
-    if (maxfd == -1) {
-        maxfd = 8192; // so take a guess
-    }
-
-    for (fd = 0; fd < maxfd; fd++) {
-        close(fd);
-    }
-
     // Reopen standard fd's to /dev/null
     close(STDIN_FILENO);
 
-    fd = open("/dev/null", O_RDWR);
+    int fd = open("/dev/null", O_RDWR);
 
     if (fd != STDIN_FILENO) {
+        spdlog::get("pktvisor-daemon")->error("open() fail");
         return -1;
     }
     if (dup2(STDIN_FILENO, STDOUT_FILENO) != STDOUT_FILENO) {
+        spdlog::get("pktvisor-daemon")->error("dup2 fail (STDOUT)");
         return -1;
     }
     if (dup2(STDIN_FILENO, STDERR_FILENO) != STDERR_FILENO) {
+        spdlog::get("pktvisor-daemon")->error("dup2 fail (STDERR)");
         return -1;
     }
 
@@ -155,39 +143,82 @@ int main(int argc, char *argv[])
         true,           // show help if requested
         VISOR_VERSION); // version string
 
-    if (args["-d"].asBool()) {
+    bool daemon{args["-d"].asBool()};
+    if (daemon) {
+        // before we daemonize, if they are using a log file, ensure it can be opened
+        if (args["--log-file"]) {
+            try {
+                auto logger_probe = spdlog::basic_logger_mt("pktvisor-log-probe", args["--log-file"].asString());
+            } catch (const spdlog::spdlog_ex &ex) {
+                // note in daemon mode, this may get swallowed because stdout is already closed
+                std::cerr << "Log init failed: " << ex.what() << std::endl;
+                exit(EXIT_FAILURE);
+            }
+        }
+        auto dlogger = spdlog::stderr_color_st("pktvisor-daemon");
+        dlogger->flush_on(spdlog::level::info);
         if (daemonize()) {
-            std::cerr << "failed to daemonize" << std::endl;
+            dlogger->error("failed to daemonize");
             exit(EXIT_FAILURE);
         }
     }
 
     std::shared_ptr<spdlog::logger> logger;
+    spdlog::flush_on(spdlog::level::err);
     if (args["--log-file"]) {
         try {
             logger = spdlog::basic_logger_mt("pktvisor", args["--log-file"].asString());
+            spdlog::flush_every(std::chrono::seconds(3));
         } catch (const spdlog::spdlog_ex &ex) {
             std::cerr << "Log init failed: " << ex.what() << std::endl;
             exit(EXIT_FAILURE);
         }
     } else if (args["--syslog"].asBool()) {
-        logger = spdlog::syslog_logger_mt("pktvisor", "pktvisord", LOG_PID);
+        logger = spdlog::syslog_logger_mt("pktvisor", "pktvisord", LOG_PID, LOG_DAEMON);
     } else {
         logger = spdlog::stdout_color_mt("pktvisor");
     }
     if (args["-v"].asBool()) {
         logger->set_level(spdlog::level::debug);
     }
 
+    logger->info("{} starting up", VISOR_VERSION);
+
+    // if we are demonized, change to root directory now that (potentially) logs are open
+    if (daemon) {
+        chdir("/");
+    }
+
     PrometheusConfig prom_config;
     if (args["--prometheus"].asBool()) {
         prom_config.path = "/metrics";
         if (args["--prom-instance"]) {
             prom_config.instance = args["--prom-instance"].asString();
         }
     }
-    CoreServer svr(!args["--admin-api"].asBool(), logger, prom_config);
-    svr.set_http_logger([&logger](const auto &req, const auto &res) {
+
+    HttpConfig http_config;
+    http_config.read_only = !args["--admin-api"].asBool();
+    if (args["--tls"].asBool()) {
+        http_config.tls_enabled = true;
+        if (!args["--tls-key"] || !args["--tls-cert"]) {
+            logger->error("you must specify --tls-key and --tls-cert to use --tls");
+            exit(EXIT_FAILURE);
+        }
+        http_config.key = args["--tls-key"].asString();
+        http_config.cert = args["--tls-cert"].asString();
+        logger->info("Enabling TLS with cert {} and key {}", http_config.key, http_config.cert);
+    }
+
+    std::unique_ptr<CoreServer> svr;
+    try {
+        svr = std::make_unique<CoreServer>(logger, http_config, prom_config);
+    } catch (const std::exception &e) {
+        logger->error(e.what());
+        logger->info("exit with failure");
+        exit(EXIT_FAILURE);
+    }
+    svr->set_http_logger([&logger](const auto &req, const auto &res) {
         logger->info("REQUEST: {} {} {}", req.method, req.path, res.status);
         if (res.status == 500) {
             logger->error(res.body);
@@ -196,7 +227,9 @@ int main(int argc, char *argv[])
 
     shutdown_handler = [&]([[maybe_unused]] int signal) {
         logger->info("Shutting down");
-        svr.stop();
+        logger->flush();
+        svr->stop();
+        logger->flush();
     };
     std::signal(SIGINT, signal_handler);
     std::signal(SIGTERM, signal_handler);
@@ -261,14 +294,18 @@ int main(int argc, char *argv[])
             input_stream->config_set("bpf", bpf);
             input_stream->config_set("host_spec", host_spec);
 
-            auto input_manager = svr.input_manager();
-            auto handler_manager = svr.handler_manager();
+            auto input_manager = svr->input_manager();
+            auto handler_manager = svr->handler_manager();
 
             input_manager->module_add(std::move(input_stream));
             auto [input_stream_, stream_mgr_lock] = input_manager->module_get_locked("pcap");
             stream_mgr_lock.unlock();
             auto pcap_stream = dynamic_cast<input::pcap::PcapInputStream *>(input_stream_);
 
+            {
+                auto pcap_module = std::make_unique<handler::pcap::PcapStreamHandler>("pcap", pcap_stream, periods, sample_rate);
+                handler_manager->module_add(std::move(pcap_module));
+            }
             {
                 auto handler_module = std::make_unique<handler::net::NetStreamHandler>("net", pcap_stream, periods, sample_rate);
                 handler_manager->module_add(std::move(handler_module));
@@ -284,6 +321,7 @@ int main(int argc, char *argv[])
 
         } catch (const std::exception &e) {
             logger->error(e.what());
+            logger->info("exit with failure");
             exit(EXIT_FAILURE);
         }
     } else if (!args["--admin-api"].asBool()) {
@@ -294,11 +332,13 @@ int main(int argc, char *argv[])
     }
 
     try {
-        svr.start(host.c_str(), port);
+        svr->start(host.c_str(), port);
     } catch (const std::exception &e) {
         logger->error(e.what());
+        logger->info("exit with failure");
         exit(EXIT_FAILURE);
     }
 
+    logger->info("exit with success");
     exit(EXIT_SUCCESS);
 }

conanfile.txt

@@ -5,8 +5,9 @@ docopt.cpp/0.6.3
 nlohmann_json/3.9.1
 cpp-httplib/0.8.0
 corrade/2020.06
-pcapplusplus/ns1-dev
+pcapplusplus/21.05
 json-schema-validator/2.1.0
+openssl/1.1.1k
 
 [build_requires]
 benchmark/1.5.2