[cell_container] compress chunks

NEWS.md

@@ -80,7 +80,7 @@ Bug Fixes:
 * Reduced indexing time for plain text and JSON-lines logs.
 * Reduced memory footprint.
 * Improved search performance.
-* Improved DB view performance.
+* Reduced DB view CPU and memory usage.
 * Reduce time to open help text.
 * Improved performance of log virtual tables when ordering the
   result by `log_line DESC`.

src/base/auto_mem.hh

@@ -32,8 +32,8 @@
 #ifndef lnav_auto_mem_hh
 #define lnav_auto_mem_hh
 
-#include <exception>
 #include <iterator>
+#include <memory>
 #include <string>
 #include <utility>
 
@@ -242,6 +242,11 @@ public:
         std::swap(this->ab_capacity, other.ab_capacity);
     }
 
+    unsigned char* u_in()
+    {
+        return reinterpret_cast<unsigned char*>(this->ab_buffer);
+    }
+
     char* in() { return this->ab_buffer; }
 
     char* at(size_t offset) { return &this->ab_buffer[offset]; }
@@ -325,6 +330,15 @@ public:
         return retval;
     }
 
+    std::unique_ptr<const unsigned char[]>
+    to_unique() const
+    {
+        auto retval = std::make_unique<unsigned char[]>(this->ab_size);
+
+        memcpy(retval.get(), this->ab_buffer, this->ab_size);
+        return retval;
+    }
+
     size_t size() const { return this->ab_size; }
 
     size_t bitmap_size() const { return this->ab_size * 8; }

src/base/cell_container.cc

@@ -34,6 +34,8 @@
 
 #include "cell_container.hh"
 
+#include <zlib.h>
+
 namespace lnav {
 
 static constexpr auto DEFAULT_CHUNK_SIZE = size_t{32 * 1024};
@@ -45,15 +47,48 @@ combine_type_value(uint8_t type, uint8_t subvalue)
     return type | subvalue << 2U;
 }
 
-cell_chunk::cell_chunk(size_t capacity)
-    : cc_data(std::make_unique<unsigned char[]>(capacity)),
-      cc_capacity(capacity)
+cell_chunk::cell_chunk(cell_container* parent,
+                       std::unique_ptr<unsigned char[]> data,
+                       size_t capacity)
+    : cc_parent(parent), cc_data(std::move(data)), cc_capacity(capacity)
+{
+}
+
+void
+cell_chunk::reset()
 {
+    this->cc_next.reset();
+    this->cc_size = 0;
+    if (this->cc_data == nullptr) {
+        this->cc_data = std::make_unique<unsigned char[]>(this->cc_capacity);
+    }
+    this->cc_compressed.reset();
+}
+
+void
+cell_chunk::load() const
+{
+    if (this->cc_data != nullptr) {
+        return;
+    }
+
+    this->cc_data = std::make_unique<unsigned char[]>(this->cc_capacity);
+    auto cap = this->cc_capacity;
+    auto rc = uncompress(this->cc_data.get(),
+                         &cap,
+                         this->cc_compressed.get(),
+                         this->cc_compressed_size);
+    ensure(rc == Z_OK);
 }
 
 cell_container::cell_container()
-    : cc_first(std::make_unique<cell_chunk>(DEFAULT_CHUNK_SIZE)),
-      cc_last(cc_first.get())
+    : cc_first(std::make_unique<cell_chunk>(
+          this,
+          std::make_unique<unsigned char[]>(DEFAULT_CHUNK_SIZE),
+          DEFAULT_CHUNK_SIZE)),
+      cc_last(cc_first.get()),
+      cc_compress_buffer(auto_buffer::alloc(compressBound(DEFAULT_CHUNK_SIZE))),
+      cc_chunk_cache({})
 {
     // log_debug("cell container %p  chunk %p", this, this->cc_last);
 }
@@ -62,9 +97,30 @@ unsigned char*
 cell_container::alloc(size_t amount)
 {
     if (this->cc_last->available() < amount) {
+        auto last = this->cc_last;
+
+        auto buflen = compressBound(last->cc_size);
+        if (this->cc_compress_buffer.capacity() < buflen) {
+            this->cc_compress_buffer.expand_to(buflen);
+        }
+
+        auto rc = compress2(this->cc_compress_buffer.u_in(),
+                            &buflen,
+                            last->cc_data.get(),
+                            last->cc_size,
+                            2);
+        require(rc == Z_OK);
+        this->cc_compress_buffer.resize(buflen);
+        last->cc_compressed = this->cc_compress_buffer.to_unique();
+        last->cc_compressed_size = buflen;
+
         auto chunk_size = std::max(amount, DEFAULT_CHUNK_SIZE);
-        this->cc_last->cc_next = std::make_unique<cell_chunk>(chunk_size);
-        this->cc_last = this->cc_last->cc_next.get();
+        if (chunk_size > last->cc_capacity) {
+            last->cc_data = std::make_unique<unsigned char[]>(chunk_size);
+        }
+        last->cc_next = std::make_unique<cell_chunk>(
+            this, std::move(last->cc_data), chunk_size);
+        this->cc_last = last->cc_next.get();
         // log_debug("allocating chunk with %lu %p", chunk_size, this->cc_last);
 
         ensure(this->cc_last->cc_capacity >= chunk_size);
@@ -73,6 +129,30 @@ cell_container::alloc(size_t amount)
     return this->cc_last->alloc(amount);
 }
 
+void
+cell_container::load_chunk_into_cache(const cell_chunk* cc)
+{
+    if (cc->cc_data != nullptr) {
+        return;
+    }
+
+    if (this->cc_chunk_cache[2] != nullptr) {
+        this->cc_chunk_cache[2]->evict();
+    }
+    this->cc_chunk_cache[2] = this->cc_chunk_cache[1];
+    this->cc_chunk_cache[1] = this->cc_chunk_cache[0];
+    this->cc_chunk_cache[0] = cc;
+    cc->load();
+}
+
+void
+cell_container::reset()
+{
+    this->cc_last = this->cc_first.get();
+    this->cc_first->reset();
+    this->cc_chunk_cache = {};
+}
+
 void
 cell_container::push_null_cell()
 {
@@ -171,6 +251,7 @@ cell_container::cursor::sync() const
     auto next_offset = this->c_offset;
 
     if (next_offset < cc->cc_size) {
+        this->c_chunk->cc_parent->load_chunk_into_cache(cc);
         return *this;
     }
 
@@ -179,6 +260,7 @@ cell_container::cursor::sync() const
         return std::nullopt;
     }
 
+    this->c_chunk->cc_parent->load_chunk_into_cache(cc);
     return cursor{cc, size_t{0}};
 }
 
@@ -209,6 +291,9 @@ cell_container::cursor::next() const
     auto next_offset = this->c_offset + advance;
     if (this->c_offset + advance >= this->c_chunk->cc_size) {
         cc = this->c_chunk->cc_next.get();
+        if (cc != nullptr) {
+            this->c_chunk->cc_parent->load_chunk_into_cache(cc);
+        }
         next_offset = 0;
 #if 0
         log_debug("advanced past current chunk %p, going to %p %lu",
@@ -233,6 +318,7 @@ cell_container::cursor::next() const
         next_offset = 0;
     }
 
+    this->c_chunk->cc_parent->load_chunk_into_cache(cc);
     return cursor{cc, next_offset};
 }
 

src/base/cell_container.hh

@@ -30,10 +30,12 @@
 #ifndef lnav_cell_container_hh
 #define lnav_cell_container_hh
 
+#include <array>
 #include <cstdint>
 #include <memory>
 #include <optional>
 
+#include "auto_mem.hh"
 #include "intern_string.hh"
 #include "lnav_log.hh"
 
@@ -48,8 +50,12 @@ constexpr uint8_t CT_TEXT = 3;
 
 }  // namespace cell_type
 
+struct cell_container;
+
 struct cell_chunk {
-    explicit cell_chunk(size_t capacity);
+    cell_chunk(cell_container* parent,
+               std::unique_ptr<unsigned char[]> data,
+               size_t capacity);
 
     size_t available() const { return this->cc_capacity - this->cc_size; }
 
@@ -61,16 +67,19 @@ struct cell_chunk {
         return retval;
     }
 
-    void reset()
-    {
-        this->cc_next.reset();
-        this->cc_size = 0;
-    }
+    void reset();
+
+    void load() const;
+
+    void evict() const { this->cc_data.reset(); }
 
+    cell_container* cc_parent;
     std::unique_ptr<cell_chunk> cc_next;
-    std::unique_ptr<unsigned char[]> cc_data;
+    mutable std::unique_ptr<unsigned char[]> cc_data;
     const size_t cc_capacity;
     size_t cc_size{0};
+    std::unique_ptr<const unsigned char[]> cc_compressed;
+    size_t cc_compressed_size{0};
 };
 
 struct cell_container {
@@ -148,14 +157,16 @@ struct cell_container {
 
     unsigned char* alloc(size_t amount);
 
-    void reset()
-    {
-        this->cc_last = this->cc_first.get();
-        this->cc_first->reset();
-    }
+    void load_chunk_into_cache(const cell_chunk* cc);
+
+    void reset();
 
     std::unique_ptr<cell_chunk> cc_first;
     cell_chunk* cc_last;
+    auto_buffer cc_compress_buffer;
+
+    static constexpr size_t CHUNK_CACHE_SIZE = 3;
+    std::array<const cell_chunk*, CHUNK_CACHE_SIZE> cc_chunk_cache;
 };
 
 }  // namespace lnav

src/base/cell_container.tests.cc

@@ -76,23 +76,24 @@ TEST_CASE("cell_container-basic")
         auto str3 = string_fragment::from_const("bye");
 
         auto cont = lnav::cell_container();
-        auto cell1 = cont.end_cursor();
+        auto start_cursor = cont.end_cursor();
         cont.push_text_cell(str1);
+        cont.push_text_cell(str2);
+        cont.push_text_cell(str3);
 
+        auto cell1 = start_cursor.sync().value();
         CHECK(cell1.get_type() == lnav::cell_type::CT_TEXT);
         auto t = cell1.get_text();
         printf("1 wow %.*s\n", t.length(), t.data());
         CHECK(t == str1);
 
-        cont.push_text_cell(str2);
         auto cell2 = cell1.next();
         CHECK(cell2.has_value());
         CHECK(cell2->get_type() == lnav::cell_type::CT_TEXT);
         auto t2 = cell2->get_text();
         printf("2 wow %.*s\n", t2.length(), t2.data());
         CHECK(cell2->get_text() == str2);
 
-        cont.push_text_cell(str3);
         auto cell3 = cell2->next();
         CHECK(cell3.has_value());
         CHECK(cell3->get_type() == lnav::cell_type::CT_TEXT);
@@ -153,9 +154,10 @@ TEST_CASE("cell_container-basic")
         auto str1 = string_fragment::from_str(short_str1);
 
         auto cont = lnav::cell_container();
-        auto cell1 = cont.end_cursor();
+        auto start_cursor = cont.end_cursor();
         cont.push_text_cell(str1);
 
+        auto cell1 = start_cursor.sync().value();
         CHECK(cell1.get_type() == lnav::cell_type::CT_TEXT);
         auto t = cell1.get_text();
         printf("1 wow %.*s\n", t.length(), t.data());

src/base/humanize.network.tests.cc

@@ -72,7 +72,7 @@ TEST_CASE("humanize::network::path")
         CHECK(!rp.p_locality.l_service.has_value());
         CHECK(rp.p_path == "/var/log");
 
-        CHECK(fmt::format("{}", rp.p_locality)
+        CHECK(fmt::format(FMT_STRING("{}"), rp.p_locality)
               == "dean@[fe80::184f:c67:baf1:fe02%en0]");
     }
 

src/command_executor.cc

@@ -483,8 +483,26 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg)
     }
 
     if (last_is_readonly && !ec.ec_label_source_stack.empty()) {
-        ec.ec_label_source_stack.back()->dls_query_end
-            = std::chrono::steady_clock::now();
+        auto& dls = *ec.ec_label_source_stack.back();
+        dls.dls_query_end = std::chrono::steady_clock::now();
+
+        size_t memory_usage = 0, total_size = 0, cached_chunks = 0;
+        for (auto cc = dls.dls_cell_container.cc_first.get(); cc != nullptr;
+             cc = cc->cc_next.get())
+        {
+            total_size += cc->cc_capacity;
+            if (cc->cc_data) {
+                cached_chunks += 1;
+                memory_usage += cc->cc_capacity;
+            } else {
+                memory_usage += cc->cc_compressed_size;
+            }
+        }
+        log_debug(
+            "cell memory footprint: total=%zu; actual=%zu; cached-chunks=%zu",
+            total_size,
+            memory_usage,
+            cached_chunks);
     }
     gettimeofday(&end_tv, nullptr);
     if (retcode == SQLITE_DONE) {

src/db_sub_source.cc

@@ -178,6 +178,9 @@ db_label_source::text_attrs_for_line(textview_curses& tc,
                                      int row,
                                      string_attrs_t& sa)
 {
+    static const auto NUM_ATTR = VC_ROLE.value(role_t::VCR_NUMBER);
+    static const auto VLINE_ATTR = VC_GRAPHIC.value(NCACS_VLINE);
+
     line_range lr(0, 0);
     const line_range lr2(0, -1);
 
@@ -205,11 +208,11 @@ db_label_source::text_attrs_for_line(textview_curses& tc,
 
         if (hm.is_graphable()) {
             lr.lr_end += this->dls_cell_width[lpc];
-            sa.emplace_back(lr, VC_ROLE.value(role_t::VCR_NUMBER));
+            sa.emplace_back(lr, NUM_ATTR);
         }
         lr.lr_start += this->dls_cell_width[lpc];
         lr.lr_end = lr.lr_start + 1;
-        sa.emplace_back(lr, VC_GRAPHIC.value(NCACS_VLINE));
+        sa.emplace_back(lr, VLINE_ATTR);
         lr.lr_start += 1;
     }
 
@@ -805,6 +808,8 @@ db_label_source::get_row_as_string(vis_line_t row)
 
     if (this->dls_headers.size() == 1) {
         return this->dls_row_cursors[row]
+            .sync()
+            .value()
             .to_string_fragment(this->dls_cell_allocator)
             .to_string();
     }