Support memory reports (#59)

display_utils.py

@@ -1189,3 +1189,30 @@ def prepare_applicable_read_stats(
             cfs_names, db_opts, counters_mngr, files_monitor)
 
     return stats if stats else None
+
+
+def prepare_mem_reps_for_display(mem_reps):
+    assert isinstance(mem_reps, dict)
+
+    disp = dict()
+    for time, rep in mem_reps.items():
+        # Remove all arena entities that do not use any memory
+        arena_stats = \
+            utils.delete_dict_keys_matching_value(rep.arena_stats, "0")
+        disp_arena_stats = {
+            "Total": rep.arena_total,
+            "Entities": arena_stats
+        }
+
+        disp_cfs_stats = {
+            "Total": rep.cfs_total,
+            "CF-s": rep.cfs_stats
+        }
+
+        disp[time] = {
+            "Arena": disp_arena_stats,
+            "CF-s": disp_cfs_stats,
+            "Misc": rep.misc_stats
+        }
+
+    return disp

json_outputter.py

@@ -216,6 +216,18 @@ def get_block_cache_json(parsed_log):
         return utils.NO_BLOCK_CACHE_STATS
 
 
+def get_mem_rep_json(parsed_log):
+    mem_rep_mngr = parsed_log.get_mem_rep_mngr()
+    mem_reps = mem_rep_mngr.get_reports()
+
+    if mem_reps:
+        display_reports = \
+            display_utils.prepare_mem_reps_for_display(mem_reps)
+        return display_reports
+    else:
+        return utils.NO_MEM_REPS
+
+
 def get_json(parsed_log):
     j = dict()
 
@@ -237,6 +249,7 @@ def get_json(parsed_log):
     j["Seeks"] = get_seeks_json(parsed_log)
     j["Warnings"] = get_warnings_json(parsed_log)
     j["Block-Cache-Stats"] = get_block_cache_json(parsed_log)
+    j["Memory-Reporting"] = get_mem_rep_json(parsed_log)
 
     return j
 

log_file.py

@@ -28,6 +28,7 @@
 from log_file_options_parser import LogFileOptionsParser
 from stats_mngr import StatsMngr
 from warnings_mngr import WarningsMngr
+from mem_rep_parser import MemRepParser
 
 get_error_context = utils.get_error_context_from_entry
 
@@ -177,6 +178,7 @@ def __init__(self, log_file_path, log_lines, should_init_baseline_info):
         self.warnings_mngr = WarningsMngr()
         self.stats_mngr = StatsMngr()
         self.counters_mngr = CountersMngr()
+        self.mem_rep_mngr = MemRepParser()
         self.not_parsed_entries = []
 
         self.parse_metadata()
@@ -425,6 +427,13 @@ def try_parse_as_counters_stats_entries(self):
 
         return result
 
+    def try_parse_as_mem_rep_entries(self):
+        result, self.entry_idx = \
+            self.mem_rep_mngr.try_adding_entries(
+                self.log_entries, self.entry_idx)
+
+        return result
+
     def try_processing_in_monitors(self):
         curr_entry = self.get_curr_entry()
         processed, cf_name = \
@@ -473,6 +482,9 @@ def parse_rest_of_log(self):
                     if self.try_parse_as_counters_stats_entries():
                         continue
 
+                    if self.try_parse_as_mem_rep_entries():
+                        continue
+
                     if not self.try_processing_in_monitors():
                         self.not_parsed_entries.append(self.get_curr_entry())
 
@@ -567,6 +579,9 @@ def get_files_monitor(self):
     def get_warnings_mngr(self):
         return self.warnings_mngr
 
+    def get_mem_rep_mngr(self):
+        return self.mem_rep_mngr
+
     def get_entry(self, entry_idx):
         return self.log_entries[entry_idx]
 

mem_rep_parser.py

@@ -0,0 +1,175 @@
+import logging
+
+import regexes
+import re
+from dataclasses import dataclass
+import utils
+
+
+class MemRepParser:
+    @dataclass
+    class Report:
+        arena_total: str = None
+        arena_stats: dict = None
+        cfs_total: str = None
+        cfs_stats: dict = None
+        misc_stats: dict = None
+
+    @staticmethod
+    def is_start_line(line):
+        return re.fullmatch(regexes.MEM_REP_TITLE, line) is not None
+
+    def __init__(self):
+        self.reports = dict()
+
+    def try_adding_entries(self, log_entries, start_entry_idx):
+        entry_idx = start_entry_idx
+        entry = log_entries[entry_idx]
+        mem_rep_lines = utils.remove_empty_lines_at_start(
+            entry.get_msg_lines())
+
+        if not mem_rep_lines:
+            return False, entry_idx
+
+        if not MemRepParser.is_start_line(mem_rep_lines[0]):
+            return False, entry_idx
+
+        try:
+            logging.debug(f"Parsing Memory Report Entry ("
+                          f"{utils.format_line_num_from_entry(entry)}")
+
+            time = entry.get_time()
+            self.add_lines(time, mem_rep_lines)
+        except utils.ParsingError:
+            logging.error(f"Error while parsing Memory Report entry, "
+                          f"Skipping.\nentry:{entry}")
+        entry_idx += 1
+
+        return True, entry_idx
+
+    def add_lines(self, time, mem_rep_lines):
+        assert len(mem_rep_lines) > 0
+        assert MemRepParser.is_start_line(mem_rep_lines[0])
+
+        report = MemRepParser.Report()
+
+        line_idx = 1
+        arena_total, arena_section_stats, line_idx = \
+            MemRepParser.parse_arena_stats_lines(mem_rep_lines, line_idx)
+        report.arena_total = arena_total
+        report.arena_stats = arena_section_stats
+
+        cfs_total, cfs_section_stats, line_idx = \
+            MemRepParser.parse_cfs_stats_lines(mem_rep_lines, line_idx)
+        report.cfs_total = cfs_total
+        report.cfs_stats = cfs_section_stats
+
+        misc_section_stats, line_idx = \
+            MemRepParser.parse_misc_stats_lines(mem_rep_lines, line_idx)
+        report.misc_stats = misc_section_stats
+
+        self.reports[time] = report
+
+    def get_reports(self):
+        return self.reports
+
+    @staticmethod
+    def parse_arena_stats_lines(mem_rep_lines, start_line_idx):
+        line_idx = start_line_idx
+
+        if not MemRepParser.is_arena_stats_title_line(mem_rep_lines[line_idx]):
+            raise utils.ParsingError(
+                f"Missing expected Arena Stats title. line:\n"
+                f"{mem_rep_lines[line_idx]}")
+
+        line_idx += 1
+        total_match = MemRepParser.parse_total_line(mem_rep_lines[line_idx])
+        if not total_match:
+            raise utils.ParsingError(
+                f"Missing expected Arena Total line. line:\n"
+                f"{mem_rep_lines[line_idx]}")
+        arena_total = total_match["usage"]
+
+        line_idx += 1
+        arena_section_stats, line_idx = \
+            MemRepParser.parse_section_stats(mem_rep_lines, line_idx,
+                                             regexes.MEM_REP_ENTITY_USAGE_LINE,
+                                             "entity",
+                                             regexes.MEM_REP_CFS_STATS_TITLE)
+
+        return arena_total, arena_section_stats, line_idx
+
+    @staticmethod
+    def parse_cfs_stats_lines(mem_rep_lines, start_line_idx):
+        line_idx = start_line_idx
+
+        if not MemRepParser.is_cfs_stats_title_line(mem_rep_lines[line_idx]):
+            raise utils.ParsingError(
+                f"Missing expected CF-s Stats title. line:\n"
+                f"{mem_rep_lines[line_idx]}")
+
+        line_idx += 1
+        total_match = MemRepParser.parse_total_line(mem_rep_lines[line_idx])
+        if not total_match:
+            raise utils.ParsingError(
+                f"Missing expected CF-s Total line. line:\n"
+                f"{mem_rep_lines[line_idx]}")
+        cfs_total = total_match["usage"]
+
+        line_idx += 1
+        arena_section_stats, line_idx = \
+            MemRepParser.parse_section_stats(mem_rep_lines, line_idx,
+                                             regexes.MEM_REP_CF_USAGE_LINE,
+                                             "cf")
+
+        return cfs_total, arena_section_stats, line_idx
+
+    @staticmethod
+    def parse_misc_stats_lines(mem_rep_lines, start_line_idx):
+        line_idx = start_line_idx
+        return \
+            MemRepParser.parse_section_stats(mem_rep_lines, line_idx,
+                                             regexes.MEM_REP_ENTITY_USAGE_LINE,
+                                             "entity")
+
+    @staticmethod
+    def parse_section_stats(mem_rep_lines, line_idx,
+                            usage_line_regex,
+                            key_name,
+                            end_line_regex=None):
+        section_stats = dict()
+        while line_idx < len(mem_rep_lines):
+            line = mem_rep_lines[line_idx].strip()
+
+            # Finish when finding the specified end regex
+            if end_line_regex:
+                if re.fullmatch(end_line_regex, line) is not None:
+                    break
+
+            # But also finish when not finding another usage line
+            match = re.fullmatch(usage_line_regex, line)
+            if match is None:
+                break
+
+            section_stats[match[key_name]] = match["usage"]
+            line_idx += 1
+
+        return section_stats, line_idx
+
+    @staticmethod
+    def parse_total_line(line):
+        return re.fullmatch(regexes.MEM_REP_TOTAL_LINE, line)
+
+    @staticmethod
+    def is_total_line(line):
+        return MemRepParser.parse_total_line(line) is not None
+
+    @staticmethod
+    def is_arena_stats_title_line(line):
+        return re.fullmatch(regexes.MEM_REP_ARENA_STATS_TITLE, line) is not \
+               None
+
+    @staticmethod
+    def is_cfs_stats_title_line(line):
+        return re.fullmatch(regexes.MEM_REP_CFS_STATS_TITLE, line) is not \
+               None

regexes.py

@@ -107,6 +107,18 @@
 DB_STATS = fr'^{WS}\*\* DB Stats \*\*{WS}$'
 COMPACTION_STATS = fr'^{WS}\*\* Compaction Stats{WS}{CF_NAME}{WS}\*\*{WS}$'
 
+#
+# MEMORY REPORTING
+#
+MEM_REP_USAGE = r"(?P<usage>.*)"
+MEM_REP_TITLE = fr"^{WS}\*\* Memory Reporting \*\*{WS}$"
+MEM_REP_ARENA_STATS_TITLE = "Arena Stats:"
+MEM_REP_CFS_STATS_TITLE = "CF Stats:"
+MEM_REP_TOTAL_LINE = fr"Total: {MEM_REP_USAGE}"
+MEM_REP_CF_USAGE_LINE = fr"{CF_NAME}: {MEM_REP_USAGE}$"
+MEM_REP_ENTITY_USAGE_LINE = fr"(?P<entity>.*?): {MEM_REP_USAGE}$"
+
+
 FILE_READ_LATENCY_STATS = \
     fr'^{WS}\*\* File Read Latency Histogram By Level{WS}{CF_NAME}' \
     fr'{WS}\*\*{WS}$'