Fix debug info size statistics for split dwarf (llvm#80218)

`statistics dump` command relies on `SymbolFile::GetDebugInfoSize()` to
get total debug info size.
The current implementation is missing debug info for split dwarf
scenarios which requires getting debug info from separate dwo/dwp files.
This patch fixes this issue for split dwarf by parsing debug info from
dwp/dwo.

New yaml tests are added.

---------

Co-authored-by: jeffreytan81 <jeffreytan@fb.com>

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

@@ -2667,6 +2667,29 @@ static bool UpdateCompilerContextForSimpleTemplateNames(TypeQuery &match) {
   }
   return any_context_updated;
 }
+
+uint64_t SymbolFileDWARF::GetDebugInfoSize() {
+  DWARFDebugInfo &info = DebugInfo();
+  uint32_t num_comp_units = info.GetNumUnits();
+
+  uint64_t debug_info_size = SymbolFileCommon::GetDebugInfoSize();
+  // In dwp scenario, debug info == skeleton debug info + dwp debug info.
+  if (std::shared_ptr<SymbolFileDWARFDwo> dwp_sp = GetDwpSymbolFile())
+    return debug_info_size + dwp_sp->GetDebugInfoSize();
+
+  // In dwo scenario, debug info == skeleton debug info + all dwo debug info.
+  for (uint32_t i = 0; i < num_comp_units; i++) {
+    DWARFUnit *cu = info.GetUnitAtIndex(i);
+    if (cu == nullptr)
+      continue;
+
+    SymbolFileDWARFDwo *dwo = cu->GetDwoSymbolFile();
+    if (dwo)
+      debug_info_size += dwo->GetDebugInfoSize();
+  }
+  return debug_info_size;
+}
+
 void SymbolFileDWARF::FindTypes(const TypeQuery &query, TypeResults &results) {
 
   // Make sure we haven't already searched this SymbolFile before.

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h

@@ -186,6 +186,8 @@ class SymbolFileDWARF : public SymbolFileCommon {
   GetMangledNamesForFunction(const std::string &scope_qualified_name,
                              std::vector<ConstString> &mangled_names) override;
 
+  uint64_t GetDebugInfoSize() override;
+
   void FindTypes(const lldb_private::TypeQuery &match,
                  lldb_private::TypeResults &results) override;
 

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp

@@ -85,6 +85,17 @@ lldb::offset_t SymbolFileDWARFDwo::GetVendorDWARFOpcodeSize(
   return GetBaseSymbolFile().GetVendorDWARFOpcodeSize(data, data_offset, op);
 }
 
+uint64_t SymbolFileDWARFDwo::GetDebugInfoSize() {
+  // Directly get debug info from current dwo object file's section list
+  // instead of asking SymbolFileCommon::GetDebugInfo() which parses from
+  // owning module which is wrong.
+  SectionList *section_list =
+      m_objfile_sp->GetSectionList(/*update_module_section_list=*/false);
+  if (section_list)
+    return section_list->GetDebugInfoSize();
+  return 0;
+}
+
 bool SymbolFileDWARFDwo::ParseVendorDWARFOpcode(
     uint8_t op, const lldb_private::DataExtractor &opcodes,
     lldb::offset_t &offset, std::vector<lldb_private::Value> &stack) const {

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h

@@ -47,6 +47,8 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
                                           const lldb::offset_t data_offset,
                                           const uint8_t op) const override;
 
+  uint64_t GetDebugInfoSize() override;
+
   bool ParseVendorDWARFOpcode(uint8_t op, const DataExtractor &opcodes,
                               lldb::offset_t &offset,
                               std::vector<Value> &stack) const override;

lldb/test/API/commands/target/debuginfo/TestDebugInfoSize.py

@@ -0,0 +1,135 @@
+"""
+Test SBTarget.GetStatistics() reporting for dwo files.
+"""
+
+import json
+import os
+
+from lldbsuite.test import lldbtest, lldbutil
+from lldbsuite.test.decorators import *
+from lldbsuite.test_event.build_exception import BuildError
+
+
+SKELETON_DEBUGINFO_SIZE = 602
+MAIN_DWO_DEBUGINFO_SIZE = 385
+FOO_DWO_DEBUGINFO_SIZE = 380
+
+
+class TestDebugInfoSize(lldbtest.TestBase):
+    # Concurrency is the primary test factor here, not debug info variants.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def get_output_from_yaml(self):
+        exe = self.getBuildArtifact("a.out")
+        main_dwo = self.getBuildArtifact("a.out-main.dwo")
+        foo_dwo = self.getBuildArtifact("a.out-foo.dwo")
+
+        src_dir = self.getSourceDir()
+        exe_yaml_path = os.path.join(src_dir, "a.out.yaml")
+        self.yaml2obj(exe_yaml_path, exe)
+
+        main_dwo_yaml_path = os.path.join(src_dir, "a.out-main.dwo.yaml")
+        self.yaml2obj(main_dwo_yaml_path, main_dwo)
+
+        foo_dwo_yaml_path = os.path.join(src_dir, "a.out-foo.dwo.yaml")
+        self.yaml2obj(foo_dwo_yaml_path, foo_dwo)
+        return (exe, main_dwo, foo_dwo)
+
+    @add_test_categories(["dwo"])
+    def test_dwo(self):
+        (exe, main_dwo, foo_dwo) = self.get_output_from_yaml()
+
+        # Make sure dwo files exist
+        self.assertTrue(os.path.exists(main_dwo), f'Make sure "{main_dwo}" file exists')
+        self.assertTrue(os.path.exists(foo_dwo), f'Make sure "{foo_dwo}" file exists')
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        stats = target.GetStatistics()
+        stream = lldb.SBStream()
+        res = stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
+        self.assertIn(
+            "totalDebugInfoByteSize",
+            debug_stats,
+            'Make sure the "totalDebugInfoByteSize" key is in target.GetStatistics()',
+        )
+        self.assertEqual(
+            debug_stats["totalDebugInfoByteSize"],
+            SKELETON_DEBUGINFO_SIZE + MAIN_DWO_DEBUGINFO_SIZE + FOO_DWO_DEBUGINFO_SIZE,
+        )
+
+    @add_test_categories(["dwo"])
+    def test_only_load_skeleton_debuginfo(self):
+        (exe, main_dwo, foo_dwo) = self.get_output_from_yaml()
+
+        # REMOVE one of the dwo files
+        os.unlink(main_dwo)
+        os.unlink(foo_dwo)
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        stats = target.GetStatistics()
+        stream = lldb.SBStream()
+        res = stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
+        self.assertIn(
+            "totalDebugInfoByteSize",
+            debug_stats,
+            'Make sure the "totalDebugInfoByteSize" key is in target.GetStatistics()',
+        )
+        self.assertEqual(debug_stats["totalDebugInfoByteSize"], SKELETON_DEBUGINFO_SIZE)
+
+    @add_test_categories(["dwo"])
+    def test_load_partial_dwos(self):
+        (exe, main_dwo, foo_dwo) = self.get_output_from_yaml()
+
+        # REMOVE one of the dwo files
+        os.unlink(main_dwo)
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        stats = target.GetStatistics()
+        stream = lldb.SBStream()
+        res = stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
+        self.assertIn(
+            "totalDebugInfoByteSize",
+            debug_stats,
+            'Make sure the "totalDebugInfoByteSize" key is in target.GetStatistics()',
+        )
+        self.assertEqual(
+            debug_stats["totalDebugInfoByteSize"],
+            SKELETON_DEBUGINFO_SIZE + FOO_DWO_DEBUGINFO_SIZE,
+        )
+
+    @add_test_categories(["dwo"])
+    def test_dwos_loaded_symbols_on_demand(self):
+        (exe, main_dwo, foo_dwo) = self.get_output_from_yaml()
+
+        # Make sure dwo files exist
+        self.assertTrue(os.path.exists(main_dwo), f'Make sure "{main_dwo}" file exists')
+        self.assertTrue(os.path.exists(foo_dwo), f'Make sure "{foo_dwo}" file exists')
+
+        # Load symbols on-demand
+        self.runCmd("settings set symbols.load-on-demand true")
+
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, lldbtest.VALID_TARGET)
+
+        stats = target.GetStatistics()
+        stream = lldb.SBStream()
+        res = stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
+        self.assertIn(
+            "totalDebugInfoByteSize",
+            debug_stats,
+            'Make sure the "totalDebugInfoByteSize" key is in target.GetStatistics()',
+        )
+        self.assertEqual(
+            debug_stats["totalDebugInfoByteSize"],
+            SKELETON_DEBUGINFO_SIZE + MAIN_DWO_DEBUGINFO_SIZE + FOO_DWO_DEBUGINFO_SIZE,
+        )

lldb/test/API/commands/target/debuginfo/a.out-foo.dwo.yaml

@@ -0,0 +1,37 @@
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+  SectionHeaderStringTable: .strtab
+Sections:
+  - Name:            .debug_str_offsets.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         180000000500000000000000040000000800000097000000F6000000
+  - Name:            .debug_str.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE, SHF_MERGE, SHF_STRINGS ]
+    AddressAlign:    0x1
+    EntSize:         0x1
+    Content:         666F6F00696E740046616365626F6F6B20636C616E672076657273696F6E2031352E302E3020287373683A2F2F6769742E7669702E66616365626F6F6B2E636F6D2F646174612F6769747265706F732F6F736D6574612F65787465726E616C2F6C6C766D2D70726F6A656374203435616538646332373465366362636264343064353734353136643533343337393662653135323729002F686F6D652F6A65666672657974616E2F6C6C766D2D73616E642F65787465726E616C2F6C6C766D2D70726F6A6563742F6C6C64622F746573742F4150492F636F6D6D616E64732F7461726765742F6465627567696E666F2F666F6F2E6300612E6F75742D666F6F2E64776F00
+  - Name:            .debug_info.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         2A0000000500050800000000495EA96AE5C99FC401021D00030402000B0000000156000003290000000301050400
+  - Name:            .debug_abbrev.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         01110125251305032576250000022E00111B1206401803253A0B3B0B49133F19000003240003253E0B0B0B000000
+  - Type:            SectionHeaderTable
+    Sections:
+      - Name:            .strtab
+      - Name:            .debug_str_offsets.dwo
+      - Name:            .debug_str.dwo
+      - Name:            .debug_info.dwo
+      - Name:            .debug_abbrev.dwo
+...

lldb/test/API/commands/target/debuginfo/a.out-main.dwo.yaml

@@ -0,0 +1,37 @@
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+  SectionHeaderStringTable: .strtab
+Sections:
+  - Name:            .debug_str_offsets.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         180000000500000000000000050000000900000098000000F8000000
+  - Name:            .debug_str.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE, SHF_MERGE, SHF_STRINGS ]
+    AddressAlign:    0x1
+    EntSize:         0x1
+    Content:         6D61696E00696E740046616365626F6F6B20636C616E672076657273696F6E2031352E302E3020287373683A2F2F6769742E7669702E66616365626F6F6B2E636F6D2F646174612F6769747265706F732F6F736D6574612F65787465726E616C2F6C6C766D2D70726F6A656374203435616538646332373465366362636264343064353734353136643533343337393662653135323729002F686F6D652F6A65666672657974616E2F6C6C766D2D73616E642F65787465726E616C2F6C6C766D2D70726F6A6563742F6C6C64622F746573742F4150492F636F6D6D616E64732F7461726765742F6465627567696E666F2F6D61696E2E6300612E6F75742D6D61696E2E64776F00
+  - Name:            .debug_info.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         2A000000050005080000000037AA38DE48449DD701021D00030402001C0000000156000103290000000301050400
+  - Name:            .debug_abbrev.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         01110125251305032576250000022E00111B1206401803253A0B3B0B271949133F19000003240003253E0B0B0B000000
+  - Type:            SectionHeaderTable
+    Sections:
+      - Name:            .strtab
+      - Name:            .debug_str_offsets.dwo
+      - Name:            .debug_str.dwo
+      - Name:            .debug_info.dwo
+      - Name:            .debug_abbrev.dwo
+...