Sourcemaps WIP

CMakeLists.txt

@@ -78,6 +78,7 @@ configure_file(
 )
 
 include_directories(src ${WABT_BINARY_DIR})
+include_directories(third_party/jsoncpp)
 
 if (COMPILER_IS_MSVC)
   # disable warning C4018: signed/unsigned mismatch
@@ -244,9 +245,18 @@ add_library(libwabt STATIC
   src/option-parser.cc
   src/stream.cc
   src/writer.cc
+  src/source-maps.cc
 )
 set_target_properties(libwabt PROPERTIES OUTPUT_NAME wabt)
 
+add_library(libjsoncpp STATIC
+  third_party/jsoncpp/jsoncpp.cpp
+)
+set_target_properties(libjsoncpp PROPERTIES OUTPUT_NAME jsoncpp)
+if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
+  target_compile_options(libjsoncpp PRIVATE "-Wno-old-style-cast")
+endif ()
+
 if (NOT EMSCRIPTEN)
   if (CODE_COVERAGE)
     add_definitions("-fprofile-arcs -ftest-coverage")
@@ -259,7 +269,7 @@ if (NOT EMSCRIPTEN)
     list(REMOVE_AT ARGV 0)
     add_executable(${name} ${ARGV})
     add_dependencies(everything ${name})
-    target_link_libraries(${name} libwabt)
+    target_link_libraries(${name} libwabt libjsoncpp)
     set_property(TARGET ${name} PROPERTY CXX_STANDARD 11)
     set_property(TARGET ${name} PROPERTY CXX_STANDARD_REQUIRED ON)
     list(APPEND WABT_EXECUTABLES ${name})
@@ -339,10 +349,11 @@ if (NOT EMSCRIPTEN)
     # wabt-unittests
     set(UNITTESTS_SRCS
       src/test-string-view.cc
+      src/test-source-maps.cc
       third_party/gtest/googletest/src/gtest_main.cc
     )
     wabt_executable(wabt-unittests ${UNITTESTS_SRCS})
-    target_link_libraries(wabt-unittests libgtest ${CMAKE_THREAD_LIBS_INIT})
+    target_link_libraries(wabt-unittests libgtest libjsoncpp ${CMAKE_THREAD_LIBS_INIT})
   endif ()
 
   # test running

src/source-maps.cc

@@ -0,0 +1,187 @@
+/*
+ * Copyright 2017 WebAssembly Community Group participants
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "source-maps.h"
+
+#include <algorithm>
+#include <cassert>
+#include <iostream>
+
+#include "json/json.h"
+
+#define INDEX_NONE static_cast<size_t>(-1)
+
+#define INVALID()       \
+  do {                  \
+    if (fatal) abort(); \
+    return false;       \
+  } while (0);
+
+bool SourceMap::Validate(bool fatal) const {
+  for (size_t i = 0; i < segment_groups.size(); ++i) {
+    const auto& group = segment_groups[i];
+    if (i > 0 && group.generated_line <= segment_groups[i - 1].generated_line) {
+      INVALID();
+    }
+    for (size_t j = 0; j < group.segments.size(); ++j) {
+      const auto& seg = group.segments[j];
+      const Segment* last_seg = nullptr;
+      if (j > 0) last_seg = &group.segments[j - 1];
+      if (seg.generated_col_delta == 0) INVALID();
+      if (!seg.has_source && seg.has_name) INVALID();
+      if (!seg.has_source) return true;
+      if (seg.source >= sources.size()) INVALID();
+      if (last_seg) {
+        if (seg.source_line_delta == 0 && seg.source_col_delta == 0) INVALID();
+        // FIXME: This has a limitation that if this seg has a source, the last
+        // one must.
+        if (last_seg->source_line + seg.source_line_delta != seg.source_line) {
+          INVALID();
+        }
+        if (last_seg->source_col + seg.source_col_delta != seg.source_col) {
+          INVALID();
+        }
+      }
+      if (!seg.has_name) return true;
+      if (seg.name >= names.size()) INVALID();
+    }
+  }
+  return true;
+}
+
+static int32_t cmpLocation(const SourceMapGenerator::SourceLocation& a,
+                           const SourceMapGenerator::SourceLocation& b) {
+  int32_t cmp = a.line - b.line;
+  return cmp ? cmp : a.col - b.col;
+}
+
+bool SourceMapGenerator::SourceMapping::operator<(
+    const SourceMapGenerator::SourceMapping& rhs) const {
+  int32_t cmp = cmpLocation(generated, rhs.generated);
+  if (cmp != 0) return cmp < 0;
+  cmp = cmpLocation(original, rhs.original);
+  if (cmp != 0) return cmp < 0;
+  if (source_idx == INDEX_NONE) return rhs.source_idx != INDEX_NONE;
+  if (rhs.source_idx == INDEX_NONE) return false;
+  return source_idx < rhs.source_idx;
+}
+
+bool SourceMapGenerator::SourceMapping::operator==(
+    const SourceMapGenerator::SourceMapping& rhs) const {
+  return !cmpLocation(generated, rhs.generated) &&
+         !cmpLocation(original, rhs.original) && source_idx == rhs.source_idx;
+}
+
+void SourceMapGenerator::SourceMapping::Dump() const {
+  std::cout << "Mapping " << original.line << ":" << original.col << " -> "
+            << generated.line << ":" << generated.col << " in " << source_idx
+            << "\n";
+}
+
+bool SourceMapGenerator::AddMapping(SourceLocation generated,
+                                    SourceLocation original,
+                                    std::string source) {
+  // Validate. For now, original, generated, and source are required
+  if (generated.line == 0 || original.line == 0) return false;
+  map_prepared = false;  // New mapping invalidates compressed map.
+  size_t source_idx = INDEX_NONE;
+  auto s = sources_map.find(source);
+  if (s == sources_map.end()) {
+    source_idx = map.sources.size();
+    map.sources.push_back(source);
+    bool inserted;
+    std::tie(std::ignore, inserted) = sources_map.insert({source, source_idx});
+    assert(inserted);
+  } else {
+    source_idx = s->second;
+  }
+  mappings.push_back({original, generated, source_idx});
+  return true;
+}
+
+void SourceMapGenerator::CompressMappings() {
+  std::sort(mappings.begin(), mappings.end());
+  uint32_t last_gen_line = 0;
+  uint32_t last_gen_col = 0;
+  uint32_t last_source_line = 0;
+  uint32_t last_source_col = 0;
+  map.segment_groups.clear();
+  const SourceMapGenerator::SourceMapping* last_mapping = nullptr;
+  for (const auto& mapping : mappings) {
+    if (mapping.generated.line != last_gen_line) {
+      // Output an empty segment group for each line between the previous
+      // and current.
+      assert(map.segment_groups.empty() ||
+             mapping.generated.line > last_gen_line);  // Not sorted.
+      while (++last_gen_line <= mapping.generated.line) {
+        map.segment_groups.push_back(
+            {last_gen_line, std::vector<SourceMap::Segment>()});
+      }
+      last_gen_line = mapping.generated.line;
+      last_gen_col = 0;
+    }
+    if (last_mapping != nullptr && mapping == *last_mapping) continue;
+    last_mapping = &mapping;
+
+    auto& group = map.segment_groups.back();
+    group.segments.emplace_back();
+    SourceMap::Segment& seg = group.segments.back();
+    seg.generated_col = mapping.generated.col;
+    seg.generated_col_delta = mapping.generated.col - last_gen_col;
+    last_gen_col = mapping.generated.col;
+    seg.has_source = mapping.source_idx != INDEX_NONE;
+    assert(seg.has_source);  // TODO(dschuff): support mappings without source
+    if (seg.has_source) {
+      seg.source_line = mapping.original.line;
+      seg.source_line_delta = mapping.original.line - last_source_line;
+      last_source_line = mapping.original.line;
+      seg.source_col = mapping.original.col;
+      seg.source_col_delta = mapping.original.col - last_source_col;
+      last_source_col = mapping.original.col;
+    }
+    seg.has_name = false;  // TODO(dschuff): add support
+  }
+  map_prepared = true;
+}
+
+std::string SourceMapGenerator::SerializeMappings() {
+  std::vector<std::string> mapping_results;
+  mapping_results.reserve(mappings.size());
+  CompressMappings();
+  Json::Value output;
+  output["version"] = SourceMap::kSourceMapVersion;
+  output["file"] = map.file;
+  output["sourceRoot"] = map.source_root;
+  Json::Value sources(Json::arrayValue);
+  for (const auto& source : map.sources) {
+    sources.append(source);
+  }
+  output["sources"] = sources;
+  std::cout << output;
+  return output.toStyledString();
+}
+
+void SourceMapGenerator::DumpRawMappings() {
+  std::sort(mappings.begin(), mappings.end());
+  std::cout << "Map: " << map.file << " " << map.source_root << "\n"
+            << "Sources [";
+  for (size_t i = 0; i < map.sources.size(); ++i) {
+    std::cout << i << ":" << map.sources[i] << ", ";
+  }
+  std::cout << "]\n";
+  for (const auto& m : mappings) {
+    m.Dump();
+  }
+}

src/source-maps.h

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2017 WebAssembly Community Group participants
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef WABT_SOURCE_MAPS_H
+#define WABT_SOURCE_MAPS_H
+#include <cstdint>
+#include <cstring>
+#include <map>
+#include <string>
+#include <vector>
+
+struct SourceMap {
+  static constexpr const int32_t kSourceMapVersion = 3;
+  // Representation of mappings
+  struct Segment {
+    // Field 1
+    uint32_t generated_col = 0;  // Start column in generated code. Remove?
+    uint32_t generated_col_delta = 0;  // Delta from previous generated col
+    bool has_source = false;           // If true, fields 2-4 will be valid.
+    // Field 2
+    size_t source = 0;  // Index into sources list
+    // Field 3
+    uint32_t source_line = 0;       // Start line in source. Remove?
+    int32_t source_line_delta = 0;  // Delta from previous source line
+    // Field 4
+    uint32_t source_col = 0;       // Start column in source. Remove?
+    int32_t source_col_delta = 0;  // Delta from previous source column
+    bool has_name = false;         // If true, field 5 will be valid.
+    // Field 5
+    size_t name = 0;  // Index into names list
+    Segment() = default;
+    // Explicit constructor, mostly used for testing.
+    Segment(std::pair<uint32_t, int32_t> generated_col_p,
+            std::pair<bool, size_t> source_p,
+            std::pair<uint32_t, int32_t> source_line_p,
+            std::pair<uint32_t, int32_t> source_col_p,
+            std::pair<bool, size_t> name_p) {
+      std::tie(generated_col, generated_col_delta) = generated_col_p;
+      std::tie(has_source, source) = source_p;
+      std::tie(source_line, source_line_delta) = source_line_p;
+      std::tie(source_col, source_col_delta) = source_col_p;
+      std::tie(has_name, name) = name_p;
+    }
+  };
+  struct SegmentGroup {
+    uint32_t generated_line;  // Line in the generated file for all segments
+    std::vector<Segment> segments;
+  };
+
+  // Top level fields
+  std::string file;         // Generated code filename; optional
+  std::string source_root;  // Prepended to entries in sources list; optional
+  std::vector<std::string> sources;          // List of sources use by mappings
+  std::vector<std::string> sources_content;  // Not supported yet.
+  std::vector<std::string> names;            // Not supported yet.
+  std::vector<SegmentGroup> segment_groups;
+
+  SourceMap(std::string file_, std::string source_root_)
+      : file(file_), source_root(source_root_) {}
+
+  void Dump();
+  bool Validate(bool fatal = false) const;
+};
+
+class SourceMapGenerator {
+ public:
+  struct SourceLocation {
+    uint32_t line;
+    uint32_t col;
+  };
+
+  SourceMapGenerator(std::string file_, std::string source_root_)
+      : map(file_, source_root_) {}
+
+  // Returns true if the mapping is valid, and if so adds to the list.
+  bool AddMapping(SourceLocation generated, SourceLocation original,
+                  std::string source);
+  void DumpMappings() { DumpRawMappings(); }
+  const SourceMap& GetMap() {
+    CompressMappings();
+    return map;
+  };
+  std::string SerializeMappings();
+ public:
+  // TODO: make this private? But need to find a way to use it in tests.
+  struct SourceMapping {
+    SourceLocation original;
+    SourceLocation generated;  // Use binary location?
+    size_t source_idx;         // pointer to src?
+    // We don't use the 'name' field currently.
+    bool operator<(const SourceMapping& other) const;
+    bool operator==(const SourceMapping& other) const;
+    void Dump() const;
+  };
+
+ private:
+  void CompressMappings();
+
+  bool map_prepared = false;  // Is the map compressed and ready for export?
+  SourceMap map;
+  std::map<std::string, size_t> sources_map;
+  std::vector<SourceMapping> mappings;
+  // TODO:
+  // Parse from file
+  // Dump to file
+  // Lookup mapping bidirectionally (future?)
+  // Future? Support modifiable mappings (e.g. a way to pin source location to
+  // IR) Add source location without generated mapping (with e.g. an opaque
+  // handle/token) Apply generated-location to each handle
+  void DumpRawMappings();
+};
+
+#endif  // WABT_SOURCE_MAPS_H