[aot] Load AOT module from Zip archive (taichi-dev#6677)

Good news: It doesn't break any existing loading code. The only
difference is that now `ti_load_aot_module` accepts a path to a `.tcm`
file.

The next step is to load AOT module from memory with
`ti_create_aot_module`.

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

c_api/tests/c_api_interface_test.cpp

@@ -138,3 +138,30 @@ TEST_F(CapiTest, DryRunOpenglAotModule) {
     }
   }
 }
+
+TEST_F(CapiTest, TestLoadTcmAotModule) {
+  if (capi::utils::is_vulkan_available()) {
+    const auto folder_dir = getenv("TAICHI_AOT_FOLDER_PATH");
+
+    std::stringstream aot_mod_ss;
+    aot_mod_ss << folder_dir << "/module.tcm";
+
+    {
+      // Vulkan Runtime
+      TiArch arch = TiArch::TI_ARCH_VULKAN;
+      ti::Runtime runtime(arch);
+      ti::AotModule aot_mod = runtime.load_aot_module(aot_mod_ss.str());
+      ti::Kernel run = aot_mod.get_kernel("run");
+      ti::NdArray<int32_t> arr =
+          runtime.allocate_ndarray<int32_t>({16}, {}, true);
+      run[0] = arr;
+      run.launch();
+      runtime.wait();
+      std::vector<int32_t> data(16);
+      arr.read(data);
+      for (int32_t i = 0; i < 16; ++i) {
+        TI_ASSERT(data.at(i) == i);
+      }
+    }
+  }
+}

taichi/common/CMakeLists.txt

@@ -8,6 +8,9 @@ target_sources(taichi_common
     json.cpp
     logging.cpp
     symbol_version.cpp
+    virtual_dir.cpp
+    zip.cpp
+    miniz.c
   )
 
 target_include_directories(taichi_common

taichi/common/json.cpp

@@ -58,12 +58,10 @@ struct JsonToken {
 };
 
 struct Tokenizer {
-  std::string lit;
-  std::string::const_iterator pos;
-  std::string::const_iterator end;
+  const char *pos;
+  const char *end;
 
-  explicit Tokenizer(const std::string &json)
-      : lit(json), pos(lit.cbegin()), end(lit.cend()) {
+  Tokenizer(const char *beg, const char *end) : pos(beg), end(end) {
   }
 
   // Check the range first before calling this method.
@@ -342,17 +340,20 @@ bool try_parse_impl(Tokenizer &tokenizer, JsonValue &out) {
   throw JsonException("unexpected program state");
 }
 
-JsonValue parse(const std::string &json_lit) {
-  if (json_lit.empty()) {
+JsonValue parse(const char *beg, const char *end) {
+  if (beg == nullptr || end == nullptr || beg >= end) {
     throw JsonException("json text is empty");
   }
   JsonValue rv;
-  Tokenizer tokenizer(json_lit);
+  Tokenizer tokenizer(beg, end);
   if (!try_parse_impl(tokenizer, rv)) {
     throw JsonException("unexpected close token");
   }
   return rv;
 }
+JsonValue parse(const std::string &json_lit) {
+  return parse(json_lit.c_str(), json_lit.c_str() + json_lit.size());
+}
 bool try_parse(const std::string &json_lit, JsonValue &out) {
   try {
     out = parse(json_lit);

taichi/common/json.h

@@ -354,6 +354,7 @@ struct JsonValue {
 
 // Parse JSON literal into and `JsonValue` object. If the JSON is invalid or
 // unsupported, `JsonException` will be raised.
+JsonValue parse(const char *beg, const char *end);
 JsonValue parse(const std::string &json_lit);
 // Returns true when JSON parsing successfully finished and parsed value is
 // returned via `out`. Otherwise, false is returned and out contains incomplete