PolusAI · sameeul · Jul 10, 2024 · Jul 2, 2024 · Jul 5, 2024 · Jul 6, 2024
diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml
@@ -54,8 +54,7 @@ jobs:
                           bash ci-utils/install_prereq_linux.sh &&
                           mkdir -p /tmp/argolid_bld &&
                           cp -r local_install /tmp/argolid_bld
-          CIBW_BEFORE_ALL_LINUX: yum -y install wget &&
-                                   wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 &&
+          CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 &&
                                    tar -xjf nasm-2.15.05.tar.bz2 &&
                                    cd nasm-2.15.05 &&
                                    ./configure && 
@@ -73,7 +72,10 @@ jobs:
           CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja"
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
-          CIBW_BEFORE_TEST_LINUX: yum -y install maven java
+          CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
+                                   sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
+                                   sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && 
+                                   yum -y install maven java
           CIBW_TEST_REQUIRES: bfio tensorstore numpy==1.24.0
           CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v
 

diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml
@@ -52,8 +52,7 @@ jobs:
                           bash ci-utils/install_prereq_linux.sh &&
                           mkdir -p /tmp/argolid_bld &&
                           cp -r local_install /tmp/argolid_bld
-          CIBW_BEFORE_ALL_LINUX: yum -y install wget &&
-                                   wget https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 &&
+          CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 &&
                                    tar -xjf nasm-2.15.05.tar.bz2 &&
                                    cd nasm-2.15.05 &&
                                    ./configure && 
@@ -71,7 +70,10 @@ jobs:
           CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\argolid\\bin;$PATH" ON_GITHUB="TRUE" ARGOLID_DEP_DIR="C:\\TEMP\\argolid_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja"
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}"
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
-          CIBW_BEFORE_TEST_LINUX: yum -y install maven java
+          CIBW_BEFORE_TEST_LINUX: sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo &&
+                                   sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo &&
+                                   sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && 
+                                   yum -y install maven java
           CIBW_TEST_REQUIRES: bfio tensorstore numpy==1.24.0
           CIBW_TEST_COMMAND: python -W default -m unittest discover -s {project}/tests -v
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -52,7 +52,9 @@ find_package(filepattern REQUIRED)
 find_package(Threads QUIET)
 if (Threads_FOUND)
     if (CMAKE_USE_PTHREADS_INIT)
-		SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+      if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+		    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+      endif()
     endif (CMAKE_USE_PTHREADS_INIT)
     list(APPEND Build_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
 else ()

diff --git a/setup.py b/setup.py
@@ -1,7 +1,6 @@
 import os
 import re
 import sys
-import sysconfig
 import versioneer
 import platform
 import subprocess
@@ -102,6 +101,7 @@ def build_extension(self, ext):
     package_dir={"": "src/python"},
     ext_modules=[CMakeExtension("argolid/libargolid")],
     test_suite="tests",
+    install_requires=["pydantic"],
     zip_safe=False,
     python_requires=">=3.8",
 )
diff --git a/src/chunked_base_to_pyr_gen.cpp b/src/chunked_base_to_pyr_gen.cpp
@@ -40,7 +40,7 @@ void ChunkedBaseToPyramid::CreatePyramidImages( const std::string& input_chunked
                                                 int base_level_key,
                                                 int min_dim, 
                                                 VisType v, 
-                                                std::unordered_map<std::int64_t, DSType>& channel_ds_config,
+                                                const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
                                                 BS::thread_pool& th_pool)
 {
     int resolution = 1; // this gets doubled in each level up
@@ -121,7 +121,7 @@ template <typename T>
 void ChunkedBaseToPyramid::WriteDownsampledImage(   const std::string& input_file, const std::string& input_scale_key, 
                                                     const std::string& output_file, const std::string& output_scale_key,
                                                     int resolution, VisType v, 
-                                                    std::unordered_map<std::int64_t, DSType>& channel_ds_config,
+                                                    const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
                                                     BS::thread_pool& th_pool)
 {
     auto [x_dim, y_dim, c_dim, num_dims] = GetZarrParams(v);

diff --git a/src/chunked_base_to_pyr_gen.h b/src/chunked_base_to_pyr_gen.h
@@ -13,15 +13,15 @@ class ChunkedBaseToPyramid{
                                 int base_scale_key,
                                 int min_dim, 
                                 VisType v, 
-                                std::unordered_map<std::int64_t, DSType>& channel_ds_config,
+                                const std::unordered_map<std::int64_t, DSType>& channel_ds_config,
                                 BS::thread_pool& th_pool);
 
 private:
     template<typename T>
     void WriteDownsampledImage( const std::string& input_file, const std::string& input_scale_key, 
                                 const std::string& output_file, const std::string& output_scale_key,
                                 int resolution, VisType v,
-                                std::unordered_map<std::int64_t, DSType>& channel_ds_config, 
+                                const std::unordered_map<std::int64_t, DSType>& channel_ds_config, 
                                 BS::thread_pool& th_pool);
 };
 } // ns argolid
diff --git a/src/pyramid_view.cpp b/src/pyramid_view.cpp
@@ -25,24 +25,21 @@
 #include "tensorstore/kvstore/kvstore.h"
 #include "tensorstore/open.h"
 #include "filepattern/filepattern.h"
-#include <nlohmann/json.hpp>
 
 #include "pyramid_view.h"
 #include "chunked_base_to_pyr_gen.h"
 #include "utilities.h"
 #include "pugixml.hpp"
 #include <plog/Log.h>
 #include "plog/Initializers/RollingFileInitializer.h"
-#include <nlohmann/json.hpp>
-using json = nlohmann::json;
 namespace fs = std::filesystem;
 
 using::tensorstore::Context;
 using::tensorstore::internal_zarr::ChooseBaseDType;
 
 namespace argolid {
 
-  void PyramidView::AssembleBaseLevel(VisType v) {
+  void PyramidView::AssembleBaseLevel(VisType v, const image_map& coordinate_map, const std::string& zarr_array_path) {
      if (v!=VisType::NG_Zarr && v!=VisType::Viv) {
       PLOG_INFO << "Unsupported Pyramid type requested";
       return;
@@ -51,30 +48,30 @@ namespace argolid {
     int grid_x_max = 0, grid_y_max = 0, grid_c_max = 0;
 
     int img_count = 0;
-    for (const auto & [name, location]: base_image_map) {
+    for (const auto & [name, location]: coordinate_map) {
       const auto[gx, gy, gc] = location;
       gc > grid_c_max ? grid_c_max = gc : grid_c_max = grid_c_max;
       gx > grid_x_max ? grid_x_max = gx : grid_x_max = grid_x_max;
       gy > grid_y_max ? grid_y_max = gy : grid_y_max = grid_y_max;
       ++img_count;
     }
-    PLOG_INFO << "Total images found: " << img_count << std::endl;
+    PLOG_DEBUG << "Total images found: " << img_count << std::endl;
     auto t1 = std::chrono::high_resolution_clock::now();
     auto [x_dim, y_dim, c_dim, num_dims] = GetZarrParams(v);
 
     ImageInfo whole_image;
 
     if (img_count != 0) {
       size_t write_failed_count = 0;
-      const auto & sample_tiff_file = image_coll_path + "/" + base_image_map.begin() -> first;
+      const auto & sample_tiff_file = image_coll_path + "/" + coordinate_map.begin() -> first;
       TENSORSTORE_CHECK_OK_AND_ASSIGN(auto test_source, tensorstore::Open(
         GetOmeTiffSpecToRead(sample_tiff_file),
         tensorstore::OpenMode::open,
         tensorstore::ReadWriteMode::read).result());
       auto test_image_shape = test_source.domain().shape();
 
-      whole_image._chunk_size_x = test_image_shape[4];
-      whole_image._chunk_size_y = test_image_shape[3];
+      whole_image._chunk_size_x = test_image_shape[4] + 2*x_spacing;
+      whole_image._chunk_size_y = test_image_shape[3] + 2*y_spacing;
       whole_image._full_image_width = (grid_x_max + 1) * whole_image._chunk_size_x;
       whole_image._full_image_height = (grid_y_max + 1) * whole_image._chunk_size_y;
       whole_image._num_channels = grid_c_max + 1;
@@ -88,8 +85,8 @@ namespace argolid {
       whole_image._data_type = test_source.dtype().name();
       new_image_shape[c_dim] = whole_image._num_channels;
 
-      auto output_spec = [&test_source, &new_image_shape, &chunk_shape, this]() {
-          return GetZarrSpecToWrite(base_zarr_path, new_image_shape, chunk_shape, ChooseBaseDType(test_source.dtype()).value().encoded_dtype);
+      auto output_spec = [&test_source, &new_image_shape, &chunk_shape, &zarr_array_path, this]() {
+          return GetZarrSpecToWrite(zarr_array_path, new_image_shape, chunk_shape, ChooseBaseDType(test_source.dtype()).value().encoded_dtype);
       }();
 
       TENSORSTORE_CHECK_OK_AND_ASSIGN(auto dest, tensorstore::Open(
@@ -99,14 +96,14 @@ namespace argolid {
         tensorstore::ReadWriteMode::write).result());
 
       auto t4 = std::chrono::high_resolution_clock::now();
-      for (const auto & [file_name, location]: base_image_map) {
+      for (const auto & [file_name, location]: coordinate_map) {
         th_pool.push_task([ &dest, file_name=file_name, location=location, x_dim=x_dim, y_dim=y_dim, c_dim=c_dim, v, &whole_image, this]() {
 
           TENSORSTORE_CHECK_OK_AND_ASSIGN(auto source, tensorstore::Open(
             GetOmeTiffSpecToRead(image_coll_path + "/" + file_name),
             tensorstore::OpenMode::open,
             tensorstore::ReadWriteMode::read).result());
-          PLOG_INFO << "Opening " << file_name;
+          PLOG_DEBUG << "Opening " << file_name;
           auto image_shape = source.domain().shape();
           auto image_width = image_shape[4];
           auto image_height = image_shape[3];
@@ -127,12 +124,12 @@ namespace argolid {
           tensorstore::IndexTransform < > transform = tensorstore::IdentityTransform(dest.domain());
           if (v == VisType::NG_Zarr) {
             transform = (std::move(transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
-              tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y, image_height) |
-              tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x, image_width)).value();
+              tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y + y_spacing, image_height) |
+              tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x + x_spacing, image_width)).value();
           } else if (v == VisType::Viv) {
             transform = (std::move(transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
-              tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y, image_height) |
-              tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x, image_width)).value();
+              tensorstore::Dims(y_dim).SizedInterval(y_grid * whole_image._chunk_size_y + y_spacing, image_height) |
+              tensorstore::Dims(x_dim).SizedInterval(x_grid * whole_image._chunk_size_x + x_spacing, image_width)).value();
           }
           tensorstore::Write(array, dest | transform).value();
         });
@@ -143,165 +140,44 @@ namespace argolid {
     base_image = whole_image;
   }
 
-  void PyramidView::ReAssembleBaseLevelWithNewMap(VisType v, const image_map& m, const std::string& output_path) {
-
-     if (v!=VisType::NG_Zarr && v!=VisType::Viv) {
-      PLOG_INFO << "Unsupported Pyramid type requested";
-      return;
-     }
-
-    auto [x_dim, y_dim, c_dim, num_dims] = GetZarrParams(v);
-
-    auto input_spec = [this]() {
-        return GetZarrSpecToRead(base_zarr_path);
-    }();
-
-    TENSORSTORE_CHECK_OK_AND_ASSIGN(auto base_store, tensorstore::Open(
-      input_spec,
-      tensorstore::OpenMode::open,
-      tensorstore::ReadWriteMode::read).result());
-    auto base_image_shape = base_store.domain().shape();
-    auto read_chunk_shape = base_store.chunk_layout().value().read_chunk_shape();
-
-    std::vector < std::int64_t > new_image_shape(num_dims, 1);
-    std::vector < std::int64_t > chunk_shape(num_dims, 1);
-
-    new_image_shape[y_dim] = base_image_shape[y_dim];
-    new_image_shape[x_dim] = base_image_shape[x_dim];
-
-    chunk_shape[y_dim] = read_chunk_shape[y_dim];
-    chunk_shape[x_dim] = read_chunk_shape[x_dim];
-
-    auto open_mode = tensorstore::OpenMode::create;
-    open_mode = open_mode | tensorstore::OpenMode::delete_existing;
-    new_image_shape[c_dim] = base_image_shape[c_dim];
-
-
-    auto output_spec = [v, &output_path, &new_image_shape, & chunk_shape, & base_store, this]() {
-      if (v == VisType::NG_Zarr) {
-        return GetZarrSpecToWrite(output_path + "/0", new_image_shape, chunk_shape, ChooseBaseDType(base_store.dtype()).value().encoded_dtype);
-      } else if (v == VisType::Viv) {
-        return GetZarrSpecToWrite(output_path + "/0", new_image_shape, chunk_shape, ChooseBaseDType(base_store.dtype()).value().encoded_dtype);
-      }
-    }();
-
-    TENSORSTORE_CHECK_OK_AND_ASSIGN(auto dest, tensorstore::Open(
-      output_spec,
-      open_mode,
-      tensorstore::ReadWriteMode::write).result());
-
-    size_t write_failed_count = 0;
-
-    for (const auto & [file_name, location]: m) {
-      // find where to read data from
-      const auto base_location = [file_name=file_name, this]() -> std::optional < std::tuple < std::uint32_t,
-        uint32_t, uint32_t >> {
-          if (auto search = base_image_map.find(file_name); search != base_image_map.end()) {
-            return std::optional {
-              search -> second
-            };
-          } else {
-            return std::nullopt;
-          }
-        }();
-
-      if (!base_location.has_value()) {
-        continue;
-      }
-
-      th_pool.push_task([ &base_store, &dest, file_name=file_name, location=location, base_location, x_dim=x_dim, y_dim=y_dim, c_dim=c_dim, v, this]() {
-
-        const auto & [x_grid_base, y_grid_base, c_grid_base] = base_location.value();
-
-        tensorstore::IndexTransform < > read_transform = tensorstore::IdentityTransform(base_store.domain());
-
-        if (v == VisType::NG_Zarr) {
-          read_transform = (std::move(read_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid_base, 1) |
-            tensorstore::Dims(y_dim).SizedInterval(y_grid_base * base_image._chunk_size_y, base_image._chunk_size_y) |
-            tensorstore::Dims(x_dim).SizedInterval(x_grid_base * base_image._chunk_size_x, base_image._chunk_size_x)).value();
-        } else if (v == VisType::Viv) {
-          read_transform = (std::move(read_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid_base, 1) |
-            tensorstore::Dims(y_dim).SizedInterval(y_grid_base * base_image._chunk_size_y, base_image._chunk_size_y) |
-            tensorstore::Dims(x_dim).SizedInterval(x_grid_base * base_image._chunk_size_x, base_image._chunk_size_x)).value();
-        }
-
-        auto array = tensorstore::AllocateArray({
-            base_image._chunk_size_y,
-            base_image._chunk_size_x
-          }, tensorstore::c_order,
-          tensorstore::value_init, base_store.dtype());
-
-        // initiate a read
-        tensorstore::Read(base_store | read_transform, array).value();
-
-        const auto & [x_grid, y_grid, c_grid] = location;
-
-        tensorstore::IndexTransform < > write_transform = tensorstore::IdentityTransform(dest.domain());
-        if (v == VisType::NG_Zarr) {
-          write_transform = (std::move(write_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
-            tensorstore::Dims(y_dim).SizedInterval(y_grid * base_image._chunk_size_y, base_image._chunk_size_y) |
-            tensorstore::Dims(x_dim).SizedInterval(x_grid * base_image._chunk_size_x, base_image._chunk_size_x)).value();
-        } else if (v == VisType::Viv) {
-          write_transform = (std::move(write_transform) | tensorstore::Dims(c_dim).SizedInterval(c_grid, 1) |
-            tensorstore::Dims(y_dim).SizedInterval(y_grid * base_image._chunk_size_y, base_image._chunk_size_y) |
-            tensorstore::Dims(x_dim).SizedInterval(x_grid * base_image._chunk_size_x, base_image._chunk_size_x)).value();
-        }
-        tensorstore::Write(array, dest | write_transform).value();
-      });
-    }
-
-    th_pool.wait_for_tasks();
-  }
-
-  void PyramidView::GeneratePyramid(std::optional<image_map> map, 
+  void PyramidView::GeneratePyramid(const image_map& map, 
                                     VisType v, 
                                     int min_dim,  
-                                    std::unordered_map<std::int64_t, DSType>& channel_ds_config)
+                                    const std::unordered_map<std::int64_t, DSType>& channel_ds_config)
   {
+    const auto image_dir = pyramid_zarr_path + "/" + image_name +".zarr";
+    if (fs::exists(image_dir)) fs::remove_all(image_dir);
+    PLOG_INFO << "GeneratePyramid Start ";
     if (v!=VisType::NG_Zarr && v!=VisType::Viv) {
       PLOG_INFO << "Unsupported Pyramid type requested";
       return;
     }
-    const auto output_zarr_path = [v, this](){
+
+
+    const auto output_zarr_path = [v, &image_dir, this](){
       if (v==VisType::Viv){
-        return pyramid_zarr_path + "/" + image_name +".zarr/data.zarr/0";
+        return image_dir +"/data.zarr/0";
       } else {
-        return pyramid_zarr_path + "/" + image_name +".zarr/0";
+        return image_dir +"/0";
       }
     }();
-
-    if (map.has_value()){
-      ReAssembleBaseLevelWithNewMap(v,map.value(),output_zarr_path);
-    } else {
-      // copy base level zarr file
-        fs::path destination{output_zarr_path+"/0"};
-        if (!fs::exists(destination)) {
-            fs::create_directories(destination);
-        }
-
-        // Iterate over files in the source directory
-        fs::path source{base_zarr_path};
-        for (const auto& entry : fs::directory_iterator(source)) {
-            const auto& path = entry.path();
-            auto destPath = destination / path.filename();
-
-            // Copy file
-            if (fs::is_regular_file(path)) {
-                fs::copy_file(path, destPath, fs::copy_options::overwrite_existing);
-            }
-        }
-    }
+    PLOG_INFO << "Starting to generate base layer ";
+    AssembleBaseLevel(v, map, output_zarr_path+"/0") ;
+    PLOG_INFO << "Finished generating base layer ";
 
     // generate pyramid
     ChunkedBaseToPyramid base_to_pyramid;
     int base_level_key = 0;
     int max_level = static_cast<int>(ceil(log2(std::max({base_image._full_image_width, base_image._full_image_width}))));
     int min_level = static_cast<int>(ceil(log2(min_dim)));
     auto max_level_key = max_level-min_level+1;
+    PLOG_INFO << "Starting to generate pyramid ";
     base_to_pyramid.CreatePyramidImages(output_zarr_path, output_zarr_path, base_level_key, min_dim, v, channel_ds_config, th_pool);
-
+    PLOG_INFO << "Finished generating pyramid ";
+
     // generate metadata
     WriteMultiscaleMetadataForImageCollection(image_name, pyramid_zarr_path, base_level_key, max_level_key, v, base_image);
+    PLOG_INFO << "GeneratePyramid end ";
   }
 
 } // ns argolid