Hashing and merkle tree API (#608)

feat: add APIs for hashing and merkle trees commit+verify

icicle/CMakeLists.txt

@@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 include(cmake/field.cmake)
 include(cmake/curve.cmake)
+include(cmake/hash.cmake)
 
 # Set the default build type to Release if not specified
 if(NOT CMAKE_BUILD_TYPE)
@@ -22,7 +23,7 @@ find_program(CCACHE_PROGRAM ccache)
 # If ccache is found, use it as the compiler launcher
 if(CCACHE_PROGRAM)
     message(STATUS "ccache found: ${CCACHE_PROGRAM}")
-    
+
     # Use ccache for C and C++ compilers
     set(CMAKE_C_COMPILER_LAUNCHER ${CCACHE_PROGRAM})
     set(CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE_PROGRAM})
@@ -44,6 +45,8 @@ option(ECNTT "Build ECNTT" ON)
 option(MSM "Build MSM" ON)
 option(G2 "Build G2 MSM" ON)
 option(EXT_FIELD "Build extension field" ON)
+option(HASH "Build hashes and tree builders" ON)
+option(POSEIDON "Build poseidon hash" ON)
 
 # device API library
 add_library(icicle_device SHARED
@@ -78,7 +81,7 @@ endif()
 if(CURVE)
   set(CURVE_INDEX -1)
   set(FEATURES_STRING "")
-  check_curve(${CURVE} CURVE_INDEX FEATURES_STRING)    
+  check_curve(${CURVE} CURVE_INDEX FEATURES_STRING)
   setup_curve_target(${CURVE} ${CURVE_INDEX} ${FEATURES_STRING})
 elseif(FIELD)
   set(FIELD_INDEX -1)
@@ -87,14 +90,18 @@ elseif(FIELD)
   setup_field_target(${FIELD} ${FIELD_INDEX} ${FEATURES_STRING})
 endif()
 
+if (HASH)
+  setup_hash_target()
+endif()
+
 if (CPU_BACKEND)
   add_subdirectory(backend/cpu)
 endif()
 
 if (CUDA_BACKEND)
   string(TOLOWER "${CUDA_BACKEND}" CUDA_BACKEND_LOWER)
   if (CUDA_BACKEND_LOWER STREQUAL "local")
-    # CUDA backend is local, no need to pull    
+    # CUDA backend is local, no need to pull
     message(STATUS "Adding CUDA backend from local path: icicle/backend/cuda")
     add_subdirectory(backend/cuda)
 

icicle/backend/cpu/CMakeLists.txt

@@ -12,6 +12,9 @@ if (FIELD)
     if (NTT)
         target_sources(icicle_field PRIVATE src/field/cpu_ntt.cpp src/polynomials/cpu_polynomial_backend.cpp)
     endif()
+    if (POSEIDON)
+        target_sources(icicle_field PRIVATE src/hash/cpu_poseidon.cpp)
+    endif()
     target_include_directories(icicle_field PRIVATE include)
 endif() # FIELD
 
@@ -30,4 +33,10 @@ if (CURVE)
     target_include_directories(icicle_curve PUBLIC include)
 endif()
 
+if (HASH)
+    target_sources(icicle_hash PRIVATE
+        src/hash/cpu_keccak.cpp
+        src/hash/cpu_merkle_tree.cpp
+    )
+endif()
 

icicle/backend/cpu/src/hash/cpu_keccak.cpp

@@ -0,0 +1,122 @@
+
+#include "icicle/backend/hash/keccak_backend.h"
+
+namespace icicle {
+
+  class KeccakBackend : public HashBackend
+  {
+  public:
+    KeccakBackend(uint64_t input_chunk_size, uint64_t output_size, uint64_t rate, int padding_const)
+        : HashBackend(output_size, input_chunk_size)
+    {
+    }
+
+    eIcicleError hash(const std::byte* input, uint64_t size, const HashConfig& config, std::byte* output) const override
+    {
+      ICICLE_LOG_INFO << "Keccak CPU hash() called";
+      // TODO implement
+      return eIcicleError::SUCCESS;
+    }
+  };
+
+  const int KECCAK_256_RATE = 136;
+  const int KECCAK_256_DIGEST = 4;
+  const int KECCAK_512_RATE = 72;
+  const int KECCAK_512_DIGEST = 8;
+  const int KECCAK_STATE_SIZE = 25;
+  const int KECCAK_PADDING_CONST = 1;
+  const int SHA3_PADDING_CONST = 6;
+
+  class Keccak256Backend : public KeccakBackend
+  {
+  public:
+    Keccak256Backend(int input_chunk_size)
+        : KeccakBackend(
+            input_chunk_size,
+            KECCAK_256_DIGEST * sizeof(uint64_t) / sizeof(std::byte),
+            KECCAK_256_RATE,
+            KECCAK_PADDING_CONST)
+    {
+    }
+  };
+
+  class Keccak512Backend : public KeccakBackend
+  {
+  public:
+    Keccak512Backend(int input_chunk_size)
+        : KeccakBackend(
+            input_chunk_size,
+            KECCAK_512_DIGEST * sizeof(uint64_t) / sizeof(std::byte),
+            KECCAK_512_RATE,
+            KECCAK_PADDING_CONST)
+    {
+    }
+  };
+
+  class Sha3_256Backend : public KeccakBackend
+  {
+  public:
+    Sha3_256Backend(int input_chunk_size)
+        : KeccakBackend(
+            input_chunk_size,
+            KECCAK_256_DIGEST * sizeof(uint64_t) / sizeof(std::byte),
+            KECCAK_256_RATE,
+            SHA3_PADDING_CONST)
+    {
+    }
+  };
+
+  class Sha3_512Backend : public KeccakBackend
+  {
+  public:
+    Sha3_512Backend(int input_chunk_size)
+        : KeccakBackend(
+            input_chunk_size,
+            KECCAK_512_DIGEST * sizeof(uint64_t) / sizeof(std::byte),
+            KECCAK_512_RATE,
+            SHA3_PADDING_CONST)
+    {
+    }
+  };
+
+  /************************ Keccak 256 registration ************************/
+  eIcicleError
+  create_keccak_256_hash_backend(const Device& device, uint64_t input_chunk_size, std::shared_ptr<HashBackend>& backend)
+  {
+    backend = std::make_shared<Keccak256Backend>(input_chunk_size);
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_KECCAK_256_FACTORY_BACKEND("CPU", create_keccak_256_hash_backend);
+
+  /************************ Keccak 512 registration ************************/
+  eIcicleError
+  create_keccak_512_hash_backend(const Device& device, uint64_t input_chunk_size, std::shared_ptr<HashBackend>& backend)
+  {
+    backend = std::make_shared<Keccak512Backend>(input_chunk_size);
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_KECCAK_512_FACTORY_BACKEND("CPU", create_keccak_512_hash_backend);
+
+  /************************ SHA3 256 registration ************************/
+  eIcicleError
+  create_sha3_256_hash_backend(const Device& device, uint64_t input_chunk_size, std::shared_ptr<HashBackend>& backend)
+  {
+    backend = std::make_shared<Sha3_256Backend>(input_chunk_size);
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_SHA3_256_FACTORY_BACKEND("CPU", create_sha3_256_hash_backend);
+
+  /************************ SHA3 512 registration ************************/
+  eIcicleError
+  create_sha3_512_hash_backend(const Device& device, uint64_t input_chunk_size, std::shared_ptr<HashBackend>& backend)
+  {
+    backend = std::make_shared<Sha3_512Backend>(input_chunk_size);
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_SHA3_512_FACTORY_BACKEND("CPU", create_sha3_512_hash_backend);
+
+} // namespace icicle

icicle/backend/cpu/src/hash/cpu_merkle_tree.cpp

@@ -0,0 +1,53 @@
+#include "icicle/backend/merkle/merkle_tree_backend.h"
+#include "icicle/errors.h"
+#include "icicle/utils/log.h"
+
+namespace icicle {
+
+  class CPUMerkleTreeBackend : public MerkleTreeBackend
+  {
+  public:
+    CPUMerkleTreeBackend(
+      const std::vector<Hash>& layer_hashes, uint64_t leaf_element_size, uint64_t output_store_min_layer = 0)
+        : MerkleTreeBackend(layer_hashes, leaf_element_size, output_store_min_layer)
+    {
+    }
+
+    eIcicleError build(const std::byte* leaves, uint64_t size, const MerkleTreeConfig& config) override
+    {
+      ICICLE_LOG_INFO << "CPU CPUMerkleTreeBackend::build() called with " << size << " bytes of leaves";
+      return eIcicleError::SUCCESS; // TODO: Implement tree-building logic
+    }
+
+    eIcicleError get_merkle_root(std::byte* root, uint64_t root_size) const override
+    {
+      ICICLE_LOG_INFO << "CPU CPUMerkleTreeBackend::get_merkle_root() called";
+      return eIcicleError::SUCCESS; // TODO: Implement root retrieval logic
+    }
+
+    eIcicleError get_merkle_proof(
+      const std::byte* leaves,
+      uint64_t element_idx,
+      const MerkleTreeConfig& config,
+      MerkleProof& merkle_proof) const override
+    {
+      ICICLE_LOG_INFO << "CPU CPUMerkleTreeBackend::get_merkle_proof() called for element index " << element_idx;
+      return eIcicleError::SUCCESS; // TODO: Implement proof generation logic
+    }
+  };
+
+  eIcicleError create_merkle_tree_backend(
+    const Device& device,
+    const std::vector<Hash>& layer_hashes,
+    uint64_t leaf_element_size,
+    uint64_t output_store_min_layer,
+    std::shared_ptr<MerkleTreeBackend>& backend)
+  {
+    ICICLE_LOG_INFO << "Creating CPU MerkleTreeBackend";
+    backend = std::make_shared<CPUMerkleTreeBackend>(layer_hashes, leaf_element_size, output_store_min_layer);
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_MERKLE_TREE_FACTORY_BACKEND("CPU", create_merkle_tree_backend);
+
+} // namespace icicle

icicle/backend/cpu/src/hash/cpu_poseidon.cpp

@@ -0,0 +1,74 @@
+#include "icicle/backend/hash/poseidon_backend.h"
+#include "icicle/utils/utils.h"
+
+namespace icicle {
+
+  template <typename S>
+  class PoseidonConstantsCPU : PoseidonConstants<S>
+  {
+    // TODO add field here
+    S* m_dummy_poseidon_constant;
+  };
+
+  static eIcicleError cpu_poseidon_init_constants(
+    const Device& device,
+    unsigned arity,
+    unsigned alpha,
+    unsigned nof_partial_rounds,
+    unsigned nof_upper_full_rounds,
+    unsigned nof_end_full_rounds,
+    const scalar_t* rounds_constants,
+    const scalar_t* mds_matrix,
+    const scalar_t* pre_matrix,
+    const scalar_t* sparse_matrix,
+    std::shared_ptr<PoseidonConstants<scalar_t>>& constants /*out*/)
+  {
+    ICICLE_LOG_INFO << "in cpu_poseidon_init_constants()";
+    // TODO implement
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_POSEIDON_INIT_CONSTANTS_BACKEND("CPU", cpu_poseidon_init_constants);
+
+  static eIcicleError cpu_poseidon_init_default_constants(
+    const Device& device, unsigned arity, std::shared_ptr<PoseidonConstants<scalar_t>>& constants /*out*/)
+  {
+    ICICLE_LOG_INFO << "in cpu_poseidon_init_default_constants()";
+    // TODO implement
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_POSEIDON_INIT_DEFAULT_CONSTANTS_BACKEND("CPU", cpu_poseidon_init_default_constants);
+
+  template <typename S>
+  class PoseidonBackendCPU : public HashBackend
+  {
+  public:
+    PoseidonBackendCPU(std::shared_ptr<PoseidonConstants<S>> constants)
+        : HashBackend(sizeof(S), 0 /*TODO get from constants arity of whatever*/), m_constants{constants}
+    {
+    }
+
+    eIcicleError hash(const std::byte* input, uint64_t size, const HashConfig& config, std::byte* output) const override
+    {
+      ICICLE_LOG_INFO << "Poseidon CPU hash() " << size << " bytes, for type " << demangle<S>();
+      // TODO implement
+      return eIcicleError::SUCCESS;
+    }
+
+  private:
+    std::shared_ptr<PoseidonConstants<S>> m_constants = nullptr;
+  };
+
+  static eIcicleError create_cpu_poseidon_hash_backend(
+    const Device& device,
+    std::shared_ptr<PoseidonConstants<scalar_t>> constants,
+    std::shared_ptr<HashBackend>& backend /*OUT*/)
+  {
+    backend = std::make_shared<PoseidonBackendCPU<scalar_t>>(constants);
+    return eIcicleError::SUCCESS;
+  }
+
+  REGISTER_CREATE_POSEIDON_BACKEND("CPU", create_cpu_poseidon_hash_backend);
+
+} // namespace icicle

icicle/cmake/field.cmake

@@ -50,6 +50,7 @@ function(setup_field_target FIELD FIELD_INDEX FEATURES_STRING)
   # Handle features
   handle_ntt(icicle_field "${FEATURES_LIST}")
   handle_ext_field(icicle_field "${FEATURES_LIST}")
+  handle_poseidon(icicle_field "${FEATURES_LIST}")
   # Add additional feature handling calls here
 
   set_target_properties(icicle_field PROPERTIES OUTPUT_NAME "icicle_field_${FIELD}")

icicle/cmake/fields_and_curves.cmake

@@ -2,17 +2,17 @@
 # Define available fields with an index and their supported features
 # Format: index:field:features
 set(ICICLE_FIELDS
-  1001:babybear:NTT,EXT_FIELD
-  1002:stark252:NTT
-  1003:m31:EXT_FIELD
+  1001:babybear:NTT,EXT_FIELD,POSEIDON
+  1002:stark252:NTT,POSEIDON
+  1003:m31:EXT_FIELD,POSEIDON
 )
 
 # Define available curves with an index and their supported features
 # Format: index:curve:features
 set(ICICLE_CURVES
-  1:bn254:NTT,MSM,G2,ECNTT
-  2:bls12_381:NTT,MSM,G2,ECNTT
-  3:bls12_377:NTT,MSM,G2,ECNTT
-  4:bw6_761:NTT,MSM,G2,ECNTT
-  5:grumpkin:MSM
+  1:bn254:NTT,MSM,G2,ECNTT,POSEIDON
+  2:bls12_381:NTT,MSM,G2,ECNTT,POSEIDON
+  3:bls12_377:NTT,MSM,G2,ECNTT,POSEIDON
+  4:bw6_761:NTT,MSM,G2,ECNTT,POSEIDON
+  5:grumpkin:MSM,POSEIDON
 )

icicle/cmake/hash.cmake

@@ -0,0 +1,17 @@
+
+
+function(setup_hash_target)
+  add_library(icicle_hash SHARED)
+  target_sources(icicle_hash PRIVATE 
+    src/hash/keccak.cpp
+    src/hash/merkle_tree.cpp
+  )
+
+  target_link_libraries(icicle_hash PUBLIC icicle_device)
+
+  install(TARGETS icicle_hash
+    RUNTIME DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/"
+    LIBRARY DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/"
+    ARCHIVE DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/")
+endfunction()
+