Sp-ization Algorithm

CMakeLists.txt

@@ -73,6 +73,7 @@ option(FF_BUILD_ALL_EXAMPLES "build all examples. Overrides others" OFF)
 option(FF_BUILD_UNIT_TESTS "build non-operator unit tests" OFF)
 option(FF_BUILD_SUBSTITUTION_TOOL "build substitution conversion tool" OFF)
 option(FF_BUILD_VISUALIZATION_TOOL "build substitution visualization tool" ON)
+option(FF_BUILD_SP_IZATION_BENCHMARKING "build sp-ization benchmarking" ON)
 option(FF_BUILD_ARG_PARSER "build command line argument parser" OFF)
 
 set(FF_CUDA_ARCH "autodetect" CACHE STRING "Target CUDA Arch")

bin/CMakeLists.txt

@@ -10,6 +10,10 @@ if(FF_BUILD_VISUALIZATION_TOOL)
   add_subdirectory(substitutions-to-dot)
 endif()
 
+if(FF_BUILD_SP_IZATION_BENCHMARKING)
+  add_subdirectory(sp_ization_benchmarking)
+endif()
+
 if(FF_BUILD_ARG_PARSER)
   add_subdirectory(arg_parser)
 endif()

bin/sp_ization_benchmarking/CMakeLists.txt

@@ -0,0 +1,9 @@
+ff_add_executable(
+  NAME
+    sp-ization-benchmarking
+  SRC_PATTERNS
+    *.cc
+  DEPS
+    utils
+    rapidcheck
+)

bin/sp_ization_benchmarking/distributions.cc

@@ -0,0 +1,49 @@
+#include "distributions.h"
+
+namespace FlexFlow {
+
+Constant::Constant(float val) : val(val) {}
+
+float Constant::operator()() const {
+  return val;
+}
+
+Uniform::Uniform(float a, float b) : a(a), b(b) {}
+
+float Uniform::operator()() const {
+  return a + ((static_cast<double>(std::rand()) / RAND_MAX) * (b - a));
+}
+
+Bernoulli::Bernoulli(float p) : p(p) {}
+
+float Bernoulli::operator()() const {
+  return (Uniform(0, 1)() < p);
+}
+
+Binary::Binary(float a, float b, float p) : a(a), b(b), p(p) {}
+
+float Binary::operator()() const {
+  return (Bernoulli(p)() ? a : b);
+}
+
+UniformNoise::UniformNoise(float lower, float upper)
+    : lower(lower), upper(upper) {}
+
+float UniformNoise::operator()() const {
+  return Uniform(lower, upper)();
+}
+
+float NoNoise::operator()() const {
+  return 1;
+}
+
+GaussianNoise::GaussianNoise(float mean, float stddev)
+    : mean(mean), stddev(stddev) {}
+
+float GaussianNoise::operator()() const {
+  static std::default_random_engine generator;
+  static std::normal_distribution<float> distribution(mean, stddev);
+  return distribution(generator);
+}
+
+} // namespace FlexFlow

bin/sp_ization_benchmarking/distributions.h

@@ -0,0 +1,65 @@
+#ifndef _FLEXFLOW_DISTRIBUTIONS_H
+#define _FLEXFLOW_DISTRIBUTIONS_H
+
+#include "utils/graph/node/node.dtg.h"
+#include <random>
+#include <unordered_map>
+#include <unordered_set>
+
+namespace FlexFlow {
+
+struct Constant {
+  float val;
+  Constant(float val = 1);
+  float operator()() const;
+};
+
+struct Uniform {
+  float a, b;
+  Uniform(float a = 0, float b = 1);
+  float operator()() const;
+};
+
+struct Bernoulli {
+  float p;
+  Bernoulli(float p = 0.5);
+  float operator()() const;
+};
+
+struct Binary {
+  float a, b, p;
+  Binary(float a = 0, float b = 1, float p = 0.5);
+  float operator()() const;
+};
+
+struct UniformNoise {
+  float lower, upper;
+  UniformNoise(float lower = 0.9, float upper = 1.1);
+  float operator()() const;
+};
+
+struct NoNoise {
+  float operator()() const;
+};
+
+struct GaussianNoise {
+  float mean, stddev;
+  GaussianNoise(float mean = 1, float stddev = .1);
+  float operator()() const;
+};
+
+template <typename Dist, typename Noise>
+std::unordered_map<Node, float>
+    make_cost_map(std::unordered_set<Node> const &nodes,
+                  Dist const &distribution,
+                  Noise const &noise) {
+  std::unordered_map<Node, float> cost_map;
+  for (Node const &node : nodes) {
+    cost_map[node] = distribution() * noise();
+  }
+  return cost_map;
+}
+
+} // namespace FlexFlow
+
+#endif