Pin simulator runs on CPU 0 via thread affinity

Summary:
In order for subsequent runs of simulator to hit the same per-CPU BPF maps, we need to execute them on the same CPU. Thus moving execution of `bpf_prog_test_run_opts` to dedicated thread that is affinitized to CPU #0.

Context https://fb.workplace.com/groups/meta.bpf/posts/2272088983164632

Reviewed By: frankfeir

Differential Revision: D68784116

fbshipit-source-id: a793b84d242dfeadcd26be31b86e2a336c5faece

katran/lib/KatranLb.cpp

@@ -2267,28 +2267,26 @@ std::unordered_map<uint32_t, std::string> KatranLb::getHealthcheckersDst() {
 
 std::string KatranLb::simulatePacket(const std::string& inPacket) {
   std::string result;
-  if (!progsLoaded_) {
-    LOG(ERROR) << "bpf programs are not loaded";
+  if (!initSimulator()) {
     return result;
   }
   auto inBuf = folly::IOBuf::copyBuffer(inPacket);
-  auto sim = KatranSimulator(getKatranProgFd());
-  auto outBuf = sim.runSimulation(std::move(inBuf));
+  auto outBuf = simulator_->runSimulation(std::move(inBuf));
   if (!outBuf) {
     LOG(ERROR) << "simulator failed to run simulation";
-    return result;
+  } else {
+    result = outBuf->moveToFbString().toStdString();
   }
-  result = outBuf->moveToFbString().toStdString();
   return result;
 }
 
 const std::string KatranLb::getRealForFlow(const KatranFlow& flow) {
-  if (!progsLoaded_) {
-    LOG(ERROR) << "bpf programs are not loaded";
-    return kEmptyString.data();
+  std::string result;
+  if (!initSimulator()) {
+    return result;
   }
-  auto sim = KatranSimulator(getKatranProgFd());
-  return sim.getRealForFlow(flow);
+  result = simulator_->getRealForFlow(flow);
+  return result;
 }
 
 bool KatranLb::updateVipMap(
@@ -2562,4 +2560,14 @@ void KatranLb::revalidateServerIds(const std::vector<QuicReal>& quicReals) {
     }
   }
 }
+
+bool KatranLb::initSimulator() {
+  if (!progsLoaded_) {
+    LOG(ERROR) << "bpf programs are not loaded";
+    return false;
+  }
+  simulator_ = std::make_unique<KatranSimulator>(getKatranProgFd());
+  return true;
+}
+
 } // namespace katran

katran/lib/KatranLb.h

@@ -1070,6 +1070,8 @@ class KatranLb {
       const std::vector<RealPos>& chPositions,
       const uint32_t vipNum);
 
+  bool initSimulator();
+
   /**
    * main configurations of katran
    */
@@ -1217,6 +1219,8 @@ class KatranLb {
    */
 
   std::unordered_set<uint64_t> invalidServerIds_;
+
+  std::unique_ptr<KatranSimulator> simulator_;
 };
 
 } // namespace katran

katran/lib/KatranSimulator.cpp

@@ -209,12 +209,23 @@ std::unique_ptr<folly::IOBuf> createPacketFromFlow(const KatranFlow& flow) {
 
 } // namespace
 
-KatranSimulator::KatranSimulator(int progFd) : progFd_(progFd) {}
+KatranSimulator::KatranSimulator(int progFd) : progFd_(progFd) {
+  affinitizeSimulatorThread();
+}
 
 KatranSimulator::~KatranSimulator() {}
 
 std::unique_ptr<folly::IOBuf> KatranSimulator::runSimulation(
     std::unique_ptr<folly::IOBuf> pckt) {
+  std::unique_ptr<folly::IOBuf> result;
+  simulatorEvb_.getEventBase()->runInEventBaseThreadAndWait(
+      [&]() { result = runSimulationInternal(std::move(pckt)); });
+  return result;
+}
+
+std::unique_ptr<folly::IOBuf> KatranSimulator::runSimulationInternal(
+    std::unique_ptr<folly::IOBuf> pckt) {
+  CHECK(simulatorEvb_.getEventBase()->isInEventBaseThread());
   if (!pckt) {
     LOG(ERROR) << "packet is empty";
     return nullptr;
@@ -265,4 +276,19 @@ const std::string KatranSimulator::getRealForFlow(const KatranFlow& flow) {
   return getPcktDst(rpckt);
 }
 
+void KatranSimulator::affinitizeSimulatorThread() {
+  simulatorEvb_.getEventBase()->runInEventBaseThreadAndWait([]() {
+    cpu_set_t cpuSet;
+    CPU_ZERO(&cpuSet);
+    CPU_SET(0, &cpuSet);
+    pthread_t currentThread = pthread_self();
+    auto ret =
+        pthread_setaffinity_np(currentThread, sizeof(cpu_set_t), &cpuSet);
+    if (ret != 0) {
+      LOG(ERROR) << "Error while affinitizing simulator thread to CPU 0: "
+                 << ret;
+    }
+  });
+}
+
 } // namespace katran

katran/lib/KatranSimulator.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <folly/io/IOBuf.h>
+#include <folly/io/async/ScopedEventBaseThread.h>
 #include <memory>
 #include <string>
 
@@ -66,6 +67,15 @@ class KatranSimulator final {
       std::unique_ptr<folly::IOBuf> pckt);
 
  private:
+  std::unique_ptr<folly::IOBuf> runSimulationInternal(
+      std::unique_ptr<folly::IOBuf> pckt);
+
+  // Affinitize simulator evb thread to CPU 0.
+  // This ensures that subsequent simulations run on the same CPU and hit
+  // same per-CPU maps.
+  void affinitizeSimulatorThread();
+
   int progFd_;
+  folly::ScopedEventBaseThread simulatorEvb_{"KatranSimulator"};
 };
 } // namespace katran