Custom rec agent

Unreal/CarlaUE4/Config/DReyeVRConfig.ini

@@ -13,6 +13,7 @@ FieldOfView=90.0
 ; other
 ActorRegistryID=501; desired (but not guaranteed) ID for the ActorRegistry (can be any nonzero number)
 DrawDebugEditor=False; draw debug lines/sphere for eye gaze in the editor
+CleanSlateRoomLocation=(X=-29663.0, Y=20500.0, Z=5278.0); location (3d world point) where camera teleports to
 
 [Mirrors]
 ; rear view mirror
@@ -93,4 +94,4 @@ FrameName=tick; title of screenshot (differentiated via tick-suffix)
 
 [Hardware]
 DeviceIdx=0; Device index of the hardware (Logitech has 2, can be 0 or 1)
-LogUpdates=True; whether or not to print debug messages
+LogUpdates=False; whether or not to print debug messages

Unreal/CarlaUE4/Config/DefaultInput.ini

@@ -95,6 +95,7 @@ DoubleClickTime=0.200000
 +ActionMappings=(ActionName="Handbrake",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=SpaceBar)
 +ActionMappings=(ActionName="ToggleManualMode",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=M)
 +ActionMappings=(ActionName="ResetCamera_DReyeVR",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=G)
++ActionMappings=(ActionName="ToggleCleanRoom_DReyeVR",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=N)
 +ActionMappings=(ActionName="Jump",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Enter)
 +ActionMappings=(ActionName="ToggleReverse",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=Q)
 +ActionMappings=(ActionName="UseTheForce",bShift=False,bCtrl=False,bAlt=False,bCmd=False,Key=F)

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Actor/DReyeVRCustomActor.cpp

@@ -0,0 +1,83 @@
+#include "DReyeVRCustomActor.h"
+#include "Carla/Game/CarlaStatics.h"    // GetEpisode
+#include "Carla/Sensor/DReyeVRSensor.h" // ADReyeVRSensor::bIsReplaying
+#include "Materials/MaterialInstance.h" // UMaterialInstance
+#include "UObject/ConstructorHelpers.h" // ConstructorHelpers
+
+#include <string>
+
+std::unordered_map<std::string, class ADReyeVRCustomActor *> ADReyeVRCustomActor::ActiveCustomActors = {};
+
+ADReyeVRCustomActor::ADReyeVRCustomActor(const FObjectInitializer &ObjectInitializer) : Super(ObjectInitializer)
+{
+    // done in child class
+    Internals.Location = this->GetActorLocation();
+    Internals.Rotation = this->GetActorRotation();
+    Internals.Scale3D = this->GetActorScale3D();
+}
+
+void ADReyeVRCustomActor::AssignSM(const FString &Path)
+{
+    static ConstructorHelpers::FObjectFinder<UStaticMesh> MeshAsset(*Path);
+    if (MeshAsset.Succeeded())
+        ActorMesh->SetStaticMesh(MeshAsset.Object);
+    else
+        UE_LOG(LogTemp, Error, TEXT("Unable to access mesh asset: %s"), *Path)
+}
+
+void ADReyeVRCustomActor::AssignMat(const FString &Path)
+{
+    static ConstructorHelpers::FObjectFinder<UMaterialInstance> MaterialAsset(*Path);
+    if (MaterialAsset.Succeeded())
+    {
+        Material = MaterialAsset.Object;
+        ActorMesh->SetMaterial(0, CastChecked<UMaterialInterface>(Material));
+    }
+    else
+        UE_LOG(LogTemp, Error, TEXT("Unable to access material asset: %s"), *Path)
+}
+
+void ADReyeVRCustomActor::Initialize(const DReyeVR::CustomActorData &In)
+{
+    SetInternals(In);
+    ADReyeVRCustomActor::ActiveCustomActors[TCHAR_TO_UTF8(*In.Name)] = this;
+}
+
+void ADReyeVRCustomActor::BeginPlay()
+{
+    Super::BeginPlay();
+}
+
+void ADReyeVRCustomActor::BeginDestroy()
+{
+    Super::BeginDestroy();
+}
+
+void ADReyeVRCustomActor::Tick(float DeltaSeconds)
+{
+    if (ADReyeVRSensor::bIsReplaying)
+    {
+        // update world state with internals
+        this->SetActorLocation(Internals.Location);
+        this->SetActorRotation(Internals.Rotation);
+        this->SetActorScale3D(Internals.Scale3D);
+    }
+    else
+    {
+        // update internals with world state
+        Internals.Location = this->GetActorLocation();
+        Internals.Rotation = this->GetActorRotation();
+        Internals.Scale3D = this->GetActorScale3D();
+    }
+    /// TODO: use other string?
+}
+
+void ADReyeVRCustomActor::SetInternals(const DReyeVR::CustomActorData &InData)
+{
+    Internals = InData;
+}
+
+const DReyeVR::CustomActorData &ADReyeVRCustomActor::GetInternals() const
+{
+    return Internals;
+}

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Actor/DReyeVRCustomActor.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "Carla/Sensor/DReyeVRData.h" // DReyeVR namespace
+#include "GameFramework/Actor.h"      // AActor
+
+#include <unordered_map> // std::unordered_map
+
+#include "DReyeVRCustomActor.generated.h"
+
+UCLASS()
+class CARLA_API ADReyeVRCustomActor : public AActor // abstract class
+{
+    GENERATED_BODY()
+
+  public:
+    ADReyeVRCustomActor(const FObjectInitializer &ObjectInitializer);
+
+    virtual void Tick(float DeltaSeconds) override;
+
+    static ADReyeVRCustomActor *RequestNewActor(UWorld *World, const DReyeVR::CustomActorData &Init);
+
+    void Initialize(const DReyeVR::CustomActorData &In);
+
+    void SetInternals(const DReyeVR::CustomActorData &In);
+
+    const DReyeVR::CustomActorData &GetInternals() const;
+
+    static std::unordered_map<std::string, class ADReyeVRCustomActor *> ActiveCustomActors;
+
+  protected:
+    void BeginPlay();
+    void BeginDestroy();
+
+    void AssignSM(const FString &Path);
+    void AssignMat(const FString &Path);
+
+    DReyeVR::CustomActorData Internals;
+
+    UPROPERTY(EditAnywhere, Category = "Materials")
+    UMaterialInstance *Material = nullptr;
+
+    UPROPERTY(EditAnywhere, Category = "Materials")
+    UMaterialInstanceDynamic *DynamicMaterial = nullptr;
+
+    UPROPERTY(EditAnywhere, Category = "Mesh")
+    UStaticMeshComponent *ActorMesh = nullptr;
+};

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaRecorder.cpp

@@ -18,7 +18,11 @@
 #include "CarlaReplayerHelper.h"
 
 // DReyeVR include
+#include "Carla/Actor/DReyeVRCustomActor.h"
+#include "Carla/Game/CarlaStatics.h"
+#include "Carla/Lights/CarlaLightSubsystem.h"
 #include "Carla/Sensor/DReyeVRSensor.h"
+#include "DReyeVRRecorder.h"
 
 #include <ctime>
 #include <sstream>
@@ -274,7 +278,18 @@ void ACarlaRecorder::AddActorBoundingBox(FCarlaActor *CarlaActor)
 void ACarlaRecorder::AddDReyeVRData()
 {
   // Add the latest instance of the DReyeVR snapshot to our data
-  DReyeVRData.Add(DReyeVRDataRecorder(ADReyeVRSensor::Data));
+  DReyeVRAggData.Add(DReyeVRDataRecorder<DReyeVR::AggregateData>(ADReyeVRSensor::Data));
+
+  TArray<AActor *> FoundActors;
+  if (Episode != nullptr && Episode->GetWorld() != nullptr)
+  {
+      UGameplayStatics::GetAllActorsOfClass(Episode->GetWorld(), ADReyeVRCustomActor::StaticClass(), FoundActors);
+  }
+  for (AActor *A : FoundActors)
+  {
+    ADReyeVRCustomActor *CustomActor = CastChecked<ADReyeVRCustomActor>(A);
+    DReyeVRCustomActorData.Add(DReyeVRDataRecorder<DReyeVR::CustomActorData>(&(CustomActor->GetInternals())));
+  }
 }
 
 void ACarlaRecorder::AddTriggerVolume(const ATrafficSignBase &TrafficSign)
@@ -407,7 +422,8 @@ void ACarlaRecorder::Clear(void)
   TriggerVolumes.Clear();
   PhysicsControls.Clear();
   TrafficLightTimes.Clear();
-  DReyeVRData.Clear();
+  DReyeVRAggData.Clear();
+  DReyeVRCustomActorData.Clear();
   Weathers.Clear();
 }
 
@@ -446,7 +462,10 @@ void ACarlaRecorder::Write(double DeltaSeconds)
     TrafficLightTimes.Write(File);
   }
   // custom DReyeVR data
-  DReyeVRData.Write(File);
+  DReyeVRAggData.Write(File);
+
+  // custom DReyeVR Actor data write
+  DReyeVRCustomActorData.Write(File);
 
   // weather state
   Weathers.Write(File);

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaRecorder.h

@@ -31,8 +31,10 @@
 #include "CarlaRecorderState.h"
 #include "CarlaRecorderWeather.h"
 #include "CarlaReplayer.h"
-// DReyeVR packet
+
+// DReyeVR includes
 #include "DReyeVRRecorder.h"
+#include "Carla/Sensor/DReyeVRData.h"
 
 #include "CarlaRecorder.generated.h"
 
@@ -43,6 +45,9 @@ class UCarlaLight;
 class ATrafficSignBase;
 class ATrafficLightBase;
 
+#define DREYEVR_PACKET_ID 139
+#define DREYEVR_CUSTOM_ACTOR_PACKET_ID 140
+
 enum class CarlaRecorderPacketId : uint8_t
 {
   FrameStart = 0,
@@ -65,7 +70,8 @@ enum class CarlaRecorderPacketId : uint8_t
   TriggerVolume,
   Weather,
   // "We suggest to use id over 100 for user custom packets, because this list will keep growing in the future"
-  DReyeVR = 139 // out custom DReyeVR packet
+  DReyeVR = DREYEVR_PACKET_ID,                        // our custom DReyeVR packet (for raw sensor data)
+  DReyeVRCustomActor = DREYEVR_CUSTOM_ACTOR_PACKET_ID // custom DReyeVR actors (not raw sensor data)
 };
 
 /// Recorder for the simulation
@@ -204,8 +210,8 @@ class CARLA_API ACarlaRecorder : public AActor
   CarlaRecorderPhysicsControls PhysicsControls;
   CarlaRecorderTrafficLightTimes TrafficLightTimes;
   CarlaRecorderWeathers Weathers;
-  DReyeVRDataRecorders DReyeVRData;
-
+  DReyeVRDataRecorders<DReyeVR::AggregateData, DREYEVR_PACKET_ID> DReyeVRAggData;
+  DReyeVRDataRecorders<DReyeVR::CustomActorData, DREYEVR_CUSTOM_ACTOR_PACKET_ID> DReyeVRCustomActorData;
 
   // replayer
   CarlaReplayer Replayer;

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaRecorderQuery.cpp

@@ -576,8 +576,29 @@ std::string CarlaRecorderQuery::QueryInfo(std::string Filename, bool bShowAll)
             Info << " DReyeVR sensor data: " << Total << std::endl;
             for (i = 0; i < Total; ++i)
             {
-                DReyeVRDataInstance.Read(File);
-                Info << DReyeVRDataInstance.Print() << std::endl;
+                DReyeVRAggDataInstance.Read(File);
+                Info << DReyeVRAggDataInstance.Print() << std::endl;
+            }
+        }
+        else
+            SkipPacket();
+        break;
+
+        // DReyeVR data
+        case static_cast<char>(CarlaRecorderPacketId::DReyeVRCustomActor):
+        if (bShowAll)
+        {
+            ReadValue<uint16_t>(File, Total);
+            if (Total > 0 && !bFramePrinted)
+            {
+                PrintFrame(Info);
+                bFramePrinted = true;
+            }
+            Info << " DReyeVR custom actor data: " << Total << std::endl;
+            for (i = 0; i < Total; ++i)
+            { 
+                DReyeVRCustomActorDataInstance.Read(File);
+                Info << DReyeVRCustomActorDataInstance.Print() << std::endl;
             }
         }
         else

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaRecorderQuery.h

@@ -70,7 +70,8 @@ class CarlaRecorderQuery
   CarlaRecorderTrafficLightTime TrafficLightTime;
   CarlaRecorderWeather Weather;
   // custom DReyeVR packets
-  DReyeVRDataRecorder DReyeVRDataInstance;
+  DReyeVRDataRecorder<DReyeVR::AggregateData> DReyeVRAggDataInstance;
+  DReyeVRDataRecorder<DReyeVR::CustomActorData> DReyeVRCustomActorDataInstance;
 
   // read next header packet
   bool ReadHeader(void);

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaReplayer.cpp

@@ -403,7 +403,15 @@ void CarlaReplayer::ProcessToTime(double Time, bool IsFirstTime)
       // DReyeVR eye logging data
       case static_cast<char>(CarlaRecorderPacketId::DReyeVR):
         if (bFrameFound)
-          ProcessDReyeVRData();
+          ProcessDReyeVRData<DReyeVRDataRecorder<DReyeVR::AggregateData>>(Per, Time);
+        else
+          SkipPacket();
+        break;
+
+      // DReyeVR eye logging data
+      case static_cast<char>(CarlaRecorderPacketId::DReyeVRCustomActor):
+        if (bFrameFound)
+          ProcessDReyeVRData<DReyeVRDataRecorder<DReyeVR::CustomActorData>>(Per, Time);
         else
           SkipPacket();
         break;
@@ -429,9 +437,6 @@ void CarlaReplayer::ProcessToTime(double Time, bool IsFirstTime)
     UpdatePositions(Per, Time);
   }
 
-  // Update the DReyeVR sensor after all moves have been made
-  UpdateDReyeVRSensor(Per, Time);
-
   // save current time
   CurrentTime = NewTime;
 
@@ -641,18 +646,20 @@ void CarlaReplayer::ProcessWeather(void)
   }
 }
 
-void CarlaReplayer::ProcessDReyeVRData()
+template <typename T> void CarlaReplayer::ProcessDReyeVRData(double Per, double DeltaTime)
 {
   uint16_t Total;
   // custom DReyeVR packets
 
-  // read Total DReyeVRevents
-  ReadValue<uint16_t>(File, Total);
-  // UE_LOG(LogCarla, Log, TEXT("Reading from file, total size of: %d"), Total);
+  // read Total DReyeVR events
+  ReadValue<uint16_t>(File, Total); // read number of events
+
   check(Total == 1); // there should only ever be one recorded DReyeVR sensor
   for (uint16_t i = 0; i < Total; ++i)
   {
+    T DReyeVRDataInstance;
     DReyeVRDataInstance.Read(File);
+    Helper.ProcessReplayerDReyeVRData<T>(DReyeVRDataInstance, Per);
   }
 }
 
@@ -689,12 +696,6 @@ void CarlaReplayer::ProcessPositions(bool IsFirstTime)
   }
 }
 
-void CarlaReplayer::UpdateDReyeVRSensor(double Per, double DeltaTime)
-{
-  // apply these operations to the sensor
-  Helper.ProcessReplayerDReyeVRData(DReyeVRDataInstance, Per);
-}
-
 void CarlaReplayer::UpdatePositions(double Per, double DeltaTime)
 {
   unsigned int i;

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaReplayer.h

@@ -165,7 +165,7 @@ class CarlaReplayer
   void ProcessWeather(void);
 
   // DReyeVR recordings
-  void ProcessDReyeVRData(void);
+  template <typename T> void ProcessDReyeVRData(double Per, double DeltaTime);
 
   // For restarting the recording with the same params
   struct LastReplayStruct
@@ -177,10 +177,6 @@ class CarlaReplayer
   };
   LastReplayStruct LastReplay;
 
-  // DReyeVR sensor data
-  DReyeVRDataRecorder DReyeVRDataInstance;
-  void UpdateDReyeVRSensor(double Per, double DeltaTime);
-
   bool bReplaySync = false;
   std::vector<double> FrameStartTimes;
   size_t SyncCurrentFrameId = 0;

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Recorder/CarlaReplayerHelper.cpp

@@ -486,10 +486,10 @@ bool CarlaReplayerHelper::ProcessReplayerFinish(bool bApplyAutopilot, bool bIgno
   return true;
 }
 
-void CarlaReplayerHelper::ProcessReplayerDReyeVRData(const DReyeVRDataRecorder &DReyeVRDataInstance, const double Per)
+template <typename T> void CarlaReplayerHelper::ProcessReplayerDReyeVRData(const T &DReyeVRDataInstance, const double Per)
 {
   if (ADReyeVRSensor::GetDReyeVRSensor())
-    ADReyeVRSensor::GetDReyeVRSensor()->UpdateWithReplayData(DReyeVRDataInstance.Data, Per);
+    ADReyeVRSensor::GetDReyeVRSensor()->UpdateData(DReyeVRDataInstance.Data, Per);
   else
     UE_LOG(LogTemp, Error, TEXT("No DReyeVR sensor available!"));
 }