feat: csharp file performance stats (#588)

* feat: csharp file performance stats * better precision stats * address comment file size naming
VowpalWabbit · Sep 29, 2023 · 224a7a5 · 224a7a5
1 parent e792832
commit 224a7a5
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 11 deletions.
diff --git a/bindings/cs/rl.net.cli/Helpers.cs b/bindings/cs/rl.net.cli/Helpers.cs
@@ -32,11 +32,15 @@ public static LiveModel CreateLiveModelOrExit(string clientJsonPath)
             {
                 WriteStatusAndExit(apiStatus);
             }
+            string trace_log = config["trace.logger.implementation"];
 
             LiveModel liveModel = new LiveModel(config);
 
             liveModel.BackgroundError += LiveModel_BackgroundError;
-            liveModel.TraceLoggerEvent += LiveModel_TraceLogEvent;
+            if (trace_log == "CONSOLE_TRACE_LOGGER")
+            {
+                liveModel.TraceLoggerEvent += LiveModel_TraceLogEvent;
+            }
 
             if (!liveModel.TryInit(apiStatus))
             {

diff --git a/bindings/cs/rl.net.cli/PerfTestCommand.cs b/bindings/cs/rl.net.cli/PerfTestCommand.cs
@@ -36,6 +36,9 @@ class PerfTestCommand : CommandBase
         [Option(longName: "parallelism", shortName: 'p', HelpText = "Degree of parallelism to use. Use 0 to use all available processors", Required = false, Default = 1)]
         public int Parallelism { get; set; }
 
+        [Option(longName: "filePath", HelpText = "DsJson file path to test from", Required = false, Default = "")]
+        public string FilePath { get; set; }
+
         public override void Run()
         {
             Console.WriteLine("The number of processors on this computer is {0}.", Environment.ProcessorCount);
@@ -64,13 +67,23 @@ public override void Run()
         private PerfTestStepProvider DoWork(string tag)
         {
             LiveModel liveModel = Helpers.CreateLiveModelOrExit(this.ConfigPath);
-
-            PerfTestStepProvider stepProvider = new PerfTestStepProvider(this.ActionsCount, this.SharedFeatures, this.ActionFeatures, this.NumSlots)
+            PerfTestStepProvider stepProvider;
+            if (this.FilePath != "")
             {
-                Duration = TimeSpan.FromMilliseconds(this.DurationMs),
-                Tag = tag,
-                DataSize = this.DataSize * 1024 * 1024 * 1024 / this.Parallelism
-            };
+                stepProvider = new PerfTestStepProvider(this.FilePath)
+                {
+                    Tag = tag,
+                };
+            }
+            else
+            {
+                stepProvider = new PerfTestStepProvider(this.ActionsCount, this.SharedFeatures, this.ActionFeatures, this.NumSlots)
+                {
+                    Duration = TimeSpan.FromMilliseconds(this.DurationMs),
+                    Tag = tag,
+                    DataSize = this.DataSize * 1024 * 1024 * 1024 / this.Parallelism
+                };
+            }
 
             Console.WriteLine(stepProvider.DataSize);
             RLDriver rlDriver = new RLDriver(liveModel, loopKind: this.GetLoopKind())

diff --git a/bindings/cs/rl.net.cli/PerfTestStepProvider.cs b/bindings/cs/rl.net.cli/PerfTestStepProvider.cs
@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.IO;
 
 namespace Rl.Net.Cli
 {
@@ -64,6 +65,10 @@ public float GetContinuousActionOutcome(float action, float pdfValue)
 
         public TimeSpan Duration { get; set; } = TimeSpan.FromSeconds(20);
 
+        public int NumLines { get; set; } = 0;
+
+        public int NumLinesProcessed { get; set; } = 0;
+
         public double DataSize { get; set; } = 0;
 
         public Statistics Stats { get; private set; }
@@ -93,11 +98,36 @@ public PerfTestStepProvider(int actionsCount, int sharedFeatures, int actionFeat
             }
         }
 
+        public PerfTestStepProvider(string filePath)
+        {
+            using (StreamReader sr = new StreamReader(filePath))
+            {
+                string line;
+                while ((line = sr.ReadLine()) != null)
+                {
+                    Contexts.Add(line);
+                }
+            }
+            this.NumLines = Contexts.Count;
+        }
+
         public IEnumerator<IStepContext<float>> GetEnumerator()
         {
             this.Stats = new Statistics();
-            while (this.Stats.Bytes < this.DataSize || this.Stats.ElapsedMs < this.Duration.TotalMilliseconds)
+            while (true)
             {
+                if (this.NumLines > 0)
+                {
+                    if (this.NumLines == this.NumLinesProcessed)
+                    {
+                        break;
+                    }
+                    this.NumLinesProcessed++;
+                }
+                else if (!(this.Stats.Bytes < this.DataSize || this.Stats.ElapsedMs < this.Duration.TotalMilliseconds))
+                {
+                    break;
+                }
                 var step = new PerfTestStep
                 {
                     EventId = $"{Tag}-{this.Stats.Messages}",

diff --git a/bindings/cs/rl.net.cli/Statistics.cs b/bindings/cs/rl.net.cli/Statistics.cs
@@ -33,11 +33,12 @@ public void Update(double byteCount)
         public void Print()
         {
             Console.WriteLine($"Data sent: {this.Bytes / (1024 * 1024)} MB");
-            Console.WriteLine($"Time taken: {(this.ElapsedMs / 1000)} secs");
+            Console.WriteLine($"Time taken: {(this.ElapsedMs / 1000.0)} secs");
             Console.WriteLine($"Throughput: {this.Bytes / ((1024 * 1024) * this.ElapsedMs / 1000)} MB / s");
             Console.WriteLine($"Messages sent: {this.Messages}");
             Console.WriteLine($"Avg Message size: {this.Bytes / (1024 * this.Messages)} KB");
-            Console.WriteLine($"Msg/s: {this.Messages / (this.ElapsedMs / 1000)}");
+            Console.WriteLine($"Msg/s: {this.Messages / (this.ElapsedMs / 1000.0)}");
+            Console.WriteLine($"Latency: {(this.ElapsedMs / 1000.0) / this.Messages * 1000000} μs");
         }
     }
 }
diff --git a/nuget/dotnet/test/client.json b/nuget/dotnet/test/client.json
@@ -8,5 +8,6 @@
     "InitialExplorationEpsilon": 1.0,
     "LearningMode": "Online",
     "model.source": "FILE_MODEL_DATA",
-    "protocol.version":"2"
+    "protocol.version":"2",
+    "model.vw.initial_command_line":"--quiet -b 18 --random_weights --interactions 6234 --interactions 652 --interactions 6B52  --cb_explore_adf --softmax --lambda 9.8316 --cb_adf --cb_type ips --csoaa_ldf multiline --csoaa_rank"
 }