Merge branch 'master' of https://dev.azure.com/translucency/Remotely/…

…_git/Remotely
immense · Apr 11, 2021 · b6774e5 · b6774e5
2 parents 7677c32 + 298fad4
commit b6774e5
Show file tree

Hide file tree

Showing 33 changed files with 487 additions and 267 deletions.
diff --git a/Agent/Agent.csproj b/Agent/Agent.csproj
@@ -23,11 +23,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="5.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="5.0.5" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="5.0.0" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="5.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="5.0.5" />
     <PackageReference Include="Microsoft.PowerShell.SDK" Version="7.1.3" />
     <PackageReference Include="Microsoft.WSMan.Management" Version="7.1.3" />
     <PackageReference Include="Microsoft.WSMan.Runtime" Version="7.1.3" />

diff --git a/Desktop.Core/Desktop.Core.csproj b/Desktop.Core/Desktop.Core.csproj
@@ -40,15 +40,13 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="5.0.4" />
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="5.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="5.0.5" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="5.0.5" />
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="5.0.0" />
     <PackageReference Include="Microsoft.MixedReality.WebRTC" Version="2.0.2" />
-    <PackageReference Include="SkiaSharp" Version="2.80.2" />
-    <PackageReference Include="SkiaSharp.Views.Desktop.Common" Version="2.80.2" />
     <PackageReference Include="System.Drawing.Common" Version="5.0.2" />
     <PackageReference Include="System.IO.FileSystem.AccessControl" Version="5.0.0" />
   </ItemGroup>

diff --git a/Desktop.Core/Interfaces/IFileTransferService.cs b/Desktop.Core/Interfaces/IFileTransferService.cs
@@ -1,6 +1,7 @@
 using Remotely.Desktop.Core.Services;
 using Remotely.Desktop.Core.ViewModels;
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Remotely.Desktop.Core.Interfaces
@@ -11,6 +12,6 @@ public interface IFileTransferService
 
         Task ReceiveFile(byte[] buffer, string fileName, string messageId, bool endOfFile, bool startOfFile);
         void OpenFileTransferWindow(Viewer viewer);
-        Task UploadFile(FileUpload file, Viewer viewer, Action<double> progressUpdateCallback);
+        Task UploadFile(FileUpload file, Viewer viewer, CancellationToken cancelToken, Action<double> progressUpdateCallback);
     }
 }
diff --git a/Desktop.Core/Services/DtoMessageHandler.cs b/Desktop.Core/Services/DtoMessageHandler.cs
@@ -12,7 +12,7 @@ namespace Remotely.Desktop.Core.Services
 {
     public interface IDtoMessageHandler
     {
-        Task ParseMessage(Services.Viewer viewer, byte[] message);
+        Task ParseMessage(Viewer viewer, byte[] message);
     }
     public class DtoMessageHandler : IDtoMessageHandler
     {
@@ -31,7 +31,7 @@ public DtoMessageHandler(IKeyboardMouseInput keyboardMouseInput,
         private IClipboardService ClipboardService { get; }
         private IFileTransferService FileTransferService { get; }
         private IKeyboardMouseInput KeyboardMouseInput { get; }
-        public async Task ParseMessage(Services.Viewer viewer, byte[] message)
+        public async Task ParseMessage(Viewer viewer, byte[] message)
         {
             try
             {
@@ -154,25 +154,19 @@ await FileTransferService.ReceiveFile(dto.Buffer,
                 dto.StartOfFile);
         }
 
-        private async Task GetWindowsSessions(Services.Viewer viewer)
+        private async Task GetWindowsSessions(Viewer viewer)
         {
             await viewer.SendWindowsSessions();
         }
 
-        private void HandleFrameReceived(Services.Viewer viewer)
+        private void HandleFrameReceived(Viewer viewer)
         {
-            for (int i = 0; i < 5; i++)
+            while (viewer.PendingSentFrames.Count > 0 &&
+                !viewer.IsStalled &&
+                viewer.IsConnected)
             {
-                if (viewer.PendingSentFrames.TryDequeue(out var frame))
+                if (viewer.PendingSentFrames.TryDequeue(out _))
                 {
-                    var roundtrip = (DateTimeOffset.Now - frame.Timestamp).TotalSeconds;
-                    var bps = frame.FrameSize / (roundtrip / 2);
-
-                    if (bps > viewer.PeakBytesPerSecond)
-                    {
-                        viewer.PeakBytesPerSecond = bps;
-                        Debug.WriteLine($"Peak Mbps: {bps / 1024 / 1024 * 8}");
-                    }
                     break;
                 }
             }
@@ -198,19 +192,19 @@ private void KeyUp(byte[] message)
             KeyboardMouseInput.SendKeyUp(dto.Key);
         }
 
-        private void MouseDown(byte[] message, Services.Viewer viewer)
+        private void MouseDown(byte[] message, Viewer viewer)
         {
             var dto = MessagePackSerializer.Deserialize<MouseDownDto>(message);
             KeyboardMouseInput.SendMouseButtonAction(dto.Button, ButtonAction.Down, dto.PercentX, dto.PercentY, viewer);
         }
 
-        private void MouseMove(byte[] message, Services.Viewer viewer)
+        private void MouseMove(byte[] message, Viewer viewer)
         {
             var dto = MessagePackSerializer.Deserialize<MouseMoveDto>(message);
             KeyboardMouseInput.SendMouseMove(dto.PercentX, dto.PercentY, viewer);
         }
 
-        private void MouseUp(byte[] message, Services.Viewer viewer)
+        private void MouseUp(byte[] message, Viewer viewer)
         {
             var dto = MessagePackSerializer.Deserialize<MouseUpDto>(message);
             KeyboardMouseInput.SendMouseButtonAction(dto.Button, ButtonAction.Up, dto.PercentX, dto.PercentY, viewer);
@@ -222,12 +216,12 @@ private void MouseWheel(byte[] message)
             KeyboardMouseInput.SendMouseWheel(-(int)dto.DeltaY);
         }
 
-        private void OpenFileTransferWindow(Services.Viewer viewer)
+        private void OpenFileTransferWindow(Viewer viewer)
         {
             FileTransferService.OpenFileTransferWindow(viewer);
         }
 
-        private void SelectScreen(byte[] message, Services.Viewer viewer)
+        private void SelectScreen(byte[] message, Viewer viewer)
         {
             var dto = MessagePackSerializer.Deserialize<SelectScreenDto>(message);
             viewer.Capturer.SetSelectedScreen(dto.DisplayName);
@@ -238,7 +232,7 @@ private void SetKeyStatesUp()
             KeyboardMouseInput.SetKeyStatesUp();
         }
 
-        private void Tap(byte[] message, Services.Viewer viewer)
+        private void Tap(byte[] message, Viewer viewer)
         {
             var dto = MessagePackSerializer.Deserialize<TapDto>(message);
             KeyboardMouseInput.SendMouseButtonAction(0, ButtonAction.Down, dto.PercentX, dto.PercentY, viewer);
@@ -257,7 +251,7 @@ private void ToggleBlockInput(byte[] message)
             KeyboardMouseInput.ToggleBlockInput(dto.ToggleOn);
         }
 
-        private void ToggleWebRtcVideo(byte[] message, Services.Viewer viewer)
+        private void ToggleWebRtcVideo(byte[] message, Viewer viewer)
         {
             var dto = MessagePackSerializer.Deserialize<ToggleWebRtcVideoDto>(message);
             viewer.ToggleWebRtcVideo(dto.ToggleOn);

diff --git a/Desktop.Core/Services/ScreenCaster.cs b/Desktop.Core/Services/ScreenCaster.cs
@@ -14,7 +14,6 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using SkiaSharp;
 
 namespace Remotely.Desktop.Core.Services
 {
@@ -55,6 +54,8 @@ public async Task BeginScreenCasting(ScreenCastRequest screenCastRequest)
                 viewer.Name = screenCastRequest.RequesterName;
                 viewer.ViewerConnectionID = screenCastRequest.ViewerID;
 
+                var screenBounds = viewer.Capturer.CurrentScreenBounds;
+
                 Logger.Write($"Starting screen cast.  Requester: {viewer.Name}. " +
                     $"Viewer ID: {viewer.ViewerConnectionID}.  App Mode: {_conductor.Mode}");
 
@@ -78,8 +79,7 @@ await viewer.SendScreenData(
                        viewer.Capturer.SelectedScreen,
                        viewer.Capturer.GetDisplayNames().ToArray());
 
-                await viewer.SendScreenSize(viewer.Capturer.CurrentScreenBounds.Width,
-                    viewer.Capturer.CurrentScreenBounds.Height);
+                await viewer.SendScreenSize(screenBounds.Width, screenBounds.Height);
 
                 await viewer.SendCursorChange(_cursorIconWatcher.GetCurrentCursor());
 
@@ -96,11 +96,11 @@ await viewer.SendScreenSize(viewer.Capturer.CurrentScreenBounds.Width,
                     {
                         await viewer.SendScreenCapture(new CaptureFrame()
                         {
-                            EncodedImageBytes = ImageUtils.EncodeWithSkia(initialFrame, SKEncodedImageFormat.Webp, _maxQuality),
-                            Left = viewer.Capturer.CurrentScreenBounds.Left,
-                            Top = viewer.Capturer.CurrentScreenBounds.Top,
-                            Width = viewer.Capturer.CurrentScreenBounds.Width,
-                            Height = viewer.Capturer.CurrentScreenBounds.Height
+                            EncodedImageBytes = ImageUtils.EncodeJpeg(initialFrame, _maxQuality),
+                            Left = screenBounds.Left,
+                            Top = screenBounds.Top,
+                            Width = screenBounds.Width,
+                            Height = screenBounds.Height
                         });
                     }
                 }
@@ -172,23 +172,23 @@ await viewer.SendScreenCapture(new CaptureFrame()
                         if (viewer.Capturer.CaptureFullscreen)
                         {
                             // Recalculate Bps.
-                            viewer.PeakBytesPerSecond = 0;
-                            encodedImageBytes = ImageUtils.EncodeWithSkia(clone, SKEncodedImageFormat.Jpeg, _maxQuality);
+                            viewer.AverageBytesPerSecond = 0;
+                            encodedImageBytes = ImageUtils.EncodeJpeg(clone, _maxQuality);
                         }
                         else
                         {
-                            if (viewer.PeakBytesPerSecond > 0)
+                            if (viewer.AverageBytesPerSecond > 0)
                             {
                                 var expectedSize = clone.Height * clone.Width * 4 * .1;
-                                var timeToSend = expectedSize / viewer.PeakBytesPerSecond;
+                                var timeToSend = expectedSize / viewer.AverageBytesPerSecond;
                                 currentQuality = Math.Max(_minQuality, Math.Min(_maxQuality, (int)(.1 / timeToSend * _maxQuality)));
                                 if (currentQuality < _maxQuality - 10)
                                 {
                                     refreshNeeded = true;
+                                    Debug.WriteLine($"Quality Reduced: {currentQuality}");
                                 }
-                                Debug.WriteLine($"Current Quality: {currentQuality}");
                             }
-                            encodedImageBytes = ImageUtils.EncodeWithSkia(clone, SKEncodedImageFormat.Jpeg, currentQuality);
+                            encodedImageBytes = ImageUtils.EncodeJpeg(clone, currentQuality);
                         }
 
                         viewer.Capturer.CaptureFullscreen = false;

diff --git a/Desktop.Core/Services/Viewer.cs b/Desktop.Core/Services/Viewer.cs
@@ -11,11 +11,16 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using System.Diagnostics;
+using System.Threading;
 
 namespace Remotely.Desktop.Core.Services
 {
     public class Viewer : IDisposable
     {
+        private long _bytesSent;
+        private TimeSpan _timeSpentSending = TimeSpan.Zero;
+
         public Viewer(ICasterSocket casterSocket,
             IScreenCapturer screenCapturer,
             IClipboardService clipboardService,
@@ -64,7 +69,7 @@ public bool IsUsingWebRtcVideo
 
         public string Name { get; set; }
 
-        public double PeakBytesPerSecond { get; set; }
+        public double AverageBytesPerSecond { get; set; }
         public ConcurrentQueue<SentFrame> PendingSentFrames { get; } = new();
 
         public WebRtcSession RtcSession { get; set; }
@@ -150,7 +155,7 @@ await SendToViewer(() => RtcSession.SendDto(dto),
                 () => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
         }
 
-        public async Task SendFile(FileUpload fileUpload, Action<double> progressUpdateCallback)
+        public async Task SendFile(FileUpload fileUpload, CancellationToken cancelToken, Action<double> progressUpdateCallback)
         {
             try
             {
@@ -163,13 +168,18 @@ public async Task SendFile(FileUpload fileUpload, Action<double> progressUpdateC
                     StartOfFile = true
                 };
 
-                await SendToViewer(() => RtcSession.SendDto(fileDto),
-                    () => CasterSocket.SendDtoToViewer(fileDto, ViewerConnectionID));
+                await SendToViewer(async () => await RtcSession.SendDto(fileDto),
+                    async () => await CasterSocket.SendDtoToViewer(fileDto, ViewerConnectionID));
 
                 using var fs = File.OpenRead(fileUpload.FilePath);
                 using var br = new BinaryReader(fs);
                 while (fs.Position < fs.Length)
                 {
+                    if (cancelToken.IsCancellationRequested)
+                    {
+                        return;
+                    }
+
                     fileDto = new FileDto()
                     {
                         Buffer = br.ReadBytes(50_000),
@@ -178,8 +188,8 @@ await SendToViewer(() => RtcSession.SendDto(fileDto),
                     };
 
                     await SendToViewer(
-                        () => RtcSession.SendDto(fileDto),
-                        () => CasterSocket.SendDtoToViewer(fileDto, ViewerConnectionID));
+                        async () => await RtcSession.SendDto(fileDto),
+                        async () => await CasterSocket.SendDtoToViewer(fileDto, ViewerConnectionID));
 
                     progressUpdateCallback((double)fs.Position / fs.Length);
                 }
@@ -192,8 +202,8 @@ await SendToViewer(
                     StartOfFile = false
                 };
 
-                await SendToViewer(() => RtcSession.SendDto(fileDto),
-                    () => CasterSocket.SendDtoToViewer(fileDto, ViewerConnectionID));
+                await SendToViewer(async () => await RtcSession.SendDto(fileDto),
+                    async () => await CasterSocket.SendDtoToViewer(fileDto, ViewerConnectionID));
 
                 progressUpdateCallback(1);
             }
@@ -219,6 +229,8 @@ public async Task SendScreenCapture(CaptureFrame screenFrame)
             var width = screenFrame.Width;
             var height = screenFrame.Height;
 
+            var sw = Stopwatch.StartNew();
+
             for (var i = 0; i < screenFrame.EncodedImageBytes.Length; i += 50_000)
             {
                 var dto = new CaptureFrameDto()
@@ -231,8 +243,8 @@ public async Task SendScreenCapture(CaptureFrame screenFrame)
                     ImageBytes = screenFrame.EncodedImageBytes.Skip(i).Take(50_000).ToArray()
                 };
 
-                await SendToViewer(() => RtcSession.SendDto(dto),
-                    () => CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
+                await SendToViewer(async () => await RtcSession.SendDto(dto),
+                    async () => await CasterSocket.SendDtoToViewer(dto, ViewerConnectionID));
             }
 
             var endOfFrameDto = new CaptureFrameDto()
@@ -244,8 +256,19 @@ await SendToViewer(() => RtcSession.SendDto(dto),
                 EndOfFrame = true
             };
 
-            await SendToViewer(() => RtcSession.SendDto(endOfFrameDto),
-                () => CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID));
+            await SendToViewer(
+                async () => await RtcSession.SendDto(endOfFrameDto),
+                async () => await CasterSocket.SendDtoToViewer(endOfFrameDto, ViewerConnectionID));
+
+            sw.Stop();
+
+            _bytesSent += screenFrame.EncodedImageBytes.Length;
+            _timeSpentSending += sw.Elapsed;
+
+
+            AverageBytesPerSecond = _bytesSent / _timeSpentSending.TotalSeconds;
+
+            Debug.WriteLine($"Mbps: {AverageBytesPerSecond / 1024 / 1024 * 8}");
         }
 
         public async Task SendScreenData(string selectedScreen, string[] displayNames)
@@ -280,7 +303,7 @@ public void ThrottleIfNeeded()
         {
             TaskHelper.DelayUntil(() =>
                 !PendingSentFrames.TryPeek(out var result) || DateTimeOffset.Now - result.Timestamp < TimeSpan.FromSeconds(1),
-                TimeSpan.MaxValue);
+                TimeSpan.FromSeconds(10));
         }
 
         public void ToggleWebRtcVideo(bool toggleOn)

diff --git a/Desktop.Core/Services/WebRtcSession.cs b/Desktop.Core/Services/WebRtcSession.cs
@@ -7,6 +7,7 @@
 using System.Drawing;
 using System.Linq;
 using System.Threading.Tasks;
+using System.Threading;
 
 namespace Remotely.Desktop.Core.Services
 {
@@ -109,7 +110,14 @@ public async Task Init(IceServerModel[] iceServers)
 
         public Task SendDto<T>(T dto) where T : BaseDto
         {
-            return Task.Run(() => CaptureChannel.SendMessage(MessagePackSerializer.Serialize(dto)));
+            return Task.Run(() =>
+            {
+                CaptureChannel.SendMessage(MessagePackSerializer.Serialize(dto));
+                while (CurrentBuffer > 64_000)
+                {
+                    Thread.Sleep(10);
+                }
+            });
         }
 
         public async Task SetRemoteDescription(string type, string sdp)