[wasm][debugger] Implement support to symbolOptions from dap. (#79284)

* Draft to implement support to symbolOptions from dap.

* removing debugger.launch

* Adding tests and fix compilation error

* Adding test case.

* Fix test cases, and implement support to PDBChecksum used on nuget.org to get symbols.

* merge

* Fixing tests.

* Tests are timing out.

* Apply suggestions from code review

Co-authored-by: Larry Ewing <lewing@microsoft.com>
Co-authored-by: Ankit Jain <radical@gmail.com>

* adressing @radical comments.

* Addressing @radical comment.

* Addressing @radical comments.

* Apply suggestions from code review

Co-authored-by: Ankit Jain <radical@gmail.com>

* Addressing @radical comments.

* Addressing @radical comments
Changing when the symbols from symbol server is loaded because it takes a long time to load, as there are a lot of assemblies loaded not found on symbol servers.

* use MicrosoftCodeAnalysisCSharpVersion for scripting package.

* Adding more tests as asked by @radical
Removing timeout change as @radical has split it into 2 files
Fixing test behavior, when justMyCode is disabled but the symbols are not loaded from symbol server.

* [wasm] some cleanup

- Don't call `UpdateSymbolStore` from `DebugStore..ctor` because that
gets called multiple times in `LoadStore`, but only once instance gets
used.
- Use an isolated symbol cache path per test

* remove debug spew

* Addressing radical comment.

Co-authored-by: Larry Ewing <lewing@microsoft.com>
Co-authored-by: Ankit Jain <radical@gmail.com>

eng/Versions.props

@@ -248,5 +248,9 @@
     <runtimewinx64MicrosoftNETCoreRuntimeJITToolsVersion>1.0.0-alpha.1.23066.1</runtimewinx64MicrosoftNETCoreRuntimeJITToolsVersion>
     <runtimeosx110arm64MicrosoftNETCoreRuntimeJITToolsVersion>1.0.0-alpha.1.23066.1</runtimeosx110arm64MicrosoftNETCoreRuntimeJITToolsVersion>
     <runtimeosx1012x64MicrosoftNETCoreRuntimeJITToolsVersion>1.0.0-alpha.1.23066.1</runtimeosx1012x64MicrosoftNETCoreRuntimeJITToolsVersion>
+
+    <!-- BrowserDebugProxy libs -->
+    <MicrosoftExtensionsLoggingVersion>3.1.7</MicrosoftExtensionsLoggingVersion>
+    <MicrosoftSymbolStoreVersion>1.0.406601</MicrosoftSymbolStoreVersion>
   </PropertyGroup>
 </Project>

src/mono/mono/component/debugger-engine.c

@@ -975,10 +975,11 @@ mono_de_ss_update (SingleStepReq *req, MonoJitInfo *ji, SeqPoint *sp, void *tls,
 	if (minfo)
 		loc = mono_debug_method_lookup_location (minfo, sp->il_offset);
 
-	if (!loc) {
-		PRINT_DEBUG_MSG (1, "[%p] No line number info for il offset %x, continuing single stepping.\n", (gpointer) (gsize) mono_native_thread_id_get (), sp->il_offset);
+	if (!loc) { //we should not continue single stepping because the client side can have symbols loaded dynamically
+		PRINT_DEBUG_MSG (1, "[%p] No line number info for il offset %x, don't know if it's in the same line single stepping.\n", (gpointer) (gsize) mono_native_thread_id_get (), sp->il_offset);
 		req->last_method = method;
-		hit = FALSE;
+		req->last_line = -1;
+		return hit;
 	} else if (loc && method == req->last_method && loc->row == req->last_line) {
 		int nframes;
 		rt_callbacks.ss_calculate_framecount (tls, ctx, FALSE, NULL, &nframes);

src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/Microsoft.NETCore.BrowserDebugHost.Transport.pkgproj

@@ -20,6 +20,8 @@
     <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Newtonsoft.Json.dll" />
     <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.CSharp.Scripting.dll" />
     <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.CodeAnalysis.Scripting.dll" />
+    <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.SymbolStore.dll" />
+    <_browserDebugHostFiles Include="$(ArtifactsDir)bin\BrowserDebugHost\$(TargetArchitecture)\$(Configuration)\Microsoft.FileFormats.dll" />
 
     <PackageFile Include="@(_browserDebugHostFiles)" TargetPath="tools\$(NetCoreAppCurrent)\" />
   </ItemGroup>

src/mono/wasm/debugger/BrowserDebugHost/Startup.cs

@@ -203,8 +203,7 @@ async Task ConnectProxy(HttpContext context)
                     try
                     {
                         var loggerFactory = context.RequestServices.GetService<ILoggerFactory>();
-                        context.Request.Query.TryGetValue("urlSymbolServer", out StringValues urlSymbolServerList);
-                        var proxy = new DebuggerProxy(loggerFactory, urlSymbolServerList.ToList(), runtimeId, options: options);
+                        var proxy = new DebuggerProxy(loggerFactory, runtimeId, options: options);
 
                         System.Net.WebSockets.WebSocket ideSocket = await context.WebSockets.AcceptWebSocketAsync();
 

src/mono/wasm/debugger/BrowserDebugProxy/BrowserDebugProxy.csproj

@@ -8,10 +8,11 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="3.7.0" />
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="3.1.7" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="$(MicrosoftCodeAnalysisCSharpVersion)" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
+    <PackageReference Include="Microsoft.SymbolStore" Version="$(MicrosoftSymbolStoreVersion)" />
     <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
-    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.7.0" />
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisCSharpVersion)" />
   </ItemGroup>
 
   <ItemGroup>

src/mono/wasm/debugger/BrowserDebugProxy/DebuggerProxy.cs

@@ -18,10 +18,10 @@ public class DebuggerProxy : DebuggerProxyBase
     {
         internal MonoProxy MonoProxy { get; }
 
-        public DebuggerProxy(ILoggerFactory loggerFactory, IList<string> urlSymbolServerList, int runtimeId = 0, string loggerId = "", ProxyOptions options = null)
+        public DebuggerProxy(ILoggerFactory loggerFactory, int runtimeId = 0, string loggerId = "", ProxyOptions options = null)
         {
             string suffix = loggerId.Length > 0 ? $"-{loggerId}" : string.Empty;
-            MonoProxy = new MonoProxy(loggerFactory.CreateLogger($"DevToolsProxy{suffix}"), urlSymbolServerList, runtimeId, loggerId, options);
+            MonoProxy = new MonoProxy(loggerFactory.CreateLogger($"DevToolsProxy{suffix}"), runtimeId, loggerId, options);
         }
 
         public Task Run(Uri browserUri, WebSocket ideSocket, CancellationTokenSource cts)

src/mono/wasm/debugger/BrowserDebugProxy/EvaluateExpression.cs

@@ -58,7 +58,7 @@ public override void Visit(SyntaxNode node)
                 if (visitCount == 0)
                 {
                     if (node is MemberAccessExpressionSyntax maes
-                        && node.Kind() == SyntaxKind.SimpleMemberAccessExpression
+                        && node.IsKind(SyntaxKind.SimpleMemberAccessExpression)
                         && !(node.Parent is MemberAccessExpressionSyntax)
                         && !(node.Parent is InvocationExpressionSyntax)
                         && !(node.Parent is ElementAccessExpressionSyntax))
@@ -401,7 +401,7 @@ internal static async Task<JObject> CompileAndRunTheExpression(
             // this fails with `"a)"`
             // because the code becomes: return (a));
             // and the returned expression from GetExpressionFromSyntaxTree is `a`!
-            if (expressionTree.Kind() == SyntaxKind.IdentifierName || expressionTree.Kind() == SyntaxKind.ThisExpression)
+            if (expressionTree.IsKind(SyntaxKind.IdentifierName) || expressionTree.IsKind(SyntaxKind.ThisExpression))
             {
                 string varName = expressionTree.ToString();
                 JObject value = await resolver.Resolve(varName, token);
@@ -416,7 +416,7 @@ internal static async Task<JObject> CompileAndRunTheExpression(
             syntaxTree = replacer.ReplaceVars(syntaxTree, memberAccessValues, identifierValues, null, null);
 
             // eg. "this.dateTime", "  dateTime.TimeOfDay"
-            if (expressionTree.Kind() == SyntaxKind.SimpleMemberAccessExpression && replacer.memberAccesses.Count == 1)
+            if (expressionTree.IsKind(SyntaxKind.SimpleMemberAccessExpression) && replacer.memberAccesses.Count == 1)
             {
                 return memberAccessValues[0];
             }

src/mono/wasm/debugger/BrowserDebugProxy/Firefox/FirefoxMonoProxy.cs

@@ -17,7 +17,7 @@ namespace Microsoft.WebAssembly.Diagnostics;
 
 internal sealed class FirefoxMonoProxy : MonoProxy
 {
-    public FirefoxMonoProxy(ILogger logger, string loggerId = null, ProxyOptions options = null) : base(logger, null, loggerId: loggerId, options: options)
+    public FirefoxMonoProxy(ILogger logger, string loggerId = null, ProxyOptions options = null) : base(logger, loggerId: loggerId, options: options)
     {
     }
 

src/mono/wasm/debugger/BrowserDebugProxy/MonoProxy.cs

@@ -18,7 +18,8 @@ namespace Microsoft.WebAssembly.Diagnostics
 {
     internal class MonoProxy : DevToolsProxy
     {
-        private IList<string> urlSymbolServerList;
+        internal List<string> UrlSymbolServerList { get; private set; }
+        internal string CachePathSymbolServer { get; private set; }
         private HashSet<SessionId> sessions = new HashSet<SessionId>();
         private static readonly string[] s_executionContextIndependentCDPCommandNames = { "DotnetDebugger.setDebuggerProperty", "DotnetDebugger.runTests" };
         protected Dictionary<SessionId, ExecutionContext> contexts = new Dictionary<SessionId, ExecutionContext>();
@@ -32,9 +33,9 @@ internal class MonoProxy : DevToolsProxy
 
         protected readonly ProxyOptions _options;
 
-        public MonoProxy(ILogger logger, IList<string> urlSymbolServerList, int runtimeId = 0, string loggerId = "", ProxyOptions options = null) : base(logger, loggerId)
+        public MonoProxy(ILogger logger, int runtimeId = 0, string loggerId = "", ProxyOptions options = null) : base(logger, loggerId)
         {
-            this.urlSymbolServerList = urlSymbolServerList ?? new List<string>();
+            UrlSymbolServerList = new List<string>();
             RuntimeId = runtimeId;
             _options = options;
             _defaultPauseOnExceptions = PauseOnExceptionsKind.Unset;
@@ -76,10 +77,10 @@ internal void SendLog(SessionId sessionId, string message, CancellationToken tok
             {
                 type,
                 args = new JArray(JObject.FromObject(new
-                                {
-                                    type = "string",
-                                    value = message,
-                                })),
+                {
+                    type = "string",
+                    value = message,
+                })),
                 executionContextId = context.Id
             });
             SendEvent(sessionId, "Runtime.consoleAPICalled", o, token);
@@ -143,7 +144,7 @@ protected override async Task<bool> AcceptEvent(SessionId sessionId, JObject par
                             bool? is_default = aux_data["isDefault"]?.Value<bool>();
                             if (is_default == true)
                             {
-                                await OnDefaultContext(sessionId, new ExecutionContext(new MonoSDBHelper (this, logger, sessionId), id, aux_data, _defaultPauseOnExceptions), token);
+                                await OnDefaultContext(sessionId, new ExecutionContext(new MonoSDBHelper(this, logger, sessionId), id, aux_data, _defaultPauseOnExceptions), token);
                             }
                         }
                         return true;
@@ -170,6 +171,8 @@ protected override async Task<bool> AcceptEvent(SessionId sessionId, JObject par
                                 {
                                     await RuntimeReady(sessionId, token);
                                     await SendResume(sessionId, token);
+                                    if (!JustMyCode)
+                                        await ReloadSymbolsFromSymbolServer(sessionId, GetContext(sessionId), token);
                                     return true;
                                 }
                             case "mono_wasm_fire_debugger_agent_message":
@@ -240,7 +243,7 @@ protected override async Task<bool> AcceptCommand(MessageId id, JObject parms, C
 
             if (!contexts.TryGetValue(id, out ExecutionContext context) && !s_executionContextIndependentCDPCommandNames.Contains(method))
             {
-                if  (method == "Debugger.setPauseOnExceptions")
+                if (method == "Debugger.setPauseOnExceptions")
                 {
                     string state = args["state"].Value<string>();
                     var pauseOnException = GetPauseOnExceptionsStatusFromString(state);
@@ -505,11 +508,11 @@ protected override async Task<bool> AcceptCommand(MessageId id, JObject parms, C
                             switch (property.Key)
                             {
                                 case "JustMyCodeStepping":
-                                    SetJustMyCode(id, (bool) property.Value, token);
-                                break;
+                                    await SetJustMyCode(id, (bool)property.Value, context, token);
+                                    break;
                                 default:
                                     logger.LogDebug($"DotnetDebugger.setDebuggerProperty failed for {property.Key} with value {property.Value}");
-                                break;
+                                    break;
                             }
                         }
                         return true;
@@ -534,11 +537,21 @@ protected override async Task<bool> AcceptCommand(MessageId id, JObject parms, C
                             SendResponse(id, Result.Err("ApplyUpdate failed."), token);
                         return true;
                     }
-                case "DotnetDebugger.addSymbolServerUrl":
+                case "DotnetDebugger.setSymbolOptions":
                     {
-                        string url = args["url"]?.Value<string>();
-                        if (!string.IsNullOrEmpty(url) && !urlSymbolServerList.Contains(url))
-                            urlSymbolServerList.Add(url);
+                        SendResponse(id, Result.OkFromObject(new { }), token);
+                        CachePathSymbolServer = args["symbolOptions"]?["cachePath"]?.Value<string>();
+                        var urls = args["symbolOptions"]?["searchPaths"]?.Value<JArray>();
+                        if (urls == null)
+                            return true;
+                        UrlSymbolServerList.Clear();
+                        UrlSymbolServerList.AddRange(urls.Values<string>());
+                        if (!JustMyCode)
+                        {
+                            if (!await IsRuntimeAlreadyReadyAlready(id, token))
+                                return true;
+                            return await ReloadSymbolsFromSymbolServer(id, context, token);
+                        }
                         return true;
                     }
                 case "DotnetDebugger.getMethodLocation":
@@ -578,6 +591,14 @@ protected override async Task<bool> AcceptCommand(MessageId id, JObject parms, C
             return method.StartsWith("DotnetDebugger.", StringComparison.OrdinalIgnoreCase);
         }
 
+        private async Task<bool> ReloadSymbolsFromSymbolServer(SessionId id, ExecutionContext context, CancellationToken token)
+        {
+            DebugStore store = await LoadStore(id, true, token);
+            store.UpdateSymbolStore(UrlSymbolServerList, CachePathSymbolServer);
+            await store.ReloadAllPDBsFromSymbolServersAndSendSources(this, id, context, token);
+            return true;
+        }
+
         private async Task<bool> ApplyUpdates(MessageId id, JObject args, CancellationToken token)
         {
             var context = GetContext(id);
@@ -590,8 +611,14 @@ private async Task<bool> ApplyUpdates(MessageId id, JObject args, CancellationTo
             return applyUpdates;
         }
 
-        private void SetJustMyCode(MessageId id, bool isEnabled, CancellationToken token)
+        private async Task SetJustMyCode(MessageId id, bool isEnabled, ExecutionContext context, CancellationToken token)
         {
+            if (JustMyCode != isEnabled && isEnabled == false)
+            {
+                JustMyCode = isEnabled;
+                if (await IsRuntimeAlreadyReadyAlready(id, token))
+                    await ReloadSymbolsFromSymbolServer(id, context, token);
+            }
             JustMyCode = isEnabled;
             SendResponse(id, Result.OkFromObject(new { justMyCodeEnabled = JustMyCode }), token);
         }
@@ -886,7 +913,7 @@ private async Task<bool> SendBreakpointsOfMethodUpdated(SessionId sessionId, Exe
             return true;
         }
 
-        protected virtual async Task<bool> ShouldSkipMethod(SessionId sessionId, ExecutionContext context, EventKind event_kind, int j, MethodInfoWithDebugInformation method, CancellationToken token)
+        protected virtual async Task<bool> ShouldSkipMethod(SessionId sessionId, ExecutionContext context, EventKind event_kind, int frameNumber, MethodInfoWithDebugInformation method, CancellationToken token)
         {
             var shouldReturn = await SkipMethod(
                     isSkippable: context.IsSkippingHiddenMethod,
@@ -904,7 +931,10 @@ protected virtual async Task<bool> ShouldSkipMethod(SessionId sessionId, Executi
             if (shouldReturn)
                 return true;
 
-            if (j == 0 && method?.Info.DebuggerAttrInfo.DoAttributesAffectCallStack(JustMyCode) == true)
+            if (frameNumber != 0)
+                return false;
+
+            if (method?.Info?.DebuggerAttrInfo?.DoAttributesAffectCallStack(JustMyCode) == true)
             {
                 if (method.Info.DebuggerAttrInfo.ShouldStepOut(event_kind))
                 {
@@ -929,6 +959,16 @@ protected virtual async Task<bool> ShouldSkipMethod(SessionId sessionId, Executi
                         context.IsResumedAfterBp = true;
                 }
             }
+            else
+            {
+                if (!JustMyCode && method?.Info?.DebuggerAttrInfo?.HasNonUserCode == true && !method.Info.hasDebugInformation)
+                {
+                    if (event_kind == EventKind.Step)
+                        context.IsSkippingHiddenMethod = true;
+                    if (await SkipMethod(isSkippable: true, shouldBeSkipped: true, StepKind.Out))
+                        return true;
+                }
+            }
             return false;
             async Task<bool> SkipMethod(bool isSkippable, bool shouldBeSkipped, StepKind stepKind)
             {
@@ -1132,49 +1172,6 @@ internal async Task<bool> OnReceiveDebuggerAgentEvent(SessionId sessionId, JObje
             return false;
         }
 
-        internal async Task<MethodInfo> LoadSymbolsOnDemand(AssemblyInfo asm, int method_token, SessionId sessionId, CancellationToken token)
-        {
-            ExecutionContext context = GetContext(sessionId);
-            if (urlSymbolServerList.Count == 0)
-                return null;
-            if (asm.TriedToLoadSymbolsOnDemand || !asm.CodeViewInformationAvailable)
-                return null;
-            asm.TriedToLoadSymbolsOnDemand = true;
-            var pdbName = Path.GetFileName(asm.PdbName);
-
-            foreach (string urlSymbolServer in urlSymbolServerList)
-            {
-                string downloadURL = $"{urlSymbolServer}/{pdbName}/{asm.PdbGuid.ToString("N").ToUpperInvariant() + asm.PdbAge}/{pdbName}";
-
-                try
-                {
-                    using HttpResponseMessage response = await HttpClient.GetAsync(downloadURL, token);
-                    if (!response.IsSuccessStatusCode)
-                    {
-                        Log("info", $"Unable to download symbols on demand url:{downloadURL} assembly: {asm.Name}");
-                        continue;
-                    }
-
-                    using Stream streamToReadFrom = await response.Content.ReadAsStreamAsync(token);
-                    asm.UpdatePdbInformation(streamToReadFrom);
-                    foreach (SourceFile source in asm.Sources)
-                    {
-                        var scriptSource = JObject.FromObject(source.ToScriptSource(context.Id, context.AuxData));
-                        await SendEvent(sessionId, "Debugger.scriptParsed", scriptSource, token);
-                    }
-                    return asm.GetMethodByToken(method_token);
-                }
-                catch (Exception e)
-                {
-                    Log("info", $"Unable to load symbols on demand exception: {e} url:{downloadURL} assembly: {asm.Name}");
-                }
-                break;
-            }
-
-            Log("info", $"Unable to load symbols on demand assembly: {asm.Name}");
-            return null;
-        }
-
         protected void OnDefaultContextUpdate(SessionId sessionId, ExecutionContext context)
         {
             if (UpdateContext(sessionId, context, out ExecutionContext previousContext))
@@ -1575,7 +1572,7 @@ private static IEnumerable<IGrouping<SourceId, SourceLocation>> GetBPReqLocation
         {
             var comparer = new SourceLocation.LocationComparer();
             // if column is specified the frontend wants the exact matches
-            // and will clear the bp if it isn't close enoug
+            // and will clear the bp if it isn't close enough
             var bpLocations = store.FindBreakpointLocations(req, ifNoneFoundThenFindNext);
             IEnumerable<IGrouping<SourceId, SourceLocation>> locations = bpLocations.Distinct(comparer)
                 .OrderBy(l => l.Column)