Allow emitting debugging symbols via high level scripting APIs

src/Compilers/Core/Portable/CorLightup.cs

@@ -58,6 +58,11 @@ private static class _Assembly
                     .GetDeclaredMethod("Load", typeof(byte[]))
                     .CreateDelegate<Func<byte[], Assembly>>();
 
+                internal static readonly Func<byte[], byte[], Assembly> Load_bytes_with_Pdb = Type
+                    .GetTypeInfo()
+                    .GetDeclaredMethod("Load", typeof(byte[]), typeof(byte[]))
+                    .CreateDelegate<Func<byte[], byte[], Assembly>>();
+
                 internal static readonly Func<string, Assembly> LoadFile = Type
                     .GetTypeInfo()
                     .GetDeclaredMethod("LoadFile", typeof(string))
@@ -125,6 +130,16 @@ internal static Assembly LoadAssembly(byte[] peImage)
                 return _Assembly.Load_bytes(peImage);
             }
 
+            internal static Assembly LoadAssembly(byte[] peImage, byte[] pdbImage)
+            {
+                if (_Assembly.Load_bytes_with_Pdb == null)
+                {
+                    throw new PlatformNotSupportedException();
+                }
+
+                return _Assembly.Load_bytes_with_Pdb(peImage, pdbImage);
+            }
+
             internal static Assembly LoadAssembly(string path)
             {
                 if (_Assembly.LoadFile == null)

src/Scripting/CSharp/CSharpScript.cs

@@ -1,10 +1,13 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+#pragma warning disable RS0026 // Do not add multiple public overloads with optional parameters
 
 using System;
+using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.Scripting;
 using Microsoft.CodeAnalysis.Scripting.Hosting;
+using Microsoft.CodeAnalysis.Text;
 
 namespace Microsoft.CodeAnalysis.CSharp.Scripting
 {
@@ -23,7 +26,23 @@ public static class CSharpScript
         /// <typeparam name="T">The return type of the script</typeparam>
         public static Script<T> Create<T>(string code, ScriptOptions options = null, Type globalsType = null, InteractiveAssemblyLoader assemblyLoader = null)
         {
-            return Script.CreateInitialScript<T>(CSharpScriptCompiler.Instance, code, options, globalsType, assemblyLoader);
+            return Script.CreateInitialScript<T>(CSharpScriptCompiler.Instance, SourceText.From(code, options?.FileEncoding), options, globalsType, assemblyLoader);
+        }
+
+        /// <summary>
+        /// Create a new C# script.
+        /// </summary>
+        /// <param name="code">The <see cref="Stream"/> representing the source code of the script.</param>
+        /// <param name="options">The script options.</param>
+        /// <param name="globalsType">Type of global object.</param>
+        /// <param name="assemblyLoader">Custom  assembly loader.</param>
+        /// <typeparam name="T">The return type of the script</typeparam>
+        /// <exception cref="ArgumentNullException">Stream is null.</exception>
+        /// <exception cref="ArgumentException">Stream is not readable or seekable.</exception>
+        public static Script<T> Create<T>(Stream code, ScriptOptions options = null, Type globalsType = null, InteractiveAssemblyLoader assemblyLoader = null)
+        {
+            if (code == null) throw new ArgumentNullException(nameof(code));
+            return Script.CreateInitialScript<T>(CSharpScriptCompiler.Instance, SourceText.From(code, options?.FileEncoding), options, globalsType, assemblyLoader);
         }
 
         /// <summary>
@@ -38,6 +57,21 @@ public static Script<object> Create(string code, ScriptOptions options = null, T
             return Create<object>(code, options, globalsType, assemblyLoader);
         }
 
+        /// <summary>
+        /// Create a new C# script.
+        /// </summary>
+        /// <param name="code">The <see cref="Stream"/> representing the source code of the script.</param>
+        /// <param name="options">The script options.</param>
+        /// <param name="globalsType">Type of global object.</param>
+        /// <param name="assemblyLoader">Custom  assembly loader.</param>
+        /// <exception cref="ArgumentNullException">Stream is null.</exception>
+        /// <exception cref="ArgumentException">Stream is not readable or seekable.</exception>
+        public static Script<object> Create(Stream code, ScriptOptions options = null, Type globalsType = null, InteractiveAssemblyLoader assemblyLoader = null)
+        {
+            if (code == null) throw new ArgumentNullException(nameof(code));
+            return Create<object>(code, options, globalsType, assemblyLoader);
+        }
+
         /// <summary>
         /// Run a C# script.
         /// </summary>

src/Scripting/CSharp/CSharpScriptCompiler.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System;
+using System.Text;
 using System.Threading;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Scripting;
@@ -41,7 +42,7 @@ public override Compilation CreateSubmission(Script script)
             // TODO: report diagnostics
             diagnostics.Free();
 
-            var tree = SyntaxFactory.ParseSyntaxTree(script.Code, s_defaultOptions, script.Options.FilePath);
+            var tree = SyntaxFactory.ParseSyntaxTree(script.SourceText, s_defaultOptions, script.Options.FilePath);
 
             string assemblyName, submissionTypeName;
             script.Builder.GenerateSubmissionId(out assemblyName, out submissionTypeName);

src/Scripting/CSharp/PublicAPI.Unshipped.txt

@@ -0,0 +1,2 @@
+static Microsoft.CodeAnalysis.CSharp.Scripting.CSharpScript.Create(System.IO.Stream code, Microsoft.CodeAnalysis.Scripting.ScriptOptions options = null, System.Type globalsType = null, Microsoft.CodeAnalysis.Scripting.Hosting.InteractiveAssemblyLoader assemblyLoader = null) -> Microsoft.CodeAnalysis.Scripting.Script<object>
+static Microsoft.CodeAnalysis.CSharp.Scripting.CSharpScript.Create<T>(System.IO.Stream code, Microsoft.CodeAnalysis.Scripting.ScriptOptions options = null, System.Type globalsType = null, Microsoft.CodeAnalysis.Scripting.Hosting.InteractiveAssemblyLoader assemblyLoader = null) -> Microsoft.CodeAnalysis.Scripting.Script<T>

src/Scripting/CSharpTest/ScriptTests.cs

@@ -11,6 +11,8 @@
 using Xunit;
 using System.IO;
 using System.Globalization;
+using System.Text;
+using System.Diagnostics;
 
 namespace Microsoft.CodeAnalysis.CSharp.Scripting.UnitTests
 {
@@ -29,6 +31,13 @@ public void TestCreateScript()
             Assert.Equal("1 + 2", script.Code);
         }
 
+        [Fact]
+        public void TestCreateFromStreamScript()
+        {
+            var script = CSharpScript.Create(new MemoryStream(Encoding.UTF8.GetBytes("1 + 2")));
+            Assert.Equal("1 + 2", script.Code);
+        }
+
         [Fact]
         public async Task TestGetCompilation()
         {
@@ -37,6 +46,14 @@ public async Task TestGetCompilation()
             Assert.Equal(state.Script.Code, compilation.SyntaxTrees.First().GetText().ToString());
         }
 
+        [Fact]
+        public async Task TestGetCompilationSourceText()
+        {
+            var state = await CSharpScript.RunAsync("1 + 2", globals: new ScriptTests());
+            var compilation = state.Script.GetCompilation();
+            Assert.Equal(state.Script.SourceText, compilation.SyntaxTrees.First().GetText());
+        }
+
         [Fact]
         public void TestCreateScriptDelegate()
         {
@@ -76,6 +93,15 @@ public async Task TestCreateAndRunScript()
             Assert.Equal(3, state.ReturnValue);
         }
 
+        [Fact]
+        public async Task TestCreateFromStreamAndRunScript()
+        {
+            var script = CSharpScript.Create(new MemoryStream(Encoding.UTF8.GetBytes("1 + 2")));
+            var state = await script.RunAsync();
+            Assert.Same(script, state.Script);
+            Assert.Equal(3, state.ReturnValue);
+        }
+
         [Fact]
         public async Task TestEvalScript()
         {
@@ -687,6 +713,99 @@ public async Task LoadedFileWithVoidReturn()
             Assert.Equal(0, result);
         }
 
+        [Fact]
+        public async Task Pdb_CreateFromString_CodeFromFile_WithEmitDebugInformation_WithoutFileEncoding_CompilationErrorException()
+        {
+            var code = "throw new System.Exception();";
+            try
+            {
+                var opts = ScriptOptions.Default.WithEmitDebugInformation(true).WithFilePath("debug.csx").WithFileEncoding(null);
+                var script = await CSharpScript.RunAsync(code, opts);
+            }
+            catch (CompilationErrorException ex)
+            {
+                //  CS8055: Cannot emit debug information for a source text without encoding.
+                ex.Diagnostics.Verify(Diagnostic(ErrorCode.ERR_EncodinglessSyntaxTree, code).WithLocation(1,1));
+            }
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_CodeFromFile_WithEmitDebugInformation_WithFileEncoding_ResultInPdbEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(true).WithFilePath("debug.csx").WithFileEncoding(Encoding.UTF8);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts), line: 1, column: 1, filename: "debug.csx");
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_CodeFromFile_WithoutEmitDebugInformation_WithoutFileEncoding_ResultInPdbNotEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(false).WithFilePath(null).WithFileEncoding(null);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts));
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_CodeFromFile_WithoutEmitDebugInformation_WithFileEncoding_ResultInPdbNotEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(false).WithFilePath("debug.csx").WithFileEncoding(Encoding.UTF8);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts));
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromStream_CodeFromFile_WithEmitDebugInformation_ResultInPdbEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(true).WithFilePath("debug.csx");
+            return VerifyStackTraceAsync(() => CSharpScript.Create(new MemoryStream(Encoding.UTF8.GetBytes("throw new System.Exception();")), opts), line: 1, column: 1, filename: "debug.csx");
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromStream_CodeFromFile_WithoutEmitDebugInformation_ResultInPdbNotEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(false).WithFilePath("debug.csx");
+            return VerifyStackTraceAsync(() => CSharpScript.Create(new MemoryStream(Encoding.UTF8.GetBytes("throw new System.Exception();")), opts));
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_InlineCode_WithEmitDebugInformation_WithoutFileEncoding_ResultInPdbEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(true).WithFileEncoding(null);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts), line: 1, column: 1, filename: "");
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_InlineCode_WithEmitDebugInformation_WithFileEncoding_ResultInPdbEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(true).WithFileEncoding(Encoding.UTF8);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts), line: 1, column: 1, filename: "");
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_InlineCode_WithoutEmitDebugInformation_WithoutFileEncoding_ResultInPdbNotEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(false).WithFileEncoding(null);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts));
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromString_InlineCode_WithoutEmitDebugInformation_WithFileEncoding_ResultInPdbNotEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(false).WithFileEncoding(Encoding.UTF8);
+            return VerifyStackTraceAsync(() => CSharpScript.Create("throw new System.Exception();", opts));
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromStream_InlineCode_WithEmitDebugInformation_ResultInPdbEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(true);
+            return VerifyStackTraceAsync(() => CSharpScript.Create(new MemoryStream(Encoding.UTF8.GetBytes("throw new System.Exception();")), opts), line: 1, column: 1, filename: "");
+        }
+
+        [Fact]
+        public Task Pdb_CreateFromStream_InlineCode_WithoutEmitDebugInformation_ResultInPdbNotEmitted()
+        {
+            var opts = ScriptOptions.Default.WithEmitDebugInformation(false);
+            return VerifyStackTraceAsync(() => CSharpScript.Create(new MemoryStream(Encoding.UTF8.GetBytes("throw new System.Exception();")), opts));
+        }
+
         [WorkItem(12348, "https://github.com/dotnet/roslyn/issues/12348")]
         [Fact]
         public async Task StreamWithOffset()
@@ -739,5 +858,23 @@ public override Stream OpenRead(string resolvedPath)
                 return stream;
             }
         }
+
+        private async Task VerifyStackTraceAsync(Func<Script<object>> scriptProvider, int line = 0, int column = 0, string filename = null)
+        {
+            try
+            {
+                var script = scriptProvider();
+                await script.RunAsync();
+            }
+            catch (Exception ex)
+            {
+                // line information is only available when PDBs have been emitted
+                var stackTrace = new StackTrace(ex, needFileInfo: true);
+                var firstFrame = stackTrace.GetFrames()[0];
+                Assert.Equal(filename, firstFrame.GetFileName());
+                Assert.Equal(line, firstFrame.GetFileLineNumber());
+                Assert.Equal(column, firstFrame.GetFileColumnNumber());
+            }
+        }
     }
 }

src/Scripting/Core/Hosting/AssemblyLoader/DesktopAssemblyLoaderImpl.cs

@@ -30,6 +30,15 @@ public override Assembly LoadFromStream(Stream peStream, Stream pdbStream)
         {
             byte[] peImage = new byte[peStream.Length];
             peStream.TryReadAll(peImage, 0, peImage.Length);
+
+            if (pdbStream != null)
+            {
+                byte[] pdbImage = new byte[pdbStream.Length];
+                pdbStream.TryReadAll(pdbImage, 0, pdbImage.Length);
+
+                return CorLightup.Desktop.LoadAssembly(peImage, pdbImage);
+            }
+
             return CorLightup.Desktop.LoadAssembly(peImage);
         }
 

src/Scripting/Core/Hosting/CommandLine/CommandLineRunner.cs

@@ -149,7 +149,9 @@ private static ScriptOptions GetScriptOptions(CommandLineArguments arguments, st
                 references: ImmutableArray.CreateRange(resolvedReferences),
                 namespaces: CommandLineHelpers.GetImports(arguments),
                 metadataResolver: metadataResolver,
-                sourceResolver: sourceResolver);
+                sourceResolver: sourceResolver,
+                emitDebugInformation: false,
+                fileEncoding: null);
         }
 
         internal static MetadataReferenceResolver GetMetadataReferenceResolver(CommandLineArguments arguments, TouchedFileLogger loggerOpt)
@@ -176,7 +178,7 @@ private int RunScript(ScriptOptions options, string code, ErrorLogger errorLogge
             var globals = new CommandLineScriptGlobals(_console.Out, _objectFormatter);
             globals.Args.AddRange(_compiler.Arguments.ScriptArguments);
 
-            var script = Script.CreateInitialScript<int>(_scriptCompiler, code, options, globals.GetType(), assemblyLoaderOpt: null);
+            var script = Script.CreateInitialScript<int>(_scriptCompiler, SourceText.From(code), options, globals.GetType(), assemblyLoaderOpt: null);
             try
             {
                 return script.RunAsync(globals, cancellationToken).Result.ReturnValue;
@@ -197,7 +199,7 @@ private void RunInteractiveLoop(ScriptOptions options, string initialScriptCodeO
 
             if (initialScriptCodeOpt != null)
             {
-                var script = Script.CreateInitialScript<object>(_scriptCompiler, initialScriptCodeOpt, options, globals.GetType(), assemblyLoaderOpt: null);
+                var script = Script.CreateInitialScript<object>(_scriptCompiler, SourceText.From(initialScriptCodeOpt), options, globals.GetType(), assemblyLoaderOpt: null);
                 BuildAndRun(script, globals, ref state, ref options, displayResult: false, cancellationToken: cancellationToken);
             }
 
@@ -249,7 +251,7 @@ private void RunInteractiveLoop(ScriptOptions options, string initialScriptCodeO
                 Script<object> newScript;
                 if (state == null)
                 {
-                    newScript = Script.CreateInitialScript<object>(_scriptCompiler, code, options, globals.GetType(), assemblyLoaderOpt: null);
+                    newScript = Script.CreateInitialScript<object>(_scriptCompiler, SourceText.From(code ?? string.Empty), options, globals.GetType(), assemblyLoaderOpt: null);
                 }
                 else
                 {

src/Scripting/Core/PublicAPI.Unshipped.txt

@@ -0,0 +1,6 @@
+Microsoft.CodeAnalysis.Scripting.Script.ContinueWith(System.IO.Stream code, Microsoft.CodeAnalysis.Scripting.ScriptOptions options = null) -> Microsoft.CodeAnalysis.Scripting.Script<object>
+Microsoft.CodeAnalysis.Scripting.Script.ContinueWith<TResult>(System.IO.Stream code, Microsoft.CodeAnalysis.Scripting.ScriptOptions options = null) -> Microsoft.CodeAnalysis.Scripting.Script<TResult>
+Microsoft.CodeAnalysis.Scripting.ScriptOptions.EmitDebugInformation.get -> bool
+Microsoft.CodeAnalysis.Scripting.ScriptOptions.FileEncoding.get -> System.Text.Encoding
+Microsoft.CodeAnalysis.Scripting.ScriptOptions.WithEmitDebugInformation(bool emitDebugInformation) -> Microsoft.CodeAnalysis.Scripting.ScriptOptions
+Microsoft.CodeAnalysis.Scripting.ScriptOptions.WithFileEncoding(System.Text.Encoding encoding) -> Microsoft.CodeAnalysis.Scripting.ScriptOptions