Bug 432: Allow for non-ASCII characters in scripts

src/grate.core/Infrastructure/ISyntax.cs

@@ -8,11 +8,6 @@ public interface ISyntax
     string StatementSeparatorRegex { get; }
     string CurrentDatabase { get; }
     string ListDatabases { get; }
-    string VarcharType { get; }
-    string TextType { get; }
-    string BigintType { get; }
-    string BooleanType { get; }
-    string CreateSchema(string schemaName);
     string CreateDatabase(string databaseName, string? password);
     /// <summary>
     /// Syntax to drop a database if it exists, and do nothing if not.
@@ -23,9 +18,5 @@ public interface ISyntax
     string TableWithSchema(string schemaName, string tableName);
     string LimitN(string sql, int n);
     string ReturnId { get; }
-    string TimestampType { get; }
-    string Quote(string text);
-    string PrimaryKeyColumn(string columnName);
-    string PrimaryKeyConstraint(string tableName, string column);
     string ResetIdentity(string schemaName, string tableName, long value);
 }

src/grate.core/Migration/AnsiSqlDatabase.cs

@@ -343,7 +343,7 @@ INSERT INTO {VersionTable}
                     newVersion,
                     entryDate = DateTime.UtcNow,
                     modifiedDate = DateTime.UtcNow,
-                    enteredBy = ClaimsPrincipal.Current?.Identity?.Name ?? Environment.UserName,
+                    enteredBy = GetUserName(),
                     status = MigrationStatus.InProgress
                 });
         } 
@@ -365,6 +365,11 @@ INSERT INTO {VersionTable}
         return versionId;
     }
 
+    protected string GetUserName()
+    {
+        return ClaimsPrincipal.Current?.Identity?.Name ?? Environment.UserName;
+    }
+
     public virtual async Task ChangeVersionStatus(string status, long versionId)
     {
         var updateSql = Parameterize($@"
@@ -532,7 +537,7 @@ INSERT INTO {ScriptsRunTable}
         scriptRun.Add(nameof(hash), hash);
         scriptRun.Add(nameof(runOnce), Bool(runOnce));
         scriptRun.Add(Now, DateTime.UtcNow);
-        scriptRun.Add(User, Environment.UserName);
+        scriptRun.Add(User, GetUserName());
 
         if (Config!.DeferWritingToRunTables)
         {
@@ -564,7 +569,7 @@ INSERT INTO {ScriptsRunErrorsTable}
         scriptRunErrors.Add(nameof(errorSql), errorSql, DbType.String);
         scriptRunErrors.Add(nameof(errorMessage), errorMessage, DbType.String);
         scriptRunErrors.Add(Now, DateTime.UtcNow);
-        scriptRunErrors.Add(User, Environment.UserName);
+        scriptRunErrors.Add(User, GetUserName());
 
         if (Config!.DeferWritingToRunTables)
         {
@@ -655,7 +660,7 @@ public async ValueTask DisposeAsync()
         if (_deferredWrites.Any())
         {
             await OpenActiveConnection();
-            foreach (var deferredWrite in _deferredWrites)
+            foreach (var deferredWrite in _deferredWrites.ToArray())
             {
                 await deferredWrite(ActiveConnection);
             }
@@ -664,7 +669,7 @@ public async ValueTask DisposeAsync()
         await CloseConnection();
         await CloseAdminConnection();
         await Close(ActiveConnection);
-
+        
         GC.SuppressFinalize(this);
     }
 

src/grate.core/Migration/GrateMigrator.cs

@@ -515,6 +515,7 @@ public static string ChangeDropFolder(GrateConfiguration config, string? server,
         .Append(':')
         .Append(',')
         .Append('+')
+        .Append('/')
         .ToArray();
 
     private static string RemoveInvalidPathChars(string? path)
@@ -557,18 +558,21 @@ private async Task RunInternalMigrations(string internalFolderName)
         // First, make sure we have created the "internal meta tables"
         // (GrateScriptsRun, GrateScriptsRunErrors, GrateVersion), which are used to track
         // changes to the grate internal tables (ScriptsRun, ScriptsRunErrors, Version).
-        await using (var migrator1 = this.WithConfiguration(await GetBootstrapInternalGrateConfiguration(internalFolderName)))
+        await using (var migrator1 =
+                         this.WithConfiguration(await GetBootstrapInternalGrateConfiguration(internalFolderName)))
         {
             await migrator1.Migrate();
         }
 
         // Then, make sure we have created the "grate tables" for the database we are migrating.
         // (ScriptsRun, ScriptsRunErrors, Version). Turtles all the way down!
-        await using var migrator2 = this.WithConfiguration(await GetInternalGrateConfiguration(internalFolderName));
-        await migrator2.Migrate();
+        await using (var migrator2 =
+            this.WithConfiguration(await GetInternalGrateConfiguration(internalFolderName)))
+        {
+            await migrator2.Migrate();
+        }
     }
 
-
     private async Task<GrateConfiguration> GetBootstrapInternalGrateConfiguration(string internalFolderName) =>
         await GetInternalGrateConfiguration(internalFolderName, "grate-internal") with
         {

src/grate.mariadb/Bootstrapping/Sql/GrateStructure/up/02_scripts_run_unicode_support.sql

@@ -0,0 +1,4 @@
+ALTER TABLE {{SchemaName}}_{{ScriptsRunTable}}
+    MODIFY script_name varchar(255) CHARACTER SET utf8mb4 NULL,
+    MODIFY text_of_script text CHARACTER SET utf8mb4 NULL,
+    MODIFY entered_by varchar(50) CHARACTER SET utf8mb4 NULL

src/grate.mariadb/Bootstrapping/Sql/GrateStructure/up/03_scripts_run_errors_unicode_support.sql

@@ -0,0 +1,8 @@
+ALTER TABLE {{SchemaName}}_{{ScriptsRunErrorsTable}}
+    MODIFY repository_path varchar(255) CHARACTER SET utf8mb4 NULL,
+    MODIFY version varchar(50) CHARACTER SET utf8mb4 NULL,
+    MODIFY script_name varchar(255) CHARACTER SET utf8mb4 NULL,
+    MODIFY text_of_script text CHARACTER SET utf8mb4 NULL,
+    MODIFY erroneous_part_of_script text CHARACTER SET utf8mb4 NULL,
+    MODIFY error_message text CHARACTER SET utf8mb4 NULL,
+    MODIFY entered_by varchar(50) CHARACTER SET utf8mb4 NULL

src/grate.mariadb/Bootstrapping/Sql/GrateStructure/up/03_version_table_unicode_support.sql

@@ -0,0 +1,5 @@
+ALTER TABLE {{SchemaName}}_{{VersionTable}}
+    MODIFY repository_path varchar(255) CHARACTER SET utf8mb4  NULL,
+    MODIFY version varchar(50) CHARACTER SET utf8mb4  NULL,
+    MODIFY entered_by varchar(50) CHARACTER SET utf8mb4  NULL,
+    MODIFY status varchar(50) CHARACTER SET utf8mb4  NULL

src/grate.mariadb/Infrastructure/MariaDbSyntax.cs

@@ -17,19 +17,10 @@ public string StatementSeparatorRegex
 
     public string CurrentDatabase => "SELECT DATABASE()";
     public string ListDatabases => "SHOW DATABASES";
-    public string VarcharType => "varchar";
-    public string TextType => "text";
-    public string BigintType => "BIGINT";
-    public string BooleanType => "boolean";
-    public string PrimaryKeyColumn(string columnName) => $"{columnName} bigint NOT NULL AUTO_INCREMENT";
-    public string CreateSchema(string schemaName) => @$"CREATE SCHEMA {schemaName}";
     public string CreateDatabase(string databaseName, string? _) => @$"CREATE DATABASE {databaseName}";
     public string DropDatabase(string databaseName) => @$"DROP DATABASE IF EXISTS `{databaseName}`;";
     public string TableWithSchema(string schemaName, string tableName) => $"{schemaName}_{tableName}";
     public string ReturnId => ";SELECT LAST_INSERT_ID();";
-    public string TimestampType => "timestamp";
-    public string Quote(string text) => $"`{text}`";
-    public string PrimaryKeyConstraint(string tableName, string column) => $",\nCONSTRAINT PK_{tableName}_{column} PRIMARY KEY ({column})";
     public string LimitN(string sql, int n) => sql + "\nLIMIT 1";
 
     // any idea to reset identity without using value is welcome.

src/grate.oracle/Infrastructure/OracleSyntax.cs

@@ -18,13 +18,6 @@ public string StatementSeparatorRegex
 
     public string CurrentDatabase => "select user from dual";
     public string ListDatabases => "SELECT * FROM all_users";
-    public string VarcharType => "VARCHAR2";
-    public string TextType => "CLOB";
-    public string BigintType => "NUMBER(19)";
-    public string BooleanType => "CHAR(1)";
-    public string PrimaryKeyColumn(string columnName) => $"{columnName} NUMBER(19) GENERATED ALWAYS AS IDENTITY NOT NULL PRIMARY KEY ";
-
-    public string CreateSchema(string schemaName) => throw new NotImplementedException("Create schema is not implemented for Oracle DB");
 
     public string CreateDatabase(string userName, string? password) => $@"
 begin 
@@ -48,9 +41,6 @@ FOR ln_cur IN (SELECT sid, serial# FROM v$session WHERE username = usr)
 ";
     public string TableWithSchema(string schemaName, string tableName) => $"{schemaName}_{tableName}";
     public string ReturnId => "RETURNING id;";
-    public string TimestampType => "timestamp";
-    public string Quote(string text) => $"\"{text}\"";
-    public string PrimaryKeyConstraint(string tableName, string column) => "";
     public string LimitN(string sql, int n) => sql + $"\nLIMIT {n}";
     public string ResetIdentity(string schemaName, string tableName, long _) => $"ALTER TABLE {TableWithSchema(schemaName, tableName)} MODIFY id NUMBER(19) GENERATED BY DEFAULT ON NULL AS IDENTITY (START WITH LIMIT VALUE)";
 }

src/grate.oracle/Migration/OracleDatabase.cs

@@ -109,7 +109,7 @@ INSERT INTO {VersionTable}
             newVersion,
             entryDate = DateTime.UtcNow,
             modifiedDate = DateTime.UtcNow,
-            enteredBy = ClaimsPrincipal.Current?.Identity?.Name ?? Environment.UserName,
+            enteredBy = GetUserName(),
             status = MigrationStatus.InProgress
         };
         var dynParams = new DynamicParameters(parameters);

src/grate.postgresql/Infrastructure/PostgreSqlSyntax.cs

@@ -19,21 +19,12 @@ public string StatementSeparatorRegex
 
     public string CurrentDatabase => "SELECT current_database()";
     public string ListDatabases => "SELECT datname FROM pg_database";
-    public string VarcharType => "varchar";
-    public string TextType => "text";
-    public string BigintType => "BIGINT";
-    public string BooleanType => "boolean";
-    public string PrimaryKeyColumn(string columnName) => $"{columnName} bigint GENERATED ALWAYS AS IDENTITY NOT NULL";
-    public string CreateSchema(string schemaName) => @$"CREATE SCHEMA ""{schemaName}"";";
     public string CreateDatabase(string databaseName, string? _) => @$"CREATE DATABASE ""{databaseName}""";
     public string DropDatabase(string databaseName) => @$"select pg_terminate_backend(pid) from pg_stat_activity where datname='{databaseName}';
                                                               COMMIT;
                                                               DROP DATABASE IF EXISTS ""{databaseName}"";";
     public string TableWithSchema(string schemaName, string tableName) => $"\"{schemaName}\".\"{tableName}\"";
     public string ReturnId => "RETURNING id;";
-    public string TimestampType => "timestamp";
-    public string Quote(string text) => $"\"{text}\"";
-    public string PrimaryKeyConstraint(string tableName, string column) => $",\nCONSTRAINT PK_{tableName}_{column} PRIMARY KEY ({column})";
     public string LimitN(string sql, int n) => sql + $"\nLIMIT {n}";
     public string ResetIdentity(string schemaName, string tableName, long _) => @$"SELECT setval(pg_get_serial_sequence('{TableWithSchema(schemaName, tableName)}', 'id'), coalesce(MAX(id), 1)) from {TableWithSchema(schemaName, tableName)};";
 }

src/grate.sqlite/Infrastructure/SqliteSyntax.cs

@@ -17,12 +17,6 @@ public string StatementSeparatorRegex
 
     public string CurrentDatabase => "SELECT name FROM pragma_database_list ORDER BY seq DESC LIMIT 1";
     public string ListDatabases => "select name from pragma_database_list";
-    public string VarcharType => "nvarchar";
-    public string TextType => "ntext";
-    public string BigintType => "BIGINT";
-    public string BooleanType => "bit";
-    public string PrimaryKeyColumn(string columnName) => $"{columnName} INTEGER PRIMARY KEY AUTOINCREMENT";
-    public string CreateSchema(string schemaName) => @$"CREATE SCHEMA ""{schemaName}"";";
 
     // The "Create database" is a no-op with Sqlite, so we just provide a dummy SQL that just selects current DB
     public string CreateDatabase(string databaseName, string? _) => CurrentDatabase;
@@ -32,9 +26,6 @@ public string StatementSeparatorRegex
 
     public string TableWithSchema(string schemaName, string tableName) => $"{schemaName}_{tableName}";
     public string ReturnId => "returning id;";
-    public string TimestampType => "datetime";
-    public string Quote(string text) => $"\"{text}\"";
-    public string PrimaryKeyConstraint(string tableName, string column) => "";
     public string LimitN(string sql, int n) => sql + $"\nLIMIT {n}";
     public string ResetIdentity(string schemaName, string tableName, long _) => @$"UPDATE `sqlite_sequence`
                                                                                     SET `seq` = (SELECT MAX(`id`) FROM '{TableWithSchema(schemaName, tableName)}')

src/grate.sqlserver/Bootstrapping/Sql/GrateStructure/up/02_scripts_run_unicode_support.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {{SchemaName}}.{{ScriptsRunTable}}
+    ALTER COLUMN text_of_script NVARCHAR(MAX) NULL

src/grate.sqlserver/Bootstrapping/Sql/GrateStructure/up/03_scripts_run_errors_unicode_support.sql

@@ -0,0 +1,8 @@
+ALTER TABLE {{SchemaName}}.{{ScriptsRunErrorsTable}}
+    ALTER COLUMN text_of_script NVARCHAR(MAX) NULL
+
+ALTER TABLE {{SchemaName}}.{{ScriptsRunErrorsTable}}
+    ALTER COLUMN erroneous_part_of_script NVARCHAR(MAX) NULL
+
+ALTER TABLE {{SchemaName}}.{{ScriptsRunErrorsTable}}
+    ALTER COLUMN error_message NVARCHAR(MAX) NULL

src/grate.sqlserver/Infrastructure/SqlServerSyntax.cs

@@ -17,12 +17,6 @@ public string StatementSeparatorRegex
 
     public string CurrentDatabase => "SELECT DB_NAME()";
     public string ListDatabases => "SELECT name FROM sys.databases";
-    public string VarcharType => "nvarchar";
-    public string TextType => "ntext";
-    public string BigintType => "BIGINT";
-    public string BooleanType => "bit";
-    public string PrimaryKeyColumn(string columnName) => $"{columnName} bigint IDENTITY(1,1) NOT NULL";
-    public string CreateSchema(string schemaName) => @$"CREATE SCHEMA ""{schemaName}"";";
     public string CreateDatabase(string databaseName, string? _) => @$"CREATE DATABASE ""{databaseName}""";
     public string DropDatabase(string databaseName) => @$"USE master; 
                         IF EXISTS(SELECT * FROM sysdatabases WHERE [name] = '{databaseName}') 
@@ -32,9 +26,6 @@ DROP DATABASE [{databaseName}]
                         END";
     public string TableWithSchema(string schemaName, string tableName) => $"{schemaName}.[{tableName}]";
     public string ReturnId => ";SELECT @@IDENTITY";
-    public string TimestampType => "datetime";
-    public string Quote(string text) => $"\"{text}\"";
-    public string PrimaryKeyConstraint(string tableName, string column) => $",\nCONSTRAINT PK_{tableName}_{column} PRIMARY KEY CLUSTERED ({column})";
     public string LimitN(string sql, int n) => $"TOP {n}\n" + sql;
     public string ResetIdentity(string schemaName, string tableName, long _) => @$"DECLARE @max INT SELECT @max=ISNULL(max([id]),0) from [{schemaName}].[{tableName}]; DBCC CHECKIDENT ('[{schemaName}].[{tableName}]', RESEED, @max );";
 }

unittests/MariaDB/DependencyInjection/ServiceCollectionTest.cs

@@ -5,4 +5,8 @@ namespace MariaDB.DependencyInjection;
 [Collection(nameof(MariaDbGrateTestContext))]
 // ReSharper disable once UnusedType.Global
 public class ServiceCollectionTest(MariaDbGrateTestContext context)
-    : TestCommon.DependencyInjection.GrateServiceCollectionTest(context);
+    : TestCommon.DependencyInjection.GrateServiceCollectionTest(context)
+{
+    protected override string VarcharType => "varchar";
+    protected override string BigintType => "BIGINT";
+}

unittests/MariaDB/Reported_issues/Non_ascii_characters_in_script/ScriptsRunErrorsTable.cs

@@ -0,0 +1,9 @@
+using MariaDB.TestInfrastructure;
+
+namespace MariaDB.Reported_issues.Non_ascii_characters_in_script;
+
+[Collection(nameof(MariaDbGrateTestContext))]
+// ReSharper disable once InconsistentNaming
+public class ScriptsRunErrorsTable(MariaDbGrateTestContext testContext, ITestOutputHelper testOutput)
+    : TestCommon.Generic.Reported_issues.Non_ascii_characters_in_script.ScriptsRunErrorsTable(testContext, testOutput);
+

unittests/MariaDB/Reported_issues/Non_ascii_characters_in_script/ScriptsRunTable.cs

@@ -0,0 +1,9 @@
+using MariaDB.TestInfrastructure;
+
+namespace MariaDB.Reported_issues.Non_ascii_characters_in_script;
+
+[Collection(nameof(MariaDbGrateTestContext))]
+// ReSharper disable once InconsistentNaming
+public class ScriptsRunTable(MariaDbGrateTestContext testContext, ITestOutputHelper testOutput)
+    : TestCommon.Generic.Reported_issues.Non_ascii_characters_in_script.ScriptsRunTable(testContext, testOutput);
+

unittests/MariaDB/TestInfrastructure/MariaDbGrateTestContext.cs

@@ -11,17 +11,10 @@ namespace MariaDB.TestInfrastructure;
 [CollectionDefinition(nameof(MariaDbGrateTestContext))]
 public class MariaDbTestCollection : ICollectionFixture<MariaDbGrateTestContext>;
 
-public class MariaDbGrateTestContext : GrateTestContext
+public class MariaDbGrateTestContext(
+    IGrateMigrator migrator,
+    ITestDatabase testDatabase) : GrateTestContext(migrator, testDatabase)
 {
-    public MariaDbGrateTestContext(
-        IGrateMigrator migrator, 
-        ITestDatabase testDatabase) : base(testDatabase)
-    {
-        Migrator = migrator;
-    }
-
-    public override IGrateMigrator Migrator { get; }
-
     public override IDbConnection GetDbConnection(string connectionString) => new MySqlConnection(connectionString);
 
     public override ISyntax Syntax { get; } = new MariaDbSyntax();

unittests/Oracle/DependencyInjection/ServiceCollectionTest.cs

@@ -1,15 +1,12 @@
-using grate.Infrastructure;
-using grate.Oracle.Migration;
-using Oracle.TestInfrastructure;
+using Oracle.TestInfrastructure;
 using TestCommon.DependencyInjection;
-using TestCommon.TestInfrastructure;
 
 namespace Oracle.DependencyInjection;
 
 [Collection(nameof(OracleGrateTestContext))]
 public class ServiceCollectionTest(OracleGrateTestContext testContext)
     : GrateServiceCollectionTest(testContext)
 {
-    protected virtual Type DatabaseType => typeof(OracleDatabase);
-    protected virtual ISyntax Syntax => OracleDatabase.Syntax;
+    protected override string VarcharType => "VARCHAR2";
+    protected override string BigintType => "NUMBER(19)";
 }

unittests/Oracle/Reported_issues/Non_ascii_characters_in_script/ScriptsRunErrorsTable.cs

@@ -0,0 +1,9 @@
+using Oracle.TestInfrastructure;
+
+namespace Oracle.Reported_issues.Non_ascii_characters_in_script;
+
+[Collection(nameof(OracleGrateTestContext))]
+// ReSharper disable once InconsistentNaming
+public class ScriptsRunErrorsTable(OracleGrateTestContext testContext, ITestOutputHelper testOutput)
+    : TestCommon.Generic.Reported_issues.Non_ascii_characters_in_script.ScriptsRunErrorsTable(testContext, testOutput);
+

unittests/Oracle/Reported_issues/Non_ascii_characters_in_script/ScriptsRunTable.cs

@@ -0,0 +1,10 @@
+
+using Oracle.TestInfrastructure;
+
+namespace Oracle.Reported_issues.Non_ascii_characters_in_script;
+
+[Collection(nameof(OracleGrateTestContext))]
+// ReSharper disable once InconsistentNaming
+public class ScriptsRunTable(OracleGrateTestContext testContext, ITestOutputHelper testOutput)
+    : TestCommon.Generic.Reported_issues.Non_ascii_characters_in_script.ScriptsRunTable(testContext, testOutput);
+

unittests/Oracle/TestInfrastructure/OracleGrateTestContext.cs

@@ -13,17 +13,10 @@ namespace Oracle.TestInfrastructure;
 [CollectionDefinition(nameof(OracleGrateTestContext))]
 public class OracleTestCollection : ICollectionFixture<OracleGrateTestContext>;
 
-public class OracleGrateTestContext : GrateTestContext
+public class OracleGrateTestContext(
+    IGrateMigrator migrator,
+    ITestDatabase testDatabase) : GrateTestContext(migrator, testDatabase)
 {
-    public OracleGrateTestContext(
-        IGrateMigrator grateMigrator, 
-        ITestDatabase testDatabase) : base(testDatabase)
-    {
-        Migrator = grateMigrator;
-    }
-
-    public override IGrateMigrator Migrator { get; }
-
     // public string DockerCommand(string serverName, string adminPassword) =>
     //     $"run -d --name {serverName} -p 1521 -e ORACLE_ENABLE_XDB=true -e ORACLE_PWD={adminPassword} -P container-registry.oracle.com/database/express:21.3.0-xe";
 
@@ -44,4 +37,5 @@ public OracleGrateTestContext(
     public override string ExpectedVersionPrefix => "Oracle Database 21c Express Edition Release 21.0.0.0.0 - Production";
     public override bool SupportsCreateDatabase => true;
     public override bool SupportsSchemas => false;
+
 }

unittests/PostgreSQL/DependencyInjection/ServiceCollectionTest.cs

@@ -5,4 +5,8 @@ namespace PostgreSQL.DependencyInjection;
 
 [Collection(nameof(PostgreSqlGrateTestContext))]
 public class ServiceCollectionTest(PostgreSqlGrateTestContext context)
-    : TestCommon.DependencyInjection.GrateServiceCollectionTest(context);
+    : TestCommon.DependencyInjection.GrateServiceCollectionTest(context)
+{
+    protected override string VarcharType => "varchar";
+    protected override string BigintType => "BIGINT";
+}