Skip to content

Commit

Permalink
Move batching from model differ to SQL generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@roji
roji committed Sep 1, 2022
1 parent 5e525f2 commit 5946c60
Show file tree
Hide file tree
Showing 4 changed files with 194 additions and 123 deletions.
183 changes: 88 additions & 95 deletions src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2196,120 +2196,113 @@ private IEnumerable<MigrationOperation> GetDataOperations(
r => r.EntityState is EntityState.Added or EntityState.Modified
|| r.EntityState is EntityState.Deleted && diffContext.FindDrop(r.Table!) == null);

var commandBatchPreparer = new CommandBatchPreparer(CommandBatchPreparerDependencies);
var commandSets = commandBatchPreparer.TopologicalSort(commands);
var commandSets = new CommandBatchPreparer(CommandBatchPreparerDependencies)
.TopologicalSort(commands);

for (var commandSetIndex = 0; commandSetIndex < commandSets.Count; commandSetIndex++)
foreach (var commandSet in commandSets)
{
var hasMoreCommandSets = commandSetIndex < commandSets.Count - 1;

foreach (var batch in commandBatchPreparer.CreateCommandBatches(commandSets[commandSetIndex], hasMoreCommandSets))
InsertDataOperation? batchInsertOperation = null;
foreach (var command in commandSet)
{
InsertDataOperation? batchInsertOperation = null;

foreach (var command in batch.ModificationCommands)
switch (command.EntityState)
{
switch (command.EntityState)
{
case EntityState.Added:
if (batchInsertOperation != null)
case EntityState.Added:
if (batchInsertOperation != null)
{
if (batchInsertOperation.Table == command.TableName
&& batchInsertOperation.Schema == command.Schema
&& batchInsertOperation.Columns.SequenceEqual(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)))
{
if (batchInsertOperation.Table == command.TableName
&& batchInsertOperation.Schema == command.Schema
&& batchInsertOperation.Columns.SequenceEqual(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)))
{
batchInsertOperation.Values =
AddToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value)
.ToList(),
batchInsertOperation.Values);
continue;
}

yield return batchInsertOperation;
batchInsertOperation.Values =
AddToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value).ToList(),
batchInsertOperation.Values);
continue;
}

if (forSource)
{
Check.DebugFail("Insert using the source model");
break;
}
yield return batchInsertOperation;
}

batchInsertOperation = new InsertDataOperation
{
Schema = command.Schema,
Table = command.TableName,
Columns = command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)
.ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value).ToList())
};
if (forSource)
{
Check.DebugFail("Insert using the source model");
break;
case EntityState.Modified:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}
}

if (forSource)
{
Check.DebugFail("Update using the source model");
break;
}
batchInsertOperation = new InsertDataOperation
{
Schema = command.Schema,
Table = command.TableName,
Columns = command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)
.ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value).ToList())
};
break;
case EntityState.Modified:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}

yield return new UpdateDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToList()),
Columns = command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.ColumnName).ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.Value).ToList()),
IsDestructiveChange = true
};
if (forSource)
{
Check.DebugFail("Update using the source model");
break;
case EntityState.Deleted:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}
}

// There shouldn't be any deletes using the target model
Check.DebugAssert(forSource, "Delete using the target model");
yield return new UpdateDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToList()),
Columns = command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.ColumnName).ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.Value).ToList()),
IsDestructiveChange = true
};
break;
case EntityState.Deleted:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}

var keyColumns = command.ColumnModifications.Where(col => col.IsKey)
.Select(c => (IColumn)c.Column!);
var anyKeyColumnDropped = keyColumns.Any(c => diffContext.FindDrop(c) != null);
// There shouldn't be any deletes using the target model
Check.DebugAssert(forSource, "Delete using the target model");

yield return new DeleteDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyColumnTypes = anyKeyColumnDropped
? keyColumns.Select(col => col.StoreType).ToArray()
: null,
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToArray()),
IsDestructiveChange = true
};
var keyColumns = command.ColumnModifications.Where(col => col.IsKey)
.Select(c => (IColumn)c.Column!);
var anyKeyColumnDropped = keyColumns.Any(c => diffContext.FindDrop(c) != null);

break;
default:
throw new InvalidOperationException(command.EntityState.ToString());
}
}
yield return new DeleteDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyColumnTypes = anyKeyColumnDropped
? keyColumns.Select(col => col.StoreType).ToArray()
: null,
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToArray()),
IsDestructiveChange = true
};

if (batchInsertOperation != null)
{
yield return batchInsertOperation;
break;
default:
throw new InvalidOperationException(command.EntityState.ToString());
}
}

if (batchInsertOperation != null)
{
yield return batchInsertOperation;
}
}
}

Expand Down
22 changes: 14 additions & 8 deletions src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Microsoft.EntityFrameworkCore.SqlServer.Internal;
using Microsoft.EntityFrameworkCore.SqlServer.Metadata.Internal;
using Microsoft.EntityFrameworkCore.SqlServer.Update.Internal;
using Microsoft.EntityFrameworkCore.Update.Internal;

// ReSharper disable once CheckNamespace
namespace Microsoft.EntityFrameworkCore.Migrations;
Expand All @@ -33,17 +34,18 @@ public class SqlServerMigrationsSqlGenerator : MigrationsSqlGenerator
private IReadOnlyList<MigrationOperation> _operations = null!;
private int _variableCounter;

private readonly ICommandBatchPreparer _commandBatchPreparer;

/// <summary>
/// Creates a new <see cref="SqlServerMigrationsSqlGenerator" /> instance.
/// </summary>
/// <param name="dependencies">Parameter object containing dependencies for this service.</param>
/// <param name="migrationsAnnotations">Provider-specific Migrations annotations to use.</param>
/// <param name="commandBatchPreparer">The command batch preparer.</param>
public SqlServerMigrationsSqlGenerator(
MigrationsSqlGeneratorDependencies dependencies,
IRelationalAnnotationProvider migrationsAnnotations)
ICommandBatchPreparer commandBatchPreparer)
: base(dependencies)
{
}
=> _commandBatchPreparer = commandBatchPreparer;

/// <summary>
/// Generates commands from a list of operations.
Expand Down Expand Up @@ -1445,10 +1447,14 @@ protected override void Generate(
GenerateIdentityInsert(builder, operation, on: true, model);

var sqlBuilder = new StringBuilder();
((ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator).AppendBulkInsertOperation(
sqlBuilder,
GenerateModificationCommands(operation, model).ToList(),
0);

var modificationCommands = GenerateModificationCommands(operation, model).ToList();
var updateSqlGenerator = (ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator;

foreach (var batch in _commandBatchPreparer.CreateCommandBatches(modificationCommands, moreCommandSets: true))
{
updateSqlGenerator.AppendBulkInsertOperation(sqlBuilder, batch.ModificationCommands, commandPosition: 0);
}

if (Options.HasFlag(MigrationsSqlGenerationOptions.Idempotent))
{
Expand Down
92 changes: 92 additions & 0 deletions test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,85 @@ IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name'
");
}

[ConditionalFact]
public virtual void InsertDataOperation_max_batch_size_is_respected()
{
// The SQL Server max batch size is 42 by default
var values = new object[50, 1];
for (var i = 0; i < 50; i++)
{
values[i, 0] = "Foo" + i;
}

Generate(
CreateGotModel,
new InsertDataOperation
{
Table = "People",
Columns = new[] { "First Name" },
Values = values
});

AssertSql(
@"IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]'))
SET IDENTITY_INSERT [dbo].[People] ON;
INSERT INTO [dbo].[People] ([First Name])
VALUES (N'Foo0'),
(N'Foo1'),
(N'Foo2'),
(N'Foo3'),
(N'Foo4'),
(N'Foo5'),
(N'Foo6'),
(N'Foo7'),
(N'Foo8'),
(N'Foo9'),
(N'Foo10'),
(N'Foo11'),
(N'Foo12'),
(N'Foo13'),
(N'Foo14'),
(N'Foo15'),
(N'Foo16'),
(N'Foo17'),
(N'Foo18'),
(N'Foo19'),
(N'Foo20'),
(N'Foo21'),
(N'Foo22'),
(N'Foo23'),
(N'Foo24'),
(N'Foo25'),
(N'Foo26'),
(N'Foo27'),
(N'Foo28'),
(N'Foo29'),
(N'Foo30'),
(N'Foo31'),
(N'Foo32'),
(N'Foo33'),
(N'Foo34'),
(N'Foo35'),
(N'Foo36'),
(N'Foo37'),
(N'Foo38'),
(N'Foo39'),
(N'Foo40'),
(N'Foo41');
INSERT INTO [dbo].[People] ([First Name])
VALUES (N'Foo42'),
(N'Foo43'),
(N'Foo44'),
(N'Foo45'),
(N'Foo46'),
(N'Foo47'),
(N'Foo48'),
(N'Foo49');
IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]'))
SET IDENTITY_INSERT [dbo].[People] OFF;
");
}

public override void InsertDataOperation_throws_for_unsupported_column_types()
=> base.InsertDataOperation_throws_for_unsupported_column_types();

Expand Down Expand Up @@ -1045,6 +1124,19 @@ public virtual void CreateIndex_generates_exec_when_legacy_filter_and_idempotent
");
}

private static void CreateGotModel(ModelBuilder b)
=> b.HasDefaultSchema("dbo").Entity(
"Person", pb =>
{
pb.ToTable("People");
pb.Property<string>("FirstName").HasColumnName("First Name");
pb.Property<string>("LastName").HasColumnName("Last Name");
pb.Property<string>("Birthplace").HasColumnName("Birthplace");
pb.Property<string>("Allegiance").HasColumnName("House Allegiance");
pb.Property<string>("Culture").HasColumnName("Culture");
pb.HasKey("FirstName", "LastName");
});

public SqlServerMigrationsSqlGeneratorTest()
: base(
SqlServerTestHelpers.Instance,
Expand Down
20 changes: 0 additions & 20 deletions test/EFCore.SqlServer.Tests/Migrations/SqlServerModelDifferTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -752,26 +752,6 @@ public void SeedData_all_operations()
v => Assert.Equal("", v));
}));

[ConditionalFact]
public void SeedData_max_batch_size_is_respected()
=> Execute(
common => common.Entity("Entity").Property<int>("Id"),
source => { },
// The default SQL Server max batch size is 42
target => target.Entity("Entity").HasData(Enumerable.Range(1, 50).Select(i => new { Id = i })),
upOps => Assert.Collection(
upOps,
o =>
{
var m = Assert.IsType<InsertDataOperation>(o);
Assert.Equal(42, m.Values.Length);
},
o =>
{
var m = Assert.IsType<InsertDataOperation>(o);
Assert.Equal(8, m.Values.Length);
}));

[ConditionalFact]
public void Dont_reseed_value_with_value_generated_on_add_property()
=> Execute(
Expand Down

0 comments on commit 5946c60

Please sign in to comment.