From d3d784bf22329cfc4fa77d0dcf8cb13ee7b30ea2 Mon Sep 17 00:00:00 2001 From: maumar Date: Wed, 12 Jul 2023 16:00:13 -0700 Subject: [PATCH] Fix to #30922 - Most cases of projecting a primitive collection don't work Problem was that for cases that compose on the collection of primitives and then project it, we need identifier to properly bucket the results. On SqlServer we can use hidden key column that is a result of OPENJSON - we need something that is unique so we can't use the value itself. Fix is to add identifier to the SelectExpression based on OPENJSON and also modify the logic that figures out if we need key (and thus need to revert to OPENJSON without WITH clause). Also some minor fixes around nullability - when projecting element of a collection of primitives using OUTER APPLY/JOIN we must set the projection binding to nullable, as the value can always be null, in case of empty collection. Fixes #30922 --- .../Query/SqlExpressions/SelectExpression.cs | 62 ++++++- .../SqlServerQueryTranslationPostprocessor.cs | 111 ++++++++---- ...yableMethodTranslatingExpressionVisitor.cs | 19 ++- ...yableMethodTranslatingExpressionVisitor.cs | 9 +- .../PrimitiveCollectionsQueryTestBase.cs | 159 +++++++++++++++--- ...imitiveCollectionsQueryOldSqlServerTest.cs | 27 +++ .../PrimitiveCollectionsQuerySqlServerTest.cs | 101 ++++++++++- .../PrimitiveCollectionsQuerySqliteTest.cs | 59 ++++++- 8 files changed, 480 insertions(+), 67 deletions(-) diff --git a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs index 092e1b225da..b9141e3d631 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs @@ -130,6 +130,51 @@ public SelectExpression( _projectionMapping[new ProjectionMember()] = columnExpression; } + /// + /// Creates a new instance of the class given a , with a single + /// column projection. + /// + /// The name of the column to use as identifier. + /// The type of the column to use as identifier. + /// The type mapping of the column to use as identifier. + /// The table expression. + /// The name of the column to add as the projection. + /// The type of the column to add as the projection. + /// The type mapping of the column to add as the projection. + /// Whether the column projected out is nullable. + public SelectExpression( + TableExpressionBase tableExpression, + string identidierColumnName, + Type identifierColumnType, + RelationalTypeMapping? identifierColumnTypeMapping, + string columnName, + Type columnType, + RelationalTypeMapping? columnTypeMapping, + bool? isColumnNullable = null) + : base(null) + { + var tableReferenceExpression = new TableReferenceExpression(this, tableExpression.Alias!); + AddTable(tableExpression, tableReferenceExpression); + + var identifierColumn = new ConcreteColumnExpression( + identidierColumnName, + tableReferenceExpression, + identifierColumnType.UnwrapNullableType(), + identifierColumnTypeMapping, + identifierColumnType.IsNullableType()); + + var columnExpression = new ConcreteColumnExpression( + columnName, + tableReferenceExpression, + columnType.UnwrapNullableType(), + columnTypeMapping, + isColumnNullable ?? columnType.IsNullableType()); + + _projectionMapping[new ProjectionMember()] = columnExpression; + + _identifier.Add((identifierColumn, identifierColumnTypeMapping!.Comparer)); + } + internal SelectExpression(IEntityType entityType, ISqlExpressionFactory sqlExpressionFactory) : base(null) { @@ -2156,7 +2201,10 @@ public void ApplyIntersect(SelectExpression source2, bool distinct) public void ApplyUnion(SelectExpression source2, bool distinct) => ApplySetOperation(SetOperationType.Union, source2, distinct); - private void ApplySetOperation(SetOperationType setOperationType, SelectExpression select2, bool distinct) + private void ApplySetOperation( + SetOperationType setOperationType, + SelectExpression select2, + bool distinct) { // TODO: Introduce clone method? See issue#24460 var select1 = new SelectExpression( @@ -2205,7 +2253,7 @@ private void ApplySetOperation(SetOperationType setOperationType, SelectExpressi : Array.Empty(); var entityProjectionIdentifiers = new List(); var entityProjectionValueComparers = new List(); - var otherExpressions = new List(); + var otherExpressions = new List<(SqlExpression, RelationalTypeMapping)>(); // Push down into a subquery if limit/offset are defined. If not, any orderings can be discarded as set operations don't preserve // them. @@ -2308,7 +2356,11 @@ private void ApplySetOperation(SetOperationType setOperationType, SelectExpressi } } - otherExpressions.Add(outerProjection); + // we need type mapping for identifiers - it may happen that one side of the set operation comes from collection parameter + // and therefore doesn't have type mapping (yet - we infer those after the translation is complete) + // but for set operation at least one side should have type mapping, otherwise whole thing would have been parameterized out + var outerTypeMapping = innerProjection1.Expression.TypeMapping ?? innerProjection2.Expression.TypeMapping!; + otherExpressions.Add((outerProjection, outerTypeMapping)); } } @@ -2349,10 +2401,10 @@ private void ApplySetOperation(SetOperationType setOperationType, SelectExpressi // If there are no other expressions then we can use all entityProjectionIdentifiers _identifier.AddRange(entityProjectionIdentifiers.Zip(entityProjectionValueComparers)); } - else if (otherExpressions.All(e => e is ColumnExpression)) + else if (otherExpressions.All(e => e.Item1 is ColumnExpression)) { _identifier.AddRange(entityProjectionIdentifiers.Zip(entityProjectionValueComparers)); - _identifier.AddRange(otherExpressions.Select(e => ((ColumnExpression)e, e.TypeMapping!.KeyComparer))); + _identifier.AddRange(otherExpressions.Select(e => ((ColumnExpression)e.Item1, e.Item2.KeyComparer))); } } } diff --git a/src/EFCore.SqlServer/Query/Internal/SqlServerQueryTranslationPostprocessor.cs b/src/EFCore.SqlServer/Query/Internal/SqlServerQueryTranslationPostprocessor.cs index a9a249cb8b1..867f93005ba 100644 --- a/src/EFCore.SqlServer/Query/Internal/SqlServerQueryTranslationPostprocessor.cs +++ b/src/EFCore.SqlServer/Query/Internal/SqlServerQueryTranslationPostprocessor.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics.CodeAnalysis; +using Microsoft.EntityFrameworkCore.Metadata.Internal; using Microsoft.EntityFrameworkCore.Query.SqlExpressions; using Microsoft.EntityFrameworkCore.SqlServer.Internal; @@ -111,30 +112,73 @@ public Expression Process(Expression expression) case ShapedQueryExpression shapedQueryExpression: return shapedQueryExpression.UpdateQueryExpression(Visit(shapedQueryExpression.QueryExpression)); - case SelectExpression + case SelectExpression selectExpression: { - Tables: [SqlServerOpenJsonExpression { ColumnInfos: not null } openJsonExpression, ..], - Orderings: - [ + var newTables = new List(); + var appliedCasts = new List<(SqlServerOpenJsonExpression, string)>(); + + foreach (var table in selectExpression.Tables) + { + if (table is SqlServerOpenJsonExpression { ColumnInfos: not null } + or JoinExpressionBase { Table: SqlServerOpenJsonExpression { ColumnInfos: not null } }) { - Expression: SqlUnaryExpression + var keyReferencedInOrderings = selectExpression.Orderings.Any(o => o.Expression is SqlUnaryExpression + { + OperatorType: ExpressionType.Convert, + Operand: ColumnExpression { Name: "key", Table: var keyColumnTable } + } && keyColumnTable == table); + + var keyReferencedInProjection = selectExpression.Projection.Any(p => p.Expression is ColumnExpression + { Name: "key", Table: var keyColumnTable } && keyColumnTable == table); + + var keyWithConvertReferencedInProjection = selectExpression.Projection.Any(p => p.Expression is SqlUnaryExpression { OperatorType: ExpressionType.Convert, Operand: ColumnExpression { Name: "key", Table: var keyColumnTable } + } && keyColumnTable == table); + + if (keyReferencedInOrderings || keyReferencedInProjection || keyWithConvertReferencedInProjection) + { + // Remove the WITH clause from the OPENJSON expression + var openJsonExpression = (SqlServerOpenJsonExpression)((table as JoinExpressionBase)?.Table ?? table); + var newOpenJsonExpression = openJsonExpression.Update( + openJsonExpression.JsonExpression, + openJsonExpression.Path, + columnInfos: null); + + TableExpressionBase newTable = table switch + { + InnerJoinExpression ij => ij.Update(newOpenJsonExpression, ij.JoinPredicate), + LeftJoinExpression lj => lj.Update(newOpenJsonExpression, lj.JoinPredicate), + CrossJoinExpression cj => cj.Update(newOpenJsonExpression), + CrossApplyExpression ca => ca.Update(newOpenJsonExpression), + OuterApplyExpression oa => oa.Update(newOpenJsonExpression), + _ => newOpenJsonExpression, + }; + + newTables.Add(newTable); + + foreach (var column in openJsonExpression.ColumnInfos!) + { + var typeMapping = _typeMappingSource.FindMapping(column.StoreType); + Check.DebugAssert( + typeMapping is not null, + $"Could not find mapping for store type {column.StoreType} when converting OPENJSON/WITH"); + + _castsToApply.Add((newOpenJsonExpression, column.Name), typeMapping); + appliedCasts.Add((newOpenJsonExpression, column.Name)); + } + } + else + { + newTables.Add(table); } } - ] - } selectExpression - when keyColumnTable == openJsonExpression: - { - // Remove the WITH clause from the OPENJSON expression - var newOpenJsonExpression = openJsonExpression.Update( - openJsonExpression.JsonExpression, - openJsonExpression.Path, - columnInfos: null); - - var newTables = selectExpression.Tables.ToArray(); - newTables[0] = newOpenJsonExpression; + else + { + newTables.Add(table); + } + } var newSelectExpression = selectExpression.Update( selectExpression.Projection, @@ -144,36 +188,37 @@ public Expression Process(Expression expression) selectExpression.Having, selectExpression.Orderings, selectExpression.Limit, - selectExpression.Offset); + selectExpression.Offset); // Record the OPENJSON expression and its projected column(s), along with the store type we just removed from the WITH // clause. Then visit the select expression, adding a cast around the matching ColumnExpressions. // TODO: Need to pass through the type mapping API for converting the JSON value (nvarchar) to the relational store type // (e.g. datetime2), see #30677 - foreach (var column in openJsonExpression.ColumnInfos) - { - var typeMapping = _typeMappingSource.FindMapping(column.StoreType); - Check.DebugAssert( - typeMapping is not null, - $"Could not find mapping for store type {column.StoreType} when converting OPENJSON/WITH"); - - _castsToApply.Add((newOpenJsonExpression, column.Name), typeMapping); - } - var result = base.Visit(newSelectExpression); - foreach (var column in openJsonExpression.ColumnInfos) + foreach (var appliedCast in appliedCasts) { - _castsToApply.Remove((newOpenJsonExpression, column.Name)); + _castsToApply.Remove(appliedCast); } return result; } - case ColumnExpression { Table: SqlServerOpenJsonExpression openJsonTable, Name: var name } columnExpression - when _castsToApply.TryGetValue((openJsonTable, name), out var typeMapping): + case ColumnExpression columnExpression: { - return _sqlExpressionFactory.Convert(columnExpression, columnExpression.Type, typeMapping); + if (columnExpression.Table is SqlServerOpenJsonExpression openJsonTable + && _castsToApply.TryGetValue((openJsonTable, columnExpression.Name), out var typeMapping)) + { + return _sqlExpressionFactory.Convert(columnExpression, columnExpression.Type, typeMapping); + } + + if (columnExpression.Table is JoinExpressionBase { Table: SqlServerOpenJsonExpression innerOpenJsonTable } + && _castsToApply.TryGetValue((innerOpenJsonTable, columnExpression.Name), out var innerTypeMapping)) + { + return _sqlExpressionFactory.Convert(columnExpression, columnExpression.Type, innerTypeMapping); + } + + return base.Visit(expression); } default: diff --git a/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs index 63ea9ea02a1..02385c071ea 100644 --- a/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs @@ -156,7 +156,14 @@ protected override Expression VisitExtension(Expression extensionExpression) var isColumnNullable = elementClrType.IsNullableType(); var selectExpression = new SelectExpression( - openJsonExpression, columnName: "value", columnType: elementClrType, columnTypeMapping: elementTypeMapping, isColumnNullable); + openJsonExpression, + identidierColumnName: "key", + identifierColumnType: typeof(string), + identifierColumnTypeMapping: _typeMappingSource.FindMapping("nvarchar(4000)"), + columnName: "value", + columnType: elementClrType, + columnTypeMapping: elementTypeMapping, + isColumnNullable); // OPENJSON doesn't guarantee the ordering of the elements coming out; when using OPENJSON without WITH, a [key] column is returned // with the JSON array's ordering, which we can ORDER BY; this option doesn't exist with OPENJSON with WITH, unfortunately. @@ -180,7 +187,15 @@ protected override Expression VisitExtension(Expression extensionExpression) _typeMappingSource.FindMapping(typeof(int))), ascending: true)); - var shaperExpression = new ProjectionBindingExpression(selectExpression, new ProjectionMember(), elementClrType); + var shaperExpression = (Expression)new ProjectionBindingExpression(selectExpression, new ProjectionMember(), elementClrType.MakeNullable()); + if (shaperExpression.Type != elementClrType) + { + Check.DebugAssert( + elementClrType.MakeNullable() == shaperExpression.Type, + "expression.Type must be nullable of targetType"); + + shaperExpression = Expression.Convert(shaperExpression, elementClrType); + } return new ShapedQueryExpression(selectExpression, shaperExpression); } diff --git a/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs index 8b1268f397a..18df3cd69e2 100644 --- a/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs @@ -220,7 +220,14 @@ protected override QueryableMethodTranslatingExpressionVisitor CreateSubqueryVis var isColumnNullable = elementClrType.IsNullableType(); var selectExpression = new SelectExpression( - jsonEachExpression, columnName: "value", columnType: elementClrType, columnTypeMapping: elementTypeMapping, isColumnNullable); + jsonEachExpression, + identidierColumnName: "key", + identifierColumnType: typeof(int), + identifierColumnTypeMapping: _typeMappingSource.FindMapping(typeof(int)), + columnName: "value", + columnType: elementClrType, + columnTypeMapping: elementTypeMapping, + isColumnNullable); // TODO: SQLite does have REAL and BLOB types, which JSON does not. Need to possibly cast to that. if (elementTypeMapping is not null) diff --git a/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs index b1d6d437487..f79f61183ba 100644 --- a/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs @@ -33,7 +33,7 @@ public virtual Task Inline_collection_of_nullable_ints_Contains_null(bool async) => AssertQuery( async, ss => ss.Set().Where(c => new int?[] { null, 999 }.Contains(c.NullableInt)), - entryCount: 2); + entryCount: 3); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -58,7 +58,7 @@ public virtual Task Inline_collection_Count_with_two_values(bool async) => AssertQuery( async, ss => ss.Set().Where(c => new[] { 2, 999 }.Count(i => i > c.Id) == 1), - entryCount: 2); + entryCount: 4); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -66,7 +66,7 @@ public virtual Task Inline_collection_Count_with_three_values(bool async) => AssertQuery( async, ss => ss.Set().Where(c => new[] { 2, 999, 1000 }.Count(i => i > c.Id) == 2), - entryCount: 2); + entryCount: 4); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -153,7 +153,7 @@ public virtual Task Inline_collection_negated_Contains_as_All(bool async) => AssertQuery( async, ss => ss.Set().Where(c => new[] { 2, 999 }.All(i => i != c.Id)), - entryCount: 2); + entryCount: 4); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -164,7 +164,7 @@ public virtual Task Parameter_collection_Count(bool async) return AssertQuery( async, ss => ss.Set().Where(c => ids.Count(i => i > c.Id) == 1), - entryCount: 2); + entryCount: 4); } [ConditionalTheory] @@ -200,7 +200,7 @@ public virtual Task Parameter_collection_of_nullable_ints_Contains_nullable_int( return AssertQuery( async, ss => ss.Set().Where(c => nullableInts.Contains(c.NullableInt)), - entryCount: 2); + entryCount: 3); } [ConditionalTheory] @@ -240,7 +240,7 @@ public virtual Task Parameter_collection_of_bools_Contains(bool async) return AssertQuery( async, ss => ss.Set().Where(c => bools.Contains(c.Bool)), - entryCount: 1); + entryCount: 2); } [ConditionalTheory] @@ -252,7 +252,7 @@ public virtual Task Parameter_collection_of_enums_Contains(bool async) return AssertQuery( async, ss => ss.Set().Where(c => enums.Contains(c.Enum)), - entryCount: 2); + entryCount: 3); } [ConditionalTheory] @@ -273,7 +273,7 @@ public virtual Task Column_collection_of_ints_Contains(bool async) => AssertQuery( async, ss => ss.Set().Where(c => c.Ints.Contains(10)), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -281,7 +281,7 @@ public virtual Task Column_collection_of_nullable_ints_Contains(bool async) => AssertQuery( async, ss => ss.Set().Where(c => c.NullableInts.Contains(10)), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -289,7 +289,7 @@ public virtual Task Column_collection_of_nullable_ints_Contains_null(bool async) => AssertQuery( async, ss => ss.Set().Where(c => c.NullableInts.Contains(null)), - entryCount: 1); + entryCount: 3); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -305,7 +305,7 @@ public virtual Task Column_collection_of_bools_Contains(bool async) => AssertQuery( async, ss => ss.Set().Where(c => c.Bools.Contains(true)), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -340,7 +340,7 @@ public virtual Task Column_collection_index_string(bool async) async, ss => ss.Set().Where(c => c.Strings[1] == "10"), ss => ss.Set().Where(c => (c.Strings.Length >= 2 ? c.Strings[1] : "-1") == "10"), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -351,7 +351,7 @@ public virtual Task Column_collection_index_datetime(bool async) c => c.DateTimes[1] == new DateTime(2020, 1, 10, 12, 30, 0, DateTimeKind.Utc)), ss => ss.Set().Where( c => (c.DateTimes.Length >= 2 ? c.DateTimes[1] : default) == new DateTime(2020, 1, 10, 12, 30, 0, DateTimeKind.Utc)), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -443,7 +443,7 @@ public virtual Task Column_collection_OrderByDescending_ElementAt(bool async) .Where(c => c.Ints.OrderByDescending(i => i).ElementAt(0) == 111), ss => ss.Set() .Where(c => c.Ints.Length > 0 && c.Ints.OrderByDescending(i => i).ElementAt(0) == 111), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -451,7 +451,7 @@ public virtual Task Column_collection_Any(bool async) => AssertQuery( async, ss => ss.Set().Where(c => c.Ints.Any()), - entryCount: 2); + entryCount: 4); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -514,7 +514,7 @@ public virtual Task Column_collection_Intersect_inline_collection(bool async) => AssertQuery( async, ss => ss.Set().Where(c => c.Ints.Intersect(new[] { 11, 111 }).Count() == 2), - entryCount: 1); + entryCount: 2); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -525,7 +525,7 @@ public virtual Task Inline_collection_Except_column_collection(bool async) async, ss => ss.Set().Where( c => new[] { 11, 111 }.Except(c.Ints).Count(i => i % 2 == 1) == 2), - entryCount: 2); + entryCount: 3); [ConditionalTheory] [MemberData(nameof(IsAsyncData))] @@ -612,6 +612,18 @@ public virtual async Task Parameter_collection_in_subquery_Union_column_collecti var results = compiledQuery(context, ints).ToList(); } + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Parameter_collection_in_subquery_Union_column_collection(bool async) + { + var ints = new[] { 10, 111 }; + + return AssertQuery( + async, + ss => ss.Set().Where(p => ints.Skip(1).Union(p.Ints).Count() == 3), + entryCount: 4); + } + [ConditionalFact] public virtual void Parameter_collection_in_subquery_and_Convert_as_compiled_query() { @@ -641,9 +653,65 @@ public virtual Task Column_collection_in_subquery_Union_parameter_collection(boo return AssertQuery( async, ss => ss.Set().Where(c => c.Ints.Skip(1).Union(ints).Count() == 3), - entryCount: 1); + entryCount: 2); } + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_collection_of_ints_simple(bool async) + => AssertQuery( + async, + ss => ss.Set().OrderBy(x => x.Id).Select(x => x.Ints), + assertOrder: true, + elementAsserter: (e, a) => AssertCollection(e, a, ordered: true)); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_collection_of_ints_ordered(bool async) + => AssertQuery( + async, + ss => ss.Set().OrderBy(x => x.Id).Select(x => x.Ints.OrderByDescending(xx => xx).ToList()), + assertOrder: true, + elementAsserter: (e, a) => AssertCollection(e, a, ordered: true)); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_empty_collection_of_nullables_and_collection_only_containing_nulls(bool async) + => AssertQuery( + async, + ss => ss.Set().OrderBy(x => x.Id).Select(x => new + { + Empty = x.NullableInts.Where(x => false).ToList(), + OnlyNull = x.NullableInts.Where(x => x == null).ToList(), + }), + assertOrder: true, + elementAsserter: (e, a) => + { + AssertCollection(e.Empty, a.Empty, ordered: true); + AssertCollection(e.OnlyNull, a.OnlyNull, ordered: true); + }); + + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Project_multiple_collections(bool async) + => AssertQuery( + async, + ss => ss.Set().OrderBy(x => x.Id).Select(x => new + { + Ints = x.Ints.ToList(), + OrderedInts = x.Ints.OrderByDescending(xx => xx).ToList(), + FilteredDateTimes = x.DateTimes.Where(xx => xx.Day != 1).ToList(), + FilteredDateTimes2 = x.DateTimes.Where(xx => xx > new DateTime(2000, 1, 1)).ToList() + }), + elementAsserter: (e, a) => + { + AssertCollection(e.Ints, a.Ints, ordered: true); + AssertCollection(e.OrderedInts, a.OrderedInts, ordered: true); + AssertCollection(e.FilteredDateTimes, a.FilteredDateTimes, elementSorter: ee => ee); + AssertCollection(e.FilteredDateTimes2, a.FilteredDateTimes2, elementSorter: ee => ee); + }, + assertOrder: true); + public abstract class PrimitiveCollectionsQueryFixtureBase : SharedStoreFixtureBase, IQueryFixtureBase { private PrimitiveArrayData _expectedData; @@ -802,6 +870,59 @@ private static IReadOnlyList CreatePrimitiveArrayEnt { Id = 3, + Int = 20, + String = "20", + DateTime = new DateTime(2022, 1, 10, 12, 30, 0, DateTimeKind.Utc), + Bool = true, + Enum = MyEnum.Value1, + NullableInt = 20, + + Ints = new[] { 1, 1, 10, 10, 10, 1, 10 }, + Strings = new[] { "1", "10", "10", "1", "1" }, + DateTimes = new DateTime[] + { + new(2020, 1, 1, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 10, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 1, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 1, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 10, 12, 30, 0, DateTimeKind.Utc), + }, + Bools = new[] { true, false }, + Enums = new[] { MyEnum.Value1, MyEnum.Value2 }, + NullableInts = new int?[] { 1, 1, 10, 10, null, 1 }, + }, + new() + { + Id = 4, + + Int = 41, + String = "41", + DateTime = new DateTime(2024, 1, 11, 12, 30, 0, DateTimeKind.Utc), + Bool = false, + Enum = MyEnum.Value2, + NullableInt = null, + + Ints = new[] { 1, 1, 111, 11, 1, 111 }, + Strings = new[] { "1", "11", "111", "11" }, + DateTimes = new DateTime[] + { + new(2020, 1, 1, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 11, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 1, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 11, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 31, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 1, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 31, 12, 30, 0, DateTimeKind.Utc), + new(2020, 1, 31, 12, 30, 0, DateTimeKind.Utc), + }, + Bools = new[] { false }, + Enums = new[] { MyEnum.Value2, MyEnum.Value3 }, + NullableInts = new int?[] { null, null }, + }, + new() + { + Id = 5, + Int = 0, String = "", DateTime = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc), diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs index 4c335a6e0b0..65695e2fdde 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs @@ -469,6 +469,9 @@ public override async Task Column_collection_equality_inline_collection_with_par public override Task Parameter_collection_in_subquery_Union_column_collection_as_compiled_query(bool async) => AssertCompatibilityLevelTooLow(() => base.Parameter_collection_in_subquery_Union_column_collection_as_compiled_query(async)); + public override Task Parameter_collection_in_subquery_Union_column_collection(bool async) + => AssertCompatibilityLevelTooLow(() => base.Parameter_collection_in_subquery_Union_column_collection(async)); + public override void Parameter_collection_in_subquery_and_Convert_as_compiled_query() { // Base implementation asserts that a different exception is thrown @@ -480,6 +483,30 @@ public override Task Parameter_collection_in_subquery_Count_as_compiled_query(bo public override Task Column_collection_in_subquery_Union_parameter_collection(bool async) => AssertCompatibilityLevelTooLow(() => base.Column_collection_in_subquery_Union_parameter_collection(async)); + public override async Task Project_collection_of_ints_simple(bool async) + { + await base.Project_collection_of_ints_simple(async); + + AssertSql( +""" +SELECT [p].[Ints] +FROM [PrimitiveCollectionsEntity] AS [p] +ORDER BY [p].[Id] +"""); + } + + public override Task Project_collection_of_ints_ordered(bool async) + // we don't propagate error details from projection + => AssertTranslationFailed(() => base.Project_collection_of_ints_ordered(async)); + + public override Task Project_multiple_collections(bool async) + // we don't propagate error details from projection + => AssertTranslationFailed(() => base.Project_multiple_collections(async)); + + public override Task Project_empty_collection_of_nullables_and_collection_only_containing_nulls(bool async) + // we don't propagate error details from projection + => AssertTranslationFailed(() => base.Project_empty_collection_of_nullables_and_collection_only_containing_nulls(async)); + [ConditionalFact] public virtual void Check_all_tests_overridden() => TestHelpers.AssertAllMethodsOverridden(GetType()); diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs index b3993a6484d..933f99f5583 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs @@ -588,7 +588,7 @@ FROM [PrimitiveCollectionsEntity] AS [p] WHERE ( SELECT COUNT(*) FROM ( - SELECT 1 AS empty + SELECT [i].[key] FROM OPENJSON([p].[Ints]) AS [i] ORDER BY CAST([i].[key] AS int) OFFSET 1 ROWS @@ -843,7 +843,7 @@ SELECT COUNT(*) FROM ( SELECT [t].[value] FROM ( - SELECT CAST([i].[value] AS int) AS [value] + SELECT CAST([i].[value] AS int) AS [value], [i].[key] FROM OPENJSON(@__ints) AS [i] ORDER BY CAST([i].[key] AS int) OFFSET 1 ROWS @@ -855,6 +855,28 @@ FROM OPENJSON([p].[Ints]) WITH ([value] int '$') AS [i0] """); } + public override async Task Parameter_collection_in_subquery_Union_column_collection(bool async) + { + await base.Parameter_collection_in_subquery_Union_column_collection(async); + + AssertSql( +""" +@__Skip_0='[111]' (Size = 4000) + +SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] +FROM [PrimitiveCollectionsEntity] AS [p] +WHERE ( + SELECT COUNT(*) + FROM ( + SELECT [s].[value] + FROM OPENJSON(@__Skip_0) WITH ([value] int '$') AS [s] + UNION + SELECT [i].[value] + FROM OPENJSON([p].[Ints]) WITH ([value] int '$') AS [i] + ) AS [t]) = 3 +"""); + } + public override void Parameter_collection_in_subquery_and_Convert_as_compiled_query() { base.Parameter_collection_in_subquery_and_Convert_as_compiled_query(); @@ -876,7 +898,7 @@ FROM [PrimitiveCollectionsEntity] AS [p] WHERE ( SELECT COUNT(*) FROM ( - SELECT CAST([i].[value] AS int) AS [value], CAST([i].[key] AS int) AS [c], CAST([i].[value] AS int) AS [value0] + SELECT CAST([i].[value] AS int) AS [value], [i].[key], CAST([i].[key] AS int) AS [c], CAST([i].[value] AS int) AS [value0] FROM OPENJSON(@__ints) AS [i] ORDER BY CAST([i].[key] AS int) OFFSET 1 ROWS @@ -900,7 +922,7 @@ SELECT COUNT(*) FROM ( SELECT [t].[value] FROM ( - SELECT CAST([i].[value] AS int) AS [value] + SELECT CAST([i].[value] AS int) AS [value], [i].[key] FROM OPENJSON([p].[Ints]) AS [i] ORDER BY CAST([i].[key] AS int) OFFSET 1 ROWS @@ -912,6 +934,77 @@ FROM OPENJSON(@__ints_0) WITH ([value] int '$') AS [i0] """); } + public override async Task Project_collection_of_ints_simple(bool async) + { + await base.Project_collection_of_ints_simple(async); + + AssertSql( +""" +SELECT [p].[Ints] +FROM [PrimitiveCollectionsEntity] AS [p] +ORDER BY [p].[Id] +"""); + } + + public override async Task Project_collection_of_ints_ordered(bool async) + { + await base.Project_collection_of_ints_ordered(async); + + AssertSql( +""" +SELECT [p].[Id], CAST([i].[value] AS int) AS [value], [i].[key] +FROM [PrimitiveCollectionsEntity] AS [p] +OUTER APPLY OPENJSON([p].[Ints]) AS [i] +ORDER BY [p].[Id], CAST([i].[value] AS int) DESC +"""); + } + + public override async Task Project_empty_collection_of_nullables_and_collection_only_containing_nulls(bool async) + { + await base.Project_empty_collection_of_nullables_and_collection_only_containing_nulls(async); + + AssertSql( +""" +SELECT [p].[Id], [t].[value], [t].[key], [t0].[value], [t0].[key] +FROM [PrimitiveCollectionsEntity] AS [p] +OUTER APPLY ( + SELECT CAST([n].[value] AS int) AS [value], [n].[key], CAST([n].[key] AS int) AS [c] + FROM OPENJSON([p].[NullableInts]) AS [n] + WHERE 0 = 1 +) AS [t] +OUTER APPLY ( + SELECT CAST([n0].[value] AS int) AS [value], [n0].[key], CAST([n0].[key] AS int) AS [c] + FROM OPENJSON([p].[NullableInts]) AS [n0] + WHERE [n0].[value] IS NULL +) AS [t0] +ORDER BY [p].[Id], [t].[c], [t].[key], [t0].[c] +"""); + } + + public override async Task Project_multiple_collections(bool async) + { + await base.Project_multiple_collections(async); + + AssertSql( +""" +SELECT [p].[Id], CAST([i].[value] AS int) AS [value], [i].[key], CAST([i0].[value] AS int) AS [value], [i0].[key], [t].[value], [t].[key], [t0].[value], [t0].[key] +FROM [PrimitiveCollectionsEntity] AS [p] +OUTER APPLY OPENJSON([p].[Ints]) AS [i] +OUTER APPLY OPENJSON([p].[Ints]) AS [i0] +OUTER APPLY ( + SELECT CAST([d].[value] AS datetime2) AS [value], [d].[key], CAST([d].[key] AS int) AS [c] + FROM OPENJSON([p].[DateTimes]) AS [d] + WHERE DATEPART(day, CAST([d].[value] AS datetime2)) <> 1 +) AS [t] +OUTER APPLY ( + SELECT CAST([d0].[value] AS datetime2) AS [value], [d0].[key], CAST([d0].[key] AS int) AS [c] + FROM OPENJSON([p].[DateTimes]) AS [d0] + WHERE CAST([d0].[value] AS datetime2) > '2000-01-01T00:00:00.0000000' +) AS [t0] +ORDER BY [p].[Id], CAST([i].[key] AS int), [i].[key], CAST([i0].[value] AS int) DESC, [i0].[key], [t].[c], [t].[key], [t0].[c] +"""); + } + [ConditionalFact] public virtual void Check_all_tests_overridden() => TestHelpers.AssertAllMethodsOverridden(GetType()); diff --git a/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs index 5cdad15f18e..7341487c537 100644 --- a/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs +++ b/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore.Sqlite.Internal; namespace Microsoft.EntityFrameworkCore.Query; @@ -575,7 +576,7 @@ public override async Task Column_collection_Skip(bool async) WHERE ( SELECT COUNT(*) FROM ( - SELECT 1 + SELECT "i"."key" FROM json_each("p"."Ints") AS "i" ORDER BY "i"."key" LIMIT -1 OFFSET 1 @@ -849,7 +850,7 @@ SELECT COUNT(*) FROM ( SELECT "t"."value" FROM ( - SELECT "i"."value" + SELECT "i"."value", "i"."key" FROM json_each(@__ints) AS "i" ORDER BY "i"."key" LIMIT -1 OFFSET 1 @@ -861,6 +862,28 @@ FROM json_each("p"."Ints") AS "i0" """); } + public override async Task Parameter_collection_in_subquery_Union_column_collection(bool async) + { + await base.Parameter_collection_in_subquery_Union_column_collection(async); + + AssertSql( +""" +@__Skip_0='[111]' (Size = 5) + +SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" +FROM "PrimitiveCollectionsEntity" AS "p" +WHERE ( + SELECT COUNT(*) + FROM ( + SELECT "s"."value" + FROM json_each(@__Skip_0) AS "s" + UNION + SELECT "i"."value" + FROM json_each("p"."Ints") AS "i" + ) AS "t") = 3 +"""); + } + public override void Parameter_collection_in_subquery_and_Convert_as_compiled_query() { base.Parameter_collection_in_subquery_and_Convert_as_compiled_query(); @@ -883,7 +906,7 @@ SELECT COUNT(*) FROM ( SELECT "t"."value" FROM ( - SELECT "i"."value" + SELECT "i"."value", "i"."key" FROM json_each("p"."Ints") AS "i" ORDER BY "i"."key" LIMIT -1 OFFSET 1 @@ -895,6 +918,36 @@ FROM json_each(@__ints_0) AS "i0" """); } + public override async Task Project_collection_of_ints_simple(bool async) + { + await base.Project_collection_of_ints_simple(async); + + AssertSql( +""" +SELECT "p"."Ints" +FROM "PrimitiveCollectionsEntity" AS "p" +ORDER BY "p"."Id" +"""); + } + + public override async Task Project_collection_of_ints_ordered(bool async) + => Assert.Equal( + SqliteStrings.ApplyNotSupported, + (await Assert.ThrowsAsync( + () => base.Project_collection_of_ints_ordered(async))).Message); + + public override async Task Project_multiple_collections(bool async) + => Assert.Equal( + SqliteStrings.ApplyNotSupported, + (await Assert.ThrowsAsync( + () => base.Project_multiple_collections(async))).Message); + + public override async Task Project_empty_collection_of_nullables_and_collection_only_containing_nulls(bool async) + => Assert.Equal( + SqliteStrings.ApplyNotSupported, + (await Assert.ThrowsAsync( + () => base.Project_empty_collection_of_nullables_and_collection_only_containing_nulls(async))).Message); + [ConditionalFact] public virtual void Check_all_tests_overridden() => TestHelpers.AssertAllMethodsOverridden(GetType());