From fb92c704f77d79b7627a55220a5a9adf7ba3b354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 13:40:31 +0100 Subject: [PATCH 01/11] Add Directory.Build.props and .editorconfig file for Databricks project --- source/Databricks/.editorconfig | 558 ++++++++++++++++++++++++ source/Databricks/Directory.Build.props | 35 ++ 2 files changed, 593 insertions(+) create mode 100644 source/Databricks/.editorconfig create mode 100644 source/Databricks/Directory.Build.props diff --git a/source/Databricks/.editorconfig b/source/Databricks/.editorconfig new file mode 100644 index 000000000..219098cc6 --- /dev/null +++ b/source/Databricks/.editorconfig @@ -0,0 +1,558 @@ +# Copyright 2020 Energinet DataHub A/S +# +# Licensed under the Apache License, Version 2.0 (the "License2"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs + +############################################################## +# Core EditorConfig Options # +############################################################## + +# See https://docs.microsoft.com/en-us/visualstudio/ide/create-portable-custom-editor-options?view=vs-2019#supported-settings + +# top-most EditorConfig file +root = true + +# All files +[*] +end_of_line = lf +indent_style = space + +# Terraform +[*.{tf}] +indent_size = 2 + +# Code files +[*.{cs,csx,vb,vbx,proto,tf}] +indent_size = 4 +insert_final_newline = true +charset = utf-8-bom +# Currently does not work in VS 2019, but works in VS Code. +trim_trailing_whitespace = true + +# XML project files +[*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}] +indent_size = 2 + +# XML config files +[*.{props,targets,ruleset,config,nuspec,resx,vsixmanifest,vsct}] +indent_size = 2 + +# JSON files +[*.json] +indent_size = 2 + +# YAML files +[*.{yml,yaml}] +indent_size = 2 + +# Powershell files +[*.ps1] +indent_size = 2 + +# Shell script files +[*.sh] +end_of_line = lf +indent_size = 2 + +############################################################## +# .NET Conventions # +############################################################## + +[*.{cs,vb}] + +############################### +# .NET Formatting # +############################### + +# See https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-formatting-conventions?view=vs-2019#net-formatting-settings + +# Sort using and Import directives with System.* appearing first +dotnet_sort_system_directives_first = true +# Place a blank line between using directive groups +dotnet_separate_import_directive_groups = false + +############################### +# .NET Code Style # +############################### + +# See https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-language-conventions?view=vs-2019#net-code-style-settings + +# Avoid "this." and "Me." if not necessary +dotnet_style_qualification_for_field = false:error +dotnet_style_qualification_for_property = false:error +dotnet_style_qualification_for_method = false:error +dotnet_style_qualification_for_event = false:error + +# Use language keywords instead of framework type names for type references +dotnet_style_predefined_type_for_locals_parameters_members = true:error +dotnet_style_predefined_type_for_member_access = true:error + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = for_non_interface_members:error +dotnet_style_readonly_field = true:error + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:error +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:error +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:error +dotnet_style_parentheses_in_other_operators = always_for_clarity:error + +# Expression-level preferences (suggest more modern language features when available) +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:error +dotnet_prefer_inferred_tuple_names = true:suggestion +dotnet_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion +dotnet_style_prefer_conditional_expression_over_return = true:suggestion +dotnet_style_prefer_compound_assignment = true:suggestion + +# Null-checking preferences +dotnet_style_coalesce_expression = true:error +dotnet_style_null_propagation = true:error + +############################### +# .NET Code Quality # +############################### + +# See https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-language-conventions?view=vs-2019#net-code-quality-settings + +# Flag methods with any accessibility that contain unused parameters +dotnet_code_quality_unused_parameters = all:suggestion + +############################### +# .NET Naming # +############################### + +# See https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-naming-conventions?view=vs-2019 + +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = error +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +# Instance fields are camelCase and start with _ +dotnet_naming_rule.instance_fields_should_be_camel_case.severity = suggestion +dotnet_naming_rule.instance_fields_should_be_camel_case.symbols = instance_fields +dotnet_naming_rule.instance_fields_should_be_camel_case.style = instance_field_style + +dotnet_naming_symbols.instance_fields.applicable_kinds = field + +dotnet_naming_style.instance_field_style.capitalization = camel_case +dotnet_naming_style.instance_field_style.required_prefix = _ + +############################################################## +# C# Conventions # +############################################################## + +[*.cs] + +############################### +# C# Code Style # +############################### + +# See https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-language-conventions?view=vs-2019#c-code-style-settings + +# Prefer "var" everywhere +csharp_style_var_for_built_in_types = true:suggestion +csharp_style_var_when_type_is_apparent = true:error +csharp_style_var_elsewhere = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:error +csharp_style_pattern_matching_over_as_with_null_check = true:error + +# Inlined variable declarations +csharp_style_inlined_variable_declaration = true:suggestion + +# Expression-level preferences +csharp_prefer_simple_default_expression = true:error + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Code block preferences +csharp_prefer_braces = when_multiline:error +csharp_braces_for_ifelse = required_for_multiline_statement +csharp_style_namespace_declarations = file_scoped:error +dotnet_diagnostic.IDE0161.severity = error # When we move to .NET 9.0 we won't need this anymore: https://learn.microsoft.com/en-us/dotnet/fundamentals/code-analysis/style-rules/language-rules#option-format + +# Unused value preferences +csharp_style_unused_value_expression_statement_preference = discard_variable:silent +csharp_style_unused_value_assignment_preference = discard_variable:suggestion + +# Index and range preferences +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true::silent + +# Misc. preferences +csharp_style_deconstructed_variable_declaration = true:suggestion +csharp_style_pattern_local_over_anonymous_function = true:suggestion +csharp_using_directive_placement = outside_namespace:error +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = false:error +csharp_style_prefer_switch_expression = true:suggestion +csharp_style_expression_bodied_methods = false:none + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion + +############################### +# C# Formatting Rules # +############################### + +# See https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-formatting-conventions?view=vs-2019#c-formatting-settings + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_case_contents = true +csharp_indent_switch_labels = true +csharp_indent_labels = flush_left +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents_when_block = true + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_after_comma = true +csharp_space_before_comma = false +csharp_space_after_dot = false +csharp_space_before_dot = false +csharp_space_after_semicolon_in_for_statement = true +csharp_space_before_semicolon_in_for_statement = false +csharp_space_around_declaration_statements = false +csharp_space_before_open_square_brackets = false +csharp_space_between_empty_square_brackets = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = false +csharp_preserve_single_line_blocks = true + +############################### +# .NET Analyzers # +############################### + +# See full list of rules here: https://github.com/dotnet/roslyn-analyzers/blob/main/src/NetAnalyzers/Core/AnalyzerReleases.Shipped.md + +dotnet_diagnostic.CA1303.severity = suggestion +dotnet_diagnostic.CA1715.severity = error +dotnet_diagnostic.CA1001.severity = warning +dotnet_diagnostic.CA1009.severity = warning +dotnet_diagnostic.CA1016.severity = warning +dotnet_diagnostic.CA1033.severity = warning +dotnet_diagnostic.CA1049.severity = warning +dotnet_diagnostic.CA1060.severity = warning +dotnet_diagnostic.CA1061.severity = warning +dotnet_diagnostic.CA1063.severity = warning +dotnet_diagnostic.CA1065.severity = warning +dotnet_diagnostic.CA1301.severity = warning +dotnet_diagnostic.CA1400.severity = warning +dotnet_diagnostic.CA1401.severity = warning +dotnet_diagnostic.CA1403.severity = warning +dotnet_diagnostic.CA1404.severity = warning +dotnet_diagnostic.CA1405.severity = warning +dotnet_diagnostic.CA1410.severity = warning +dotnet_diagnostic.CA1415.severity = warning +dotnet_diagnostic.CA1812.severity = suggestion +dotnet_diagnostic.CA1821.severity = warning +dotnet_diagnostic.CA1900.severity = warning +dotnet_diagnostic.CA1901.severity = warning +dotnet_diagnostic.CA2002.severity = warning +dotnet_diagnostic.CA2007.severity = error +dotnet_diagnostic.CA2100.severity = warning +dotnet_diagnostic.CA2101.severity = warning +dotnet_diagnostic.CA2108.severity = warning +dotnet_diagnostic.CA2111.severity = warning +dotnet_diagnostic.CA2112.severity = warning +dotnet_diagnostic.CA2114.severity = warning +dotnet_diagnostic.CA2116.severity = warning +dotnet_diagnostic.CA2117.severity = warning +dotnet_diagnostic.CA2122.severity = warning +dotnet_diagnostic.CA2123.severity = warning +dotnet_diagnostic.CA2124.severity = warning +dotnet_diagnostic.CA2126.severity = warning +dotnet_diagnostic.CA2131.severity = warning +dotnet_diagnostic.CA2132.severity = warning +dotnet_diagnostic.CA2133.severity = warning +dotnet_diagnostic.CA2134.severity = warning +dotnet_diagnostic.CA2137.severity = warning +dotnet_diagnostic.CA2138.severity = warning +dotnet_diagnostic.CA2140.severity = warning +dotnet_diagnostic.CA2141.severity = warning +dotnet_diagnostic.CA2146.severity = warning +dotnet_diagnostic.CA2147.severity = warning +dotnet_diagnostic.CA2149.severity = warning +dotnet_diagnostic.CA2200.severity = warning +dotnet_diagnostic.CA2202.severity = warning +dotnet_diagnostic.CA2207.severity = warning +dotnet_diagnostic.CA2212.severity = warning +dotnet_diagnostic.CA2213.severity = warning +dotnet_diagnostic.CA2214.severity = warning +dotnet_diagnostic.CA2216.severity = warning +dotnet_diagnostic.CA2220.severity = warning +dotnet_diagnostic.CA2229.severity = warning +dotnet_diagnostic.CA2231.severity = warning +dotnet_diagnostic.CA2232.severity = warning +dotnet_diagnostic.CA2235.severity = warning +dotnet_diagnostic.CA2236.severity = warning +dotnet_diagnostic.CA2237.severity = warning +dotnet_diagnostic.CA2238.severity = warning +dotnet_diagnostic.CA2240.severity = warning +dotnet_diagnostic.CA2241.severity = warning +dotnet_diagnostic.CA2242.severity = warning +dotnet_diagnostic.CS1591.severity = none + +############################### +# VS Threading Analyzers # +############################### + +# See rules here: https://github.com/microsoft/vs-threading/blob/main/doc/analyzers/index.md + +dotnet_diagnostic.VSTHRD111.severity = error # VSTHRD111: [Similar to CA2007] Use ConfigureAwait(bool) +dotnet_diagnostic.VSTHRD200.severity = error # VSTHRD200: Use "Async" suffix for async methods + +############################### +# StyleCop Analyzers # +############################### + +# See rules here: https://github.com/DotNetAnalyzers/StyleCopAnalyzers/tree/master/documentation + +dotnet_diagnostic.SA0001.severity = none +dotnet_diagnostic.SA1000.severity = error +dotnet_diagnostic.SA1001.severity = error +dotnet_diagnostic.SA1002.severity = error +dotnet_diagnostic.SA1003.severity = error +dotnet_diagnostic.SA1004.severity = error +dotnet_diagnostic.SA1005.severity = suggestion +dotnet_diagnostic.SA1006.severity = error +dotnet_diagnostic.SA1007.severity = error +dotnet_diagnostic.SA1008.severity = error +dotnet_diagnostic.SA1009.severity = error +dotnet_diagnostic.SA1010.severity = error +dotnet_diagnostic.SA1011.severity = none +dotnet_diagnostic.SA1012.severity = error +dotnet_diagnostic.SA1013.severity = error +dotnet_diagnostic.SA1014.severity = error +dotnet_diagnostic.SA1015.severity = error +dotnet_diagnostic.SA1016.severity = error +dotnet_diagnostic.SA1017.severity = error +dotnet_diagnostic.SA1018.severity = error +dotnet_diagnostic.SA1019.severity = error +dotnet_diagnostic.SA1020.severity = error +dotnet_diagnostic.SA1021.severity = error +dotnet_diagnostic.SA1022.severity = error +dotnet_diagnostic.SA1023.severity = error +dotnet_diagnostic.SA1024.severity = error +dotnet_diagnostic.SA1025.severity = error +dotnet_diagnostic.SA1026.severity = error +dotnet_diagnostic.SA1027.severity = error +dotnet_diagnostic.SA1028.severity = error +dotnet_diagnostic.SA1100.severity = error +dotnet_diagnostic.SA1101.severity = none +dotnet_diagnostic.SA1102.severity = error +dotnet_diagnostic.SA1103.severity = error +dotnet_diagnostic.SA1104.severity = error +dotnet_diagnostic.SA1105.severity = error +dotnet_diagnostic.SA1106.severity = error +dotnet_diagnostic.SA1107.severity = error +dotnet_diagnostic.SA1108.severity = error +dotnet_diagnostic.SA1110.severity = error +dotnet_diagnostic.SA1111.severity = error +dotnet_diagnostic.SA1112.severity = error +dotnet_diagnostic.SA1113.severity = error +dotnet_diagnostic.SA1114.severity = error +dotnet_diagnostic.SA1115.severity = error +dotnet_diagnostic.SA1116.severity = error +dotnet_diagnostic.SA1117.severity = error +dotnet_diagnostic.SA1118.severity = error +dotnet_diagnostic.SA1119.severity = error +dotnet_diagnostic.SA1120.severity = error +dotnet_diagnostic.SA1121.severity = error +dotnet_diagnostic.SA1122.severity = error +dotnet_diagnostic.SA1123.severity = error +dotnet_diagnostic.SA1124.severity = none +dotnet_diagnostic.SA1125.severity = error +dotnet_diagnostic.SA1127.severity = error +dotnet_diagnostic.SA1128.severity = error +dotnet_diagnostic.SA1129.severity = error +dotnet_diagnostic.SA1130.severity = error +dotnet_diagnostic.SA1131.severity = none +dotnet_diagnostic.SA1132.severity = error +dotnet_diagnostic.SA1133.severity = error +dotnet_diagnostic.SA1134.severity = error +dotnet_diagnostic.SA1200.severity = none +dotnet_diagnostic.SA1201.severity = none +dotnet_diagnostic.SA1202.severity = error +dotnet_diagnostic.SA1203.severity = error +dotnet_diagnostic.SA1204.severity = none +dotnet_diagnostic.SA1205.severity = error +dotnet_diagnostic.SA1206.severity = error +dotnet_diagnostic.SA1207.severity = error +dotnet_diagnostic.SA1208.severity = error +dotnet_diagnostic.SA1209.severity = error +dotnet_diagnostic.SA1210.severity = error +dotnet_diagnostic.SA1211.severity = error +dotnet_diagnostic.SA1212.severity = error +dotnet_diagnostic.SA1213.severity = error +dotnet_diagnostic.SA1214.severity = error +dotnet_diagnostic.SA1216.severity = error +dotnet_diagnostic.SA1217.severity = error +dotnet_diagnostic.SA1300.severity = error +dotnet_diagnostic.SA1302.severity = error +dotnet_diagnostic.SA1303.severity = error +dotnet_diagnostic.SA1304.severity = error +dotnet_diagnostic.SA1305.severity = error +dotnet_diagnostic.SA1306.severity = error +dotnet_diagnostic.SA1307.severity = error +dotnet_diagnostic.SA1308.severity = error +dotnet_diagnostic.SA1309.severity = none +dotnet_diagnostic.SA1310.severity = error +dotnet_diagnostic.SA1311.severity = error +dotnet_diagnostic.SA1312.severity = error +dotnet_diagnostic.SA1313.severity = error +dotnet_diagnostic.SA1316.severity = error +dotnet_diagnostic.SA1400.severity = error +dotnet_diagnostic.SA1401.severity = error +dotnet_diagnostic.SA1402.severity = error +dotnet_diagnostic.SA1403.severity = error +dotnet_diagnostic.SA1404.severity = error +dotnet_diagnostic.SA1405.severity = error +dotnet_diagnostic.SA1406.severity = error +dotnet_diagnostic.SA1407.severity = error +dotnet_diagnostic.SA1408.severity = error +dotnet_diagnostic.SA1410.severity = error +dotnet_diagnostic.SA1411.severity = error +dotnet_diagnostic.SA1412.severity = error +dotnet_diagnostic.SA1413.severity = error +dotnet_diagnostic.SA1500.severity = error +dotnet_diagnostic.SA1501.severity = none +dotnet_diagnostic.SA1502.severity = none +dotnet_diagnostic.SA1503.severity = none +dotnet_diagnostic.SA1504.severity = error +dotnet_diagnostic.SA1505.severity = error +dotnet_diagnostic.SA1506.severity = error +dotnet_diagnostic.SA1507.severity = error +dotnet_diagnostic.SA1508.severity = error +dotnet_diagnostic.SA1509.severity = error +dotnet_diagnostic.SA1510.severity = error +dotnet_diagnostic.SA1511.severity = error +dotnet_diagnostic.SA1512.severity = error +dotnet_diagnostic.SA1513.severity = error +dotnet_diagnostic.SA1514.severity = error +dotnet_diagnostic.SA1515.severity = suggestion +dotnet_diagnostic.SA1516.severity = error +dotnet_diagnostic.SA1517.severity = error +dotnet_diagnostic.SA1518.severity = error +dotnet_diagnostic.SA1519.severity = error +dotnet_diagnostic.SA1520.severity = error +dotnet_diagnostic.SA1600.severity = suggestion +dotnet_diagnostic.SA1601.severity = error +dotnet_diagnostic.SA1602.severity = error +dotnet_diagnostic.SA1604.severity = error +dotnet_diagnostic.SA1605.severity = error +dotnet_diagnostic.SA1606.severity = error +dotnet_diagnostic.SA1607.severity = error +dotnet_diagnostic.SA1608.severity = error +dotnet_diagnostic.SA1610.severity = error +dotnet_diagnostic.SA1611.severity = none +dotnet_diagnostic.SA1612.severity = error +dotnet_diagnostic.SA1613.severity = error +dotnet_diagnostic.SA1614.severity = none +dotnet_diagnostic.SA1615.severity = none +dotnet_diagnostic.SA1616.severity = error +dotnet_diagnostic.SA1617.severity = none +dotnet_diagnostic.SA1618.severity = none +dotnet_diagnostic.SA1619.severity = error +dotnet_diagnostic.SA1620.severity = error +dotnet_diagnostic.SA1621.severity = error +dotnet_diagnostic.SA1622.severity = error +dotnet_diagnostic.SA1623.severity = none +dotnet_diagnostic.SA1624.severity = error +dotnet_diagnostic.SA1625.severity = error +dotnet_diagnostic.SA1626.severity = error +dotnet_diagnostic.SA1627.severity = error +dotnet_diagnostic.SA1629.severity = none +dotnet_diagnostic.SA1633.severity = error +dotnet_diagnostic.SA1634.severity = none +dotnet_diagnostic.SA1635.severity = none +dotnet_diagnostic.SA1636.severity = none +dotnet_diagnostic.SA1637.severity = none +dotnet_diagnostic.SA1638.severity = none +dotnet_diagnostic.SA1640.severity = none +dotnet_diagnostic.SA1641.severity = none +dotnet_diagnostic.SA1642.severity = none +dotnet_diagnostic.SA1643.severity = none +dotnet_diagnostic.SA1648.severity = error +dotnet_diagnostic.SA1649.severity = none +dotnet_diagnostic.SA1651.severity = error +dotnet_diagnostic.SA1652.severity = none +dotnet_diagnostic.SX1101.severity = error +dotnet_diagnostic.SX1309.severity = error + +############################################################## +# C# Test Conventions (overrides default configuration) # +############################################################## + +# .NET Analyzers (CAxxxx) +# See full list of rules here: https://github.com/dotnet/roslyn-analyzers/blob/main/src/NetAnalyzers/Core/AnalyzerReleases.Shipped.md +# +# VS Threading Analyzers (VSTHRDxxx) +# See rules here: https://github.com/microsoft/vs-threading/blob/main/doc/analyzers/index.md + +# +# The intention of this section is to match c# code files +# that is part of a test project. This includes fixtures and +# code that supports tests. +# +[**{Test,test}**.cs] + +dotnet_diagnostic.CA2007.severity = none # CA2007: Consider calling ConfigureAwait on the awaited task +dotnet_diagnostic.VSTHRD111.severity = none # VSTHRD111: [Similar to CA2007] Use ConfigureAwait(bool) + +# +# The intention of this section is to match c# code files +# that is part of a class that contains tests or scenarios. +# +[*{Tests,Scenario}.cs] + +dotnet_diagnostic.VSTHRD200.severity = none # VSTHRD200: Use "Async" suffix for async methods \ No newline at end of file diff --git a/source/Databricks/Directory.Build.props b/source/Databricks/Directory.Build.props new file mode 100644 index 000000000..e0b0b37d9 --- /dev/null +++ b/source/Databricks/Directory.Build.props @@ -0,0 +1,35 @@ + + + net8.0 + enable + enable + true + true + + true + true + + true + true + latest + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + \ No newline at end of file From 206e07d27be41a189fb0e24639857eb32258b629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 13:40:48 +0100 Subject: [PATCH 02/11] Update target framework and language version --- .../Jobs.IntegrationTests/Jobs.IntegrationTests.csproj | 3 ++- .../Databricks/source/Jobs.UnitTests/Jobs.UnitTests.csproj | 3 ++- source/Databricks/source/Jobs/Jobs.csproj | 6 +++--- .../SqlStatementExecution.IntegrationTests.csproj | 3 ++- .../SqlStatementExecution.UnitTests.csproj | 3 ++- .../SqlStatementExecution/SqlStatementExecution.csproj | 6 +++--- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/Databricks/source/Jobs.IntegrationTests/Jobs.IntegrationTests.csproj b/source/Databricks/source/Jobs.IntegrationTests/Jobs.IntegrationTests.csproj index cdacc8ae4..c64468334 100644 --- a/source/Databricks/source/Jobs.IntegrationTests/Jobs.IntegrationTests.csproj +++ b/source/Databricks/source/Jobs.IntegrationTests/Jobs.IntegrationTests.csproj @@ -17,8 +17,9 @@ limitations under the License. - net7.0 + net8.0 enable + 12 enable true true diff --git a/source/Databricks/source/Jobs.UnitTests/Jobs.UnitTests.csproj b/source/Databricks/source/Jobs.UnitTests/Jobs.UnitTests.csproj index 04b4d74a6..3df2b1355 100644 --- a/source/Databricks/source/Jobs.UnitTests/Jobs.UnitTests.csproj +++ b/source/Databricks/source/Jobs.UnitTests/Jobs.UnitTests.csproj @@ -17,8 +17,9 @@ limitations under the License. - net7.0 + net8.0 enable + 12 enable true true diff --git a/source/Databricks/source/Jobs/Jobs.csproj b/source/Databricks/source/Jobs/Jobs.csproj index 299c21443..ca87b5121 100644 --- a/source/Databricks/source/Jobs/Jobs.csproj +++ b/source/Databricks/source/Jobs/Jobs.csproj @@ -17,12 +17,12 @@ limitations under the License. - net7.0 + net8.0 enable enable true true - 10 + 12 true true Energinet.DataHub.Core.Databricks.Jobs @@ -31,7 +31,7 @@ limitations under the License. Energinet.DataHub.Core.Databricks.Jobs - 9.0.2$(VersionSuffix) + 10.0.0$(VersionSuffix) Databricks Jobs Energinet-DataHub Energinet-DataHub diff --git a/source/Databricks/source/SqlStatementExecution.IntegrationTests/SqlStatementExecution.IntegrationTests.csproj b/source/Databricks/source/SqlStatementExecution.IntegrationTests/SqlStatementExecution.IntegrationTests.csproj index c9c84f66e..b62648dad 100644 --- a/source/Databricks/source/SqlStatementExecution.IntegrationTests/SqlStatementExecution.IntegrationTests.csproj +++ b/source/Databricks/source/SqlStatementExecution.IntegrationTests/SqlStatementExecution.IntegrationTests.csproj @@ -17,13 +17,14 @@ limitations under the License. - net7.0 + net8.0 enable enable true true false true + 12 Energinet.DataHub.Core.Databricks.SqlStatementExecution.IntegrationTests Energinet.DataHub.Core.Databricks.SqlStatementExecution.IntegrationTests diff --git a/source/Databricks/source/SqlStatementExecution.UnitTests/SqlStatementExecution.UnitTests.csproj b/source/Databricks/source/SqlStatementExecution.UnitTests/SqlStatementExecution.UnitTests.csproj index 172cc112a..6921aa87b 100644 --- a/source/Databricks/source/SqlStatementExecution.UnitTests/SqlStatementExecution.UnitTests.csproj +++ b/source/Databricks/source/SqlStatementExecution.UnitTests/SqlStatementExecution.UnitTests.csproj @@ -17,12 +17,13 @@ limitations under the License. - net7.0 + net8.0 enable enable true true false + 12 Energinet.DataHub.Core.Databricks.SqlStatementExecution.UnitTests Energinet.DataHub.Core.Databricks.SqlStatementExecution.UnitTests diff --git a/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj b/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj index 22c8cf0b2..40e3b6f0d 100644 --- a/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj +++ b/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj @@ -17,11 +17,11 @@ limitations under the License. - net7.0 + net8.0 enable true true - 10 + 12 true true Energinet.DataHub.Core.Databricks.SqlStatementExecution @@ -30,7 +30,7 @@ limitations under the License. Energinet.DataHub.Core.Databricks.SqlStatementExecution - 9.0.2$(VersionSuffix) + 10.0.0$(VersionSuffix) Databricks SQL Statement Execution Energinet-DataHub Energinet-DataHub From 6530473ddccf41f897f8b37a1569220255405388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 13:45:00 +0100 Subject: [PATCH 03/11] Remove unused using statements --- .../DatabricksSqlStatementExecutionExtensions.cs | 2 -- .../DatabricksSqlWarehouseQueryExecutor.cs | 4 ---- .../source/SqlStatementExecution/DatabricksStatement.cs | 4 ---- .../SqlStatementExecution/DatabricksStatementBuilder.cs | 2 -- .../HealthChecks/DatabricksSqlStatementApiHealthCheck.cs | 4 ---- ...DatabricksSqlStatementsApiHealthCheckBuilderExtensions.cs | 3 --- .../SqlStatementExecution/Exceptions/DatabricksException.cs | 1 - .../Exceptions/DatabricksSqlException.cs | 2 -- .../SqlStatementExecution/Formats/ApacheArrowFormat.cs | 3 --- .../SqlStatementExecution/Formats/ArrowFieldAttribute.cs | 2 -- .../source/SqlStatementExecution/Formats/Format.cs | 2 -- .../SqlStatementExecution/Formats/IArrowArrayExtensions.cs | 1 - .../source/SqlStatementExecution/Formats/JsonArrayFormat.cs | 3 --- .../SqlStatementExecution/Formats/RecordBatchExtensions.cs | 1 - .../source/SqlStatementExecution/Formats/Reflections.cs | 2 -- .../Formats/StronglyTypedApacheArrowFormat.cs | 3 --- .../source/SqlStatementExecution/HttpClientNameConstants.cs | 2 -- .../source/SqlStatementExecution/IExecuteStrategy.cs | 3 --- .../source/SqlStatementExecution/RawSqlStatement.cs | 1 - .../Statement/DatabricksStatementRequest.cs | 5 ----- 20 files changed, 50 deletions(-) diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs b/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs index cceab7045..1ad6bf5ab 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Net.Http; using System.Net.Http.Headers; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs index b59a54ec1..68ce11e63 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs @@ -12,13 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Collections.Generic; using System.Dynamic; -using System.Net.Http; using System.Net.Http.Json; using System.Runtime.CompilerServices; -using System.Threading; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; using Microsoft.Extensions.Options; diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksStatement.cs b/source/Databricks/source/SqlStatementExecution/DatabricksStatement.cs index 1188e1d5e..c2142fbad 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksStatement.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksStatement.cs @@ -12,10 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Collections.Generic; -using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; - namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution; /// diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksStatementBuilder.cs b/source/Databricks/source/SqlStatementExecution/DatabricksStatementBuilder.cs index 911ce26b1..4aa6736d9 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksStatementBuilder.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksStatementBuilder.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Collections.Generic; - namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution; public sealed class DatabricksStatementBuilder diff --git a/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementApiHealthCheck.cs b/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementApiHealthCheck.cs index 5576e038f..6d0d02087 100644 --- a/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementApiHealthCheck.cs +++ b/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementApiHealthCheck.cs @@ -12,11 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Net.Http; using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; using NodaTime; diff --git a/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementsApiHealthCheckBuilderExtensions.cs b/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementsApiHealthCheckBuilderExtensions.cs index 50f80a36f..4c754e445 100644 --- a/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementsApiHealthCheckBuilderExtensions.cs +++ b/source/Databricks/source/SqlStatementExecution/Diagnostics/HealthChecks/DatabricksSqlStatementsApiHealthCheckBuilderExtensions.cs @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Collections.Generic; -using System.Net.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; diff --git a/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksException.cs b/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksException.cs index 91f261eeb..759f836b9 100644 --- a/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksException.cs +++ b/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksException.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Exceptions; diff --git a/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksSqlException.cs b/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksSqlException.cs index 7a413f82f..9a73de541 100644 --- a/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksSqlException.cs +++ b/source/Databricks/source/SqlStatementExecution/Exceptions/DatabricksSqlException.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; - namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Exceptions; public class DatabricksSqlException : Exception diff --git a/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs b/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs index 2ca611404..06778ff5f 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Collections.Generic; using System.Dynamic; -using System.IO; using System.Runtime.CompilerServices; -using System.Threading; using Apache.Arrow.Ipc; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; diff --git a/source/Databricks/source/SqlStatementExecution/Formats/ArrowFieldAttribute.cs b/source/Databricks/source/SqlStatementExecution/Formats/ArrowFieldAttribute.cs index 805c08f52..74d3e6af6 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/ArrowFieldAttribute.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/ArrowFieldAttribute.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; - namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; [AttributeUsage(AttributeTargets.Property)] diff --git a/source/Databricks/source/SqlStatementExecution/Formats/Format.cs b/source/Databricks/source/SqlStatementExecution/Formats/Format.cs index 703ce63c4..0b788753f 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/Format.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/Format.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; - namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; public class Format diff --git a/source/Databricks/source/SqlStatementExecution/Formats/IArrowArrayExtensions.cs b/source/Databricks/source/SqlStatementExecution/Formats/IArrowArrayExtensions.cs index fb4023afb..6b47c9be7 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/IArrowArrayExtensions.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/IArrowArrayExtensions.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using Apache.Arrow; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; diff --git a/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs b/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs index 937d2d31c..2daf7a5ef 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs @@ -12,12 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Collections.Generic; using System.Dynamic; -using System.IO; using System.Runtime.CompilerServices; using System.Text.Json; -using System.Threading; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; diff --git a/source/Databricks/source/SqlStatementExecution/Formats/RecordBatchExtensions.cs b/source/Databricks/source/SqlStatementExecution/Formats/RecordBatchExtensions.cs index 36d1cbbc6..676289364 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/RecordBatchExtensions.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/RecordBatchExtensions.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Linq; using Apache.Arrow; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; diff --git a/source/Databricks/source/SqlStatementExecution/Formats/Reflections.cs b/source/Databricks/source/SqlStatementExecution/Formats/Reflections.cs index ed7b24695..d21bf8c1e 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/Reflections.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/Reflections.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Linq; using System.Linq.Expressions; using System.Reflection; diff --git a/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs b/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs index a6b80ed5b..9dc3a2597 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs @@ -12,10 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Collections.Generic; -using System.IO; using System.Runtime.CompilerServices; -using System.Threading; using Apache.Arrow.Ipc; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; diff --git a/source/Databricks/source/SqlStatementExecution/HttpClientNameConstants.cs b/source/Databricks/source/SqlStatementExecution/HttpClientNameConstants.cs index 6b70fea42..d62634699 100644 --- a/source/Databricks/source/SqlStatementExecution/HttpClientNameConstants.cs +++ b/source/Databricks/source/SqlStatementExecution/HttpClientNameConstants.cs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Net.Http; - namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution; /// diff --git a/source/Databricks/source/SqlStatementExecution/IExecuteStrategy.cs b/source/Databricks/source/SqlStatementExecution/IExecuteStrategy.cs index a7d940a76..9322e5a8d 100644 --- a/source/Databricks/source/SqlStatementExecution/IExecuteStrategy.cs +++ b/source/Databricks/source/SqlStatementExecution/IExecuteStrategy.cs @@ -12,9 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Collections.Generic; -using System.IO; -using System.Threading; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution; diff --git a/source/Databricks/source/SqlStatementExecution/RawSqlStatement.cs b/source/Databricks/source/SqlStatementExecution/RawSqlStatement.cs index 0b75057d6..e70ba6835 100644 --- a/source/Databricks/source/SqlStatementExecution/RawSqlStatement.cs +++ b/source/Databricks/source/SqlStatementExecution/RawSqlStatement.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System.Collections.Generic; using System.Collections.ObjectModel; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution; diff --git a/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs b/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs index 6a529db30..c900f3997 100644 --- a/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs +++ b/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs @@ -12,13 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.Linq; -using System.Net.Http; using System.Net.Http.Json; using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; using Energinet.DataHub.Core.Databricks.SqlStatementExecution.Exceptions; namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; From 78a47a88d60f8d14f6cb662a4da5dd53e73cc6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 13:47:46 +0100 Subject: [PATCH 04/11] Fix warnings in DatabricksStatementResponse.cs --- .../Statement/DatabricksStatementResponse.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementResponse.cs b/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementResponse.cs index 700fa21e9..9a6c0ee0e 100644 --- a/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementResponse.cs +++ b/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementResponse.cs @@ -15,6 +15,7 @@ // ReSharper disable InconsistentNaming #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. #pragma warning disable SA1300 // Element should begin with upper-case letter. +#pragma warning disable SA1402 // File may only contain a single class. Justification: This is a json mapping class. namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Statement; /// @@ -121,3 +122,4 @@ internal class External_links #pragma warning restore SA1300 #pragma warning restore CS8618 +#pragma warning restore SA1402 From 3415bc33770a105b7efba254f049e73ad325c644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 13:53:28 +0100 Subject: [PATCH 05/11] Fix file-scope namespace and .ConfigureAwait() --- .../Fixtures/HealthChecksFixture.cs | 165 +++++++++--------- .../source/Jobs/Client/JobsApiClient.cs | 55 +++--- .../DatabricksJobsExtensions.cs | 77 ++++---- .../EmbeddedResources.cs | 5 +- .../Fixtures/HealthChecksFixture.cs | 135 +++++++------- ...tabricksSqlStatementExecutionExtensions.cs | 77 ++++---- .../DatabricksSqlWarehouseQueryExecutor.cs | 30 ++-- .../Formats/ApacheArrowFormat.cs | 4 +- .../Formats/StronglyTypedApacheArrowFormat.cs | 11 +- .../Statement/DatabricksStatementRequest.cs | 18 +- 10 files changed, 285 insertions(+), 292 deletions(-) diff --git a/source/Databricks/source/Jobs.IntegrationTests/Fixtures/HealthChecksFixture.cs b/source/Databricks/source/Jobs.IntegrationTests/Fixtures/HealthChecksFixture.cs index 56bb07b15..9ab65de8d 100644 --- a/source/Databricks/source/Jobs.IntegrationTests/Fixtures/HealthChecksFixture.cs +++ b/source/Databricks/source/Jobs.IntegrationTests/Fixtures/HealthChecksFixture.cs @@ -28,93 +28,92 @@ using Moq.Protected; using NodaTime; -namespace Energinet.DataHub.Core.Databricks.Jobs.IntegrationTests.Fixtures +namespace Energinet.DataHub.Core.Databricks.Jobs.IntegrationTests.Fixtures; + +public sealed class HealthChecksFixture : IDisposable { - public sealed class HealthChecksFixture : IDisposable + private readonly TestServer _server; + + public HealthChecksFixture() { - private readonly TestServer _server; - - public HealthChecksFixture() - { - var webHostBuilder = CreateWebHostBuilder(); - _server = new TestServer(webHostBuilder); - HttpClient = _server.CreateClient(); - } - - public HttpClient HttpClient { get; } - - public void Dispose() - { - _server.Dispose(); - } - - private static IWebHostBuilder CreateWebHostBuilder() - { - return new WebHostBuilder() - .ConfigureServices(services => - { - services.AddOptions().Configure(options => - { - options.WarehouseId = "baz"; - options.WorkspaceToken = "bar"; - options.WorkspaceUrl = "https://foo.com"; - options.DatabricksHealthCheckStartHour = 0; - options.DatabricksHealthCheckEndHour = 23; - }); - - services.AddRouting(); - - RegisterHttpClientFactoryMock(services); - RegisterJobsApiClientMock(services); - - services.AddScoped(typeof(IClock), _ => SystemClock.Instance); - - services.AddHealthChecks() - .AddLiveCheck() - .AddDatabricksJobsApiHealthCheck(); - }) - .Configure(app => + var webHostBuilder = CreateWebHostBuilder(); + _server = new TestServer(webHostBuilder); + HttpClient = _server.CreateClient(); + } + + public HttpClient HttpClient { get; } + + public void Dispose() + { + _server.Dispose(); + } + + private static IWebHostBuilder CreateWebHostBuilder() + { + return new WebHostBuilder() + .ConfigureServices(services => + { + services.AddOptions().Configure(options => { - app.UseRouting(); + options.WarehouseId = "baz"; + options.WorkspaceToken = "bar"; + options.WorkspaceUrl = "https://foo.com"; + options.DatabricksHealthCheckStartHour = 0; + options.DatabricksHealthCheckEndHour = 23; + }); + + services.AddRouting(); + + RegisterHttpClientFactoryMock(services); + RegisterJobsApiClientMock(services); + + services.AddScoped(typeof(IClock), _ => SystemClock.Instance); - app.UseEndpoints(endpoints => - { - endpoints.MapLiveHealthChecks(); - endpoints.MapReadyHealthChecks(); - }); + services.AddHealthChecks() + .AddLiveCheck() + .AddDatabricksJobsApiHealthCheck(); + }) + .Configure(app => + { + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapLiveHealthChecks(); + endpoints.MapReadyHealthChecks(); }); - } - - private static void RegisterJobsApiClientMock(IServiceCollection services) - { - var jobsApiClientMock = new Mock(); - jobsApiClientMock.Setup(x => x.Jobs).Returns(new Mock().Object); - services.AddScoped(_ => jobsApiClientMock.Object); - } - - private static void RegisterHttpClientFactoryMock(IServiceCollection services) - { - var httpMessageHandlerMock = new Mock(); - - var response = new HttpResponseMessage { StatusCode = HttpStatusCode.OK }; - - httpMessageHandlerMock - .Protected() - .Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny()) - .ReturnsAsync(response); - - var httpClient = new HttpClient(httpMessageHandlerMock.Object); - services.AddScoped(_ => httpClient); - - var httpClientFactoryMock = new Mock(); - httpClientFactoryMock - .Setup(x => x.CreateClient(Options.DefaultName)) - .Returns(() => httpClient); - - services.AddScoped(_ => httpClientFactoryMock.Object); - } + }); + } + + private static void RegisterJobsApiClientMock(IServiceCollection services) + { + var jobsApiClientMock = new Mock(); + jobsApiClientMock.Setup(x => x.Jobs).Returns(new Mock().Object); + services.AddScoped(_ => jobsApiClientMock.Object); + } + + private static void RegisterHttpClientFactoryMock(IServiceCollection services) + { + var httpMessageHandlerMock = new Mock(); + + var response = new HttpResponseMessage { StatusCode = HttpStatusCode.OK }; + + httpMessageHandlerMock + .Protected() + .Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); + + var httpClient = new HttpClient(httpMessageHandlerMock.Object); + services.AddScoped(_ => httpClient); + + var httpClientFactoryMock = new Mock(); + httpClientFactoryMock + .Setup(x => x.CreateClient(Options.DefaultName)) + .Returns(() => httpClient); + + services.AddScoped(_ => httpClientFactoryMock.Object); } } diff --git a/source/Databricks/source/Jobs/Client/JobsApiClient.cs b/source/Databricks/source/Jobs/Client/JobsApiClient.cs index 85599cff7..e01a2ae84 100644 --- a/source/Databricks/source/Jobs/Client/JobsApiClient.cs +++ b/source/Databricks/source/Jobs/Client/JobsApiClient.cs @@ -16,42 +16,41 @@ using Energinet.DataHub.Core.Databricks.Jobs.Constants; using Microsoft.Azure.Databricks.Client; -namespace Energinet.DataHub.Core.Databricks.Jobs.Client +namespace Energinet.DataHub.Core.Databricks.Jobs.Client; + +/// +/// A databricks client based on the Microsoft.Azure.JobsApiClient, which is using Job API 2.0. +/// The client is extended with a method for reading jobs created using Python Wheels, using Job API 2.1. +/// Because the Job API 2.0 does not support reading python wheel settings. +/// Which is used when we run new jobs and need to know the existing parameters of the job. +/// The code is based on https://github.com/Azure/azure-databricks-client and can be replaced by the official +/// package when support for Job API 2.1 is added. +/// +public sealed class JobsApiClient : IDisposable, IJobsApiClient { /// - /// A databricks client based on the Microsoft.Azure.JobsApiClient, which is using Job API 2.0. - /// The client is extended with a method for reading jobs created using Python Wheels, using Job API 2.1. - /// Because the Job API 2.0 does not support reading python wheel settings. - /// Which is used when we run new jobs and need to know the existing parameters of the job. - /// The code is based on https://github.com/Azure/azure-databricks-client and can be replaced by the official - /// package when support for Job API 2.1 is added. + /// Create Databricks Jobs client object with a Http client. /// - public sealed class JobsApiClient : IDisposable, IJobsApiClient + /// The . + public JobsApiClient(IHttpClientFactory httpClientFactory) { - /// - /// Create Databricks Jobs client object with a Http client. - /// - /// The . - public JobsApiClient(IHttpClientFactory httpClientFactory) - { - var httpClient = httpClientFactory.CreateClient(HttpClientNameConstants.DatabricksJobsApi); - Jobs = new Microsoft.Azure.Databricks.Client.JobsApiClient(httpClient); - } + var httpClient = httpClientFactory.CreateClient(HttpClientNameConstants.DatabricksJobsApi); + Jobs = new Microsoft.Azure.Databricks.Client.JobsApiClient(httpClient); + } - public IJobsApi Jobs { get; } + public IJobsApi Jobs { get; } - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } - private void Dispose(bool disposing) + private void Dispose(bool disposing) + { + if (disposing) { - if (disposing) - { - Jobs.Dispose(); - } + Jobs.Dispose(); } } } diff --git a/source/Databricks/source/Jobs/Extensions/DependencyInjection/DatabricksJobsExtensions.cs b/source/Databricks/source/Jobs/Extensions/DependencyInjection/DatabricksJobsExtensions.cs index 4932aa6ad..98a6752e4 100644 --- a/source/Databricks/source/Jobs/Extensions/DependencyInjection/DatabricksJobsExtensions.cs +++ b/source/Databricks/source/Jobs/Extensions/DependencyInjection/DatabricksJobsExtensions.cs @@ -22,49 +22,48 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -namespace Energinet.DataHub.Core.Databricks.Jobs.Extensions.DependencyInjection +namespace Energinet.DataHub.Core.Databricks.Jobs.Extensions.DependencyInjection; + +public static class DatabricksJobsExtensions { - public static class DatabricksJobsExtensions + /// + /// Adds the and required options to the service collection. + /// + /// IServiceCollection containing elements needed to request Databricks Jobs API. + public static IServiceCollection AddDatabricksJobs(this IServiceCollection serviceCollection, IConfiguration configuration, long timeout = 30) { - /// - /// Adds the and required options to the service collection. - /// - /// IServiceCollection containing elements needed to request Databricks Jobs API. - public static IServiceCollection AddDatabricksJobs(this IServiceCollection serviceCollection, IConfiguration configuration, long timeout = 30) - { - serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); - serviceCollection - .AddOptions() - .Bind(configuration) - .ValidateDataAnnotations() - .Validate( - options => - { - return options.DatabricksHealthCheckStartHour < options.DatabricksHealthCheckEndHour; - }, - "Databricks Jobs Health Check end hour must be greater than start hour."); + serviceCollection + .AddOptions() + .Bind(configuration) + .ValidateDataAnnotations() + .Validate( + options => + { + return options.DatabricksHealthCheckStartHour < options.DatabricksHealthCheckEndHour; + }, + "Databricks Jobs Health Check end hour must be greater than start hour."); - serviceCollection - .AddHttpClient( - HttpClientNameConstants.DatabricksJobsApi, - (services, httpClient) => - { - var options = services.GetRequiredService>().Value; - var url = new Uri(new Uri(options.WorkspaceUrl), "api/"); - httpClient.BaseAddress = url; - httpClient.Timeout = TimeSpan.FromSeconds(timeout); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.WorkspaceToken); - httpClient.DefaultRequestHeaders.Accept.Clear(); - httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); - }) - .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - }); + serviceCollection + .AddHttpClient( + HttpClientNameConstants.DatabricksJobsApi, + (services, httpClient) => + { + var options = services.GetRequiredService>().Value; + var url = new Uri(new Uri(options.WorkspaceUrl), "api/"); + httpClient.BaseAddress = url; + httpClient.Timeout = TimeSpan.FromSeconds(timeout); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.WorkspaceToken); + httpClient.DefaultRequestHeaders.Accept.Clear(); + httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, + }); - return serviceCollection; - } + return serviceCollection; } } diff --git a/source/Databricks/source/SqlStatementExecution.UnitTests/EmbeddedResources.cs b/source/Databricks/source/SqlStatementExecution.UnitTests/EmbeddedResources.cs index e2cc8f5b8..03063f27a 100644 --- a/source/Databricks/source/SqlStatementExecution.UnitTests/EmbeddedResources.cs +++ b/source/Databricks/source/SqlStatementExecution.UnitTests/EmbeddedResources.cs @@ -29,9 +29,8 @@ public static Stream GetStream(string relativeDocumentPath) { var resourceName = $"{typeof(Root).Namespace}.{relativeDocumentPath}"; - var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName); - - if (stream == null) throw new ArgumentException($"Resource '{resourceName}' not found."); + var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(resourceName) + ?? throw new ArgumentException($"Resource '{resourceName}' not found."); return stream; } } diff --git a/source/Databricks/source/SqlStatementExecution.UnitTests/Fixtures/HealthChecksFixture.cs b/source/Databricks/source/SqlStatementExecution.UnitTests/Fixtures/HealthChecksFixture.cs index 855d282dd..26ab1ea49 100644 --- a/source/Databricks/source/SqlStatementExecution.UnitTests/Fixtures/HealthChecksFixture.cs +++ b/source/Databricks/source/SqlStatementExecution.UnitTests/Fixtures/HealthChecksFixture.cs @@ -25,84 +25,83 @@ using Moq.Protected; using NodaTime; -namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.UnitTests.Fixtures +namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.UnitTests.Fixtures; + +public sealed class HealthChecksFixture : IDisposable { - public sealed class HealthChecksFixture : IDisposable + private readonly TestServer _server; + + public HealthChecksFixture() { - private readonly TestServer _server; - - public HealthChecksFixture() - { - var webHostBuilder = CreateWebHostBuilder(); - _server = new TestServer(webHostBuilder); - HttpClient = _server.CreateClient(); - } - - public HttpClient HttpClient { get; } - - public void Dispose() - { - _server.Dispose(); - } - - private static IWebHostBuilder CreateWebHostBuilder() - { - return new WebHostBuilder() - .ConfigureServices(services => - { - services.AddOptions().Configure(options => - { - options.WarehouseId = "baz"; - options.WorkspaceToken = "bar"; - options.WorkspaceUrl = "https://foo.com"; - options.DatabricksHealthCheckStartHour = 0; - options.DatabricksHealthCheckEndHour = 23; - }); - - services.AddRouting(); - services.AddScoped(typeof(IClock), _ => SystemClock.Instance); - - RegisterHttpClientFactoryMock(services); - - services.AddHealthChecks() - .AddLiveCheck() - .AddDatabricksSqlStatementApiHealthCheck(); - }) - .Configure(app => + var webHostBuilder = CreateWebHostBuilder(); + _server = new TestServer(webHostBuilder); + HttpClient = _server.CreateClient(); + } + + public HttpClient HttpClient { get; } + + public void Dispose() + { + _server.Dispose(); + } + + private static IWebHostBuilder CreateWebHostBuilder() + { + return new WebHostBuilder() + .ConfigureServices(services => + { + services.AddOptions().Configure(options => { - app.UseRouting(); + options.WarehouseId = "baz"; + options.WorkspaceToken = "bar"; + options.WorkspaceUrl = "https://foo.com"; + options.DatabricksHealthCheckStartHour = 0; + options.DatabricksHealthCheckEndHour = 23; + }); + + services.AddRouting(); + services.AddScoped(typeof(IClock), _ => SystemClock.Instance); + + RegisterHttpClientFactoryMock(services); - app.UseEndpoints(endpoints => - { - endpoints.MapLiveHealthChecks(); - endpoints.MapReadyHealthChecks(); - }); + services.AddHealthChecks() + .AddLiveCheck() + .AddDatabricksSqlStatementApiHealthCheck(); + }) + .Configure(app => + { + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapLiveHealthChecks(); + endpoints.MapReadyHealthChecks(); }); - } + }); + } - private static void RegisterHttpClientFactoryMock(IServiceCollection services) - { - var httpMessageHandlerMock = new Mock(); + private static void RegisterHttpClientFactoryMock(IServiceCollection services) + { + var httpMessageHandlerMock = new Mock(); - var response = new HttpResponseMessage { StatusCode = HttpStatusCode.OK }; + var response = new HttpResponseMessage { StatusCode = HttpStatusCode.OK }; - httpMessageHandlerMock - .Protected() - .Setup>( - "SendAsync", - ItExpr.IsAny(), - ItExpr.IsAny()) - .ReturnsAsync(response); + httpMessageHandlerMock + .Protected() + .Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync(response); - var httpClient = new HttpClient(httpMessageHandlerMock.Object); - services.AddScoped(_ => httpClient); + var httpClient = new HttpClient(httpMessageHandlerMock.Object); + services.AddScoped(_ => httpClient); - var httpClientFactoryMock = new Mock(); - httpClientFactoryMock - .Setup(x => x.CreateClient(Options.DefaultName)) - .Returns(() => httpClient); + var httpClientFactoryMock = new Mock(); + httpClientFactoryMock + .Setup(x => x.CreateClient(Options.DefaultName)) + .Returns(() => httpClient); - services.AddScoped(_ => httpClientFactoryMock.Object); - } + services.AddScoped(_ => httpClientFactoryMock.Object); } } diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs b/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs index 1ad6bf5ab..57f8c58e0 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksSqlStatementExecutionExtensions.cs @@ -17,51 +17,50 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution +namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution; + +public static class DatabricksSqlStatementExecutionExtensions { - public static class DatabricksSqlStatementExecutionExtensions + /// + /// Adds the and related services to the service collection. + /// + /// IServiceCollection containing elements needed to request Databricks SQL Statement Execution API + public static IServiceCollection AddDatabricksSqlStatementExecution(this IServiceCollection serviceCollection, IConfiguration configuration) { - /// - /// Adds the and related services to the service collection. - /// - /// IServiceCollection containing elements needed to request Databricks SQL Statement Execution API - public static IServiceCollection AddDatabricksSqlStatementExecution(this IServiceCollection serviceCollection, IConfiguration configuration) - { - serviceCollection - .AddOptions() - .Bind(configuration) - .ValidateDataAnnotations() - .Validate( - options => - { - return options.DatabricksHealthCheckStartHour < options.DatabricksHealthCheckEndHour; - }, - "Databricks Jobs Health Check end hour must be greater than start hour."); + serviceCollection + .AddOptions() + .Bind(configuration) + .ValidateDataAnnotations() + .Validate( + options => + { + return options.DatabricksHealthCheckStartHour < options.DatabricksHealthCheckEndHour; + }, + "Databricks Jobs Health Check end hour must be greater than start hour."); - return serviceCollection.AddSqlStatementExecutionInner(); - } + return serviceCollection.AddSqlStatementExecutionInner(); + } - private static IServiceCollection AddSqlStatementExecutionInner(this IServiceCollection serviceCollection) - { - serviceCollection.AddHttpClient( - HttpClientNameConstants.Databricks, - (serviceProvider, client) => - { - var options = serviceProvider.GetRequiredService>().Value; + private static IServiceCollection AddSqlStatementExecutionInner(this IServiceCollection serviceCollection) + { + serviceCollection.AddHttpClient( + HttpClientNameConstants.Databricks, + (serviceProvider, client) => + { + var options = serviceProvider.GetRequiredService>().Value; - client.BaseAddress = new Uri(options.WorkspaceUrl); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.WorkspaceToken); - client.DefaultRequestHeaders.Accept.Clear(); - client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - client.DefaultRequestHeaders.TryAddWithoutValidation("Content-Type", "application/json"); - }); + client.BaseAddress = new Uri(options.WorkspaceUrl); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.WorkspaceToken); + client.DefaultRequestHeaders.Accept.Clear(); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + client.DefaultRequestHeaders.TryAddWithoutValidation("Content-Type", "application/json"); + }); - serviceCollection.AddSingleton(sp => - new DatabricksSqlWarehouseQueryExecutor( - sp.GetRequiredService(), - sp.GetRequiredService>())); + serviceCollection.AddSingleton(sp => + new DatabricksSqlWarehouseQueryExecutor( + sp.GetRequiredService(), + sp.GetRequiredService>())); - return serviceCollection; - } + return serviceCollection; } } diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs index 68ce11e63..b2805d44b 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs @@ -124,7 +124,7 @@ public virtual async IAsyncEnumerable ExecuteStatementAsync( { var strategy = new StronglyTypedApacheArrowFormat(_options); var request = strategy.GetStatementRequest(statement); - var response = await request.WaitForSqlWarehouseResultAsync(_httpClient, StatementsEndpointPath, cancellationToken); + var response = await request.WaitForSqlWarehouseResultAsync(_httpClient, StatementsEndpointPath, cancellationToken).ConfigureAwait(false); if (_httpClient.BaseAddress == null) throw new InvalidOperationException(); @@ -137,17 +137,19 @@ public virtual async IAsyncEnumerable ExecuteStatementAsync( { var uri = StatementsEndpointPath + $"/{response.statement_id}/result/chunks/{chunk.chunk_index}?row_offset={chunk.row_offset}"; - var chunkResponse = await _httpClient.GetFromJsonAsync(uri, cancellationToken); + var chunkResponse = await _httpClient.GetFromJsonAsync(uri, cancellationToken).ConfigureAwait(false); if (chunkResponse?.external_links == null) continue; - await using var stream = await _externalHttpClient.GetStreamAsync( + var stream = await _externalHttpClient.GetStreamAsync( chunkResponse.external_links[0].external_link, - cancellationToken); - - await foreach (var row in strategy.ExecuteAsync(stream, cancellationToken)) + cancellationToken).ConfigureAwait(false); + await using (stream.ConfigureAwait(false)) { - yield return row; + await foreach (var row in strategy.ExecuteAsync(stream, cancellationToken)) + { + yield return row; + } } } } @@ -159,7 +161,7 @@ private async IAsyncEnumerable DoExecuteStatementAsync( { var strategy = format.GetStrategy(_options); var request = strategy.GetStatementRequest(statement); - var response = await request.WaitForSqlWarehouseResultAsync(_httpClient, StatementsEndpointPath, cancellationToken); + var response = await request.WaitForSqlWarehouseResultAsync(_httpClient, StatementsEndpointPath, cancellationToken).ConfigureAwait(false); if (_httpClient.BaseAddress == null) throw new InvalidOperationException(); @@ -172,15 +174,17 @@ private async IAsyncEnumerable DoExecuteStatementAsync( { var uri = StatementsEndpointPath + $"/{response.statement_id}/result/chunks/{chunk.chunk_index}?row_offset={chunk.row_offset}"; - var chunkResponse = await _httpClient.GetFromJsonAsync(uri, cancellationToken); + var chunkResponse = await _httpClient.GetFromJsonAsync(uri, cancellationToken).ConfigureAwait(false); if (chunkResponse?.external_links == null) continue; - await using var stream = await _externalHttpClient.GetStreamAsync(chunkResponse.external_links[0].external_link, cancellationToken); - - await foreach (var row in strategy.ExecuteAsync(stream, response, cancellationToken)) + var stream = await _externalHttpClient.GetStreamAsync(chunkResponse.external_links[0].external_link, cancellationToken).ConfigureAwait(false); + await using (stream.ConfigureAwait(false)) { - yield return row; + await foreach (var row in strategy.ExecuteAsync(stream, response, cancellationToken)) + { + yield return row; + } } } } diff --git a/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs b/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs index 06778ff5f..63b494834 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/ApacheArrowFormat.cs @@ -35,7 +35,7 @@ public async IAsyncEnumerable ExecuteAsync(Stream content, DatabricksSt { using var reader = new ArrowStreamReader(content); - var batch = await reader.ReadNextRecordBatchAsync(cancellationToken); + var batch = await reader.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); while (batch != null) { for (var i = 0; i < batch.Length; i++) @@ -52,7 +52,7 @@ public async IAsyncEnumerable ExecuteAsync(Stream content, DatabricksSt yield return record; } - batch = await reader.ReadNextRecordBatchAsync(cancellationToken); + batch = await reader.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); } } } diff --git a/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs b/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs index 9dc3a2597..d4b427dc4 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/StronglyTypedApacheArrowFormat.cs @@ -17,13 +17,8 @@ namespace Energinet.DataHub.Core.Databricks.SqlStatementExecution.Formats; -internal class StronglyTypedApacheArrowFormat : ApacheArrowFormat +internal class StronglyTypedApacheArrowFormat(DatabricksSqlStatementOptions options) : ApacheArrowFormat(options) { - public StronglyTypedApacheArrowFormat(DatabricksSqlStatementOptions options) - : base(options) - { - } - public async IAsyncEnumerable ExecuteAsync( Stream content, [EnumeratorCancellation] CancellationToken cancellationToken) @@ -31,7 +26,7 @@ public async IAsyncEnumerable ExecuteAsync( { using var reader = new ArrowStreamReader(content); - var batch = await reader.ReadNextRecordBatchAsync(cancellationToken); + var batch = await reader.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); while (batch != null) { for (var i = 0; i < batch.Length; i++) @@ -39,7 +34,7 @@ public async IAsyncEnumerable ExecuteAsync( yield return batch.ReadRecord(i); } - batch = await reader.ReadNextRecordBatchAsync(cancellationToken); + batch = await reader.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); } } } diff --git a/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs b/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs index c900f3997..42fbba2bc 100644 --- a/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs +++ b/source/Databricks/source/SqlStatementExecution/Statement/DatabricksStatementRequest.cs @@ -67,7 +67,7 @@ public async ValueTask WaitForSqlWarehouseResultAsy { try { - response = await GetResponseFromDataWarehouseAsync(client, endpoint, response, cancellationToken); + response = await GetResponseFromDataWarehouseAsync(client, endpoint, response, cancellationToken).ConfigureAwait(false); if (response.IsSucceeded) return response; if (cancellationToken.IsCancellationRequested) @@ -75,13 +75,13 @@ public async ValueTask WaitForSqlWarehouseResultAsy if (string.IsNullOrEmpty(response.statement_id)) throw new InvalidOperationException("The statement_id is missing from databricks response"); // Cancel the statement without cancellation token since it is already cancelled - await CancelStatementAsync(client, endpoint, response.statement_id); + await CancelStatementAsync(client, endpoint, response.statement_id).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); } } catch (TaskCanceledException tce) { - if (!string.IsNullOrEmpty(response?.statement_id)) await CancelStatementAsync(client, endpoint, response.statement_id); + if (!string.IsNullOrEmpty(response?.statement_id)) await CancelStatementAsync(client, endpoint, response.statement_id).ConfigureAwait(false); throw new OperationCanceledException("The operation was cancelled", tce, cancellationToken); } } @@ -100,16 +100,16 @@ private async Task GetResponseFromDataWarehouseAsyn { // No cancellation token is used because we want to wait for the result // With the response we are able to cancel the statement if needed - using var httpResponse = await client.PostAsJsonAsync(endpoint, this); - response = await httpResponse.Content.ReadFromJsonAsync(); + using var httpResponse = await client.PostAsJsonAsync(endpoint, this).ConfigureAwait(false); + response = await httpResponse.Content.ReadFromJsonAsync().ConfigureAwait(false); } else { - await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken).ConfigureAwait(false); var path = $"{endpoint}/{response.statement_id}"; - using var httpResponse = await client.GetAsync(path, cancellationToken); - response = await httpResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); + using var httpResponse = await client.GetAsync(path, cancellationToken).ConfigureAwait(false); + response = await httpResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } if (response == null) @@ -123,6 +123,6 @@ private async Task GetResponseFromDataWarehouseAsyn private static async Task CancelStatementAsync(HttpClient client, string endpoint, string statementId) { var path = $"{endpoint}/{statementId}/cancel"; - using var httpResponse = await client.PostAsync(path, new StringContent(string.Empty)); + using var httpResponse = await client.PostAsync(path, new StringContent(string.Empty)).ConfigureAwait(false); } } From 6f12d2d895273722cbb8276b7e6b27231d719c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 14:03:24 +0100 Subject: [PATCH 06/11] Remove unused packages --- .../source/SqlStatementExecution/SqlStatementExecution.csproj | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj b/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj index 40e3b6f0d..085218b23 100644 --- a/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj +++ b/source/Databricks/source/SqlStatementExecution/SqlStatementExecution.csproj @@ -81,9 +81,7 @@ limitations under the License. all runtime; build; native; contentfiles; analyzers; buildtransitive - - From 07d02fa0168724e58f3ceab4027bbe9305efc320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Fri, 8 Mar 2024 14:06:00 +0100 Subject: [PATCH 07/11] Bump target framework to .NET8 --- source/Databricks/documents/release-notes/release-notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/Databricks/documents/release-notes/release-notes.md b/source/Databricks/documents/release-notes/release-notes.md index 95219aa9f..f7ea8b03c 100644 --- a/source/Databricks/documents/release-notes/release-notes.md +++ b/source/Databricks/documents/release-notes/release-notes.md @@ -1,5 +1,9 @@ # Databricks Release Notes +## Version 10.0.0 + +Bump to .NET 8.0 for TargetFramework. + ## Version 9.0.2 Cancel statement if token cancellation is requested. From b988bc1a42e09c3ad50999966d781a0da0eef71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Mon, 11 Mar 2024 09:01:37 +0100 Subject: [PATCH 08/11] Remove unused rules from .editorconfig --- source/Databricks/.editorconfig | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/source/Databricks/.editorconfig b/source/Databricks/.editorconfig index 219098cc6..509dc6eff 100644 --- a/source/Databricks/.editorconfig +++ b/source/Databricks/.editorconfig @@ -342,14 +342,6 @@ dotnet_diagnostic.CA2241.severity = warning dotnet_diagnostic.CA2242.severity = warning dotnet_diagnostic.CS1591.severity = none -############################### -# VS Threading Analyzers # -############################### - -# See rules here: https://github.com/microsoft/vs-threading/blob/main/doc/analyzers/index.md - -dotnet_diagnostic.VSTHRD111.severity = error # VSTHRD111: [Similar to CA2007] Use ConfigureAwait(bool) -dotnet_diagnostic.VSTHRD200.severity = error # VSTHRD200: Use "Async" suffix for async methods ############################### # StyleCop Analyzers # @@ -536,10 +528,7 @@ dotnet_diagnostic.SX1309.severity = error # .NET Analyzers (CAxxxx) # See full list of rules here: https://github.com/dotnet/roslyn-analyzers/blob/main/src/NetAnalyzers/Core/AnalyzerReleases.Shipped.md # -# VS Threading Analyzers (VSTHRDxxx) -# See rules here: https://github.com/microsoft/vs-threading/blob/main/doc/analyzers/index.md -# # The intention of this section is to match c# code files # that is part of a test project. This includes fixtures and # code that supports tests. @@ -547,12 +536,3 @@ dotnet_diagnostic.SX1309.severity = error [**{Test,test}**.cs] dotnet_diagnostic.CA2007.severity = none # CA2007: Consider calling ConfigureAwait on the awaited task -dotnet_diagnostic.VSTHRD111.severity = none # VSTHRD111: [Similar to CA2007] Use ConfigureAwait(bool) - -# -# The intention of this section is to match c# code files -# that is part of a class that contains tests or scenarios. -# -[*{Tests,Scenario}.cs] - -dotnet_diagnostic.VSTHRD200.severity = none # VSTHRD200: Use "Async" suffix for async methods \ No newline at end of file From 44e495b01b17b5e38724cc76f281200af65fd141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Mon, 11 Mar 2024 09:06:04 +0100 Subject: [PATCH 09/11] Update async/await configuration in DatabricksSqlWarehouseQueryExecutor.cs --- .../DatabricksSqlWarehouseQueryExecutor.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs index b2805d44b..7a91693ed 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs @@ -93,7 +93,7 @@ public virtual async IAsyncEnumerable ExecuteStatementAsync( Format format, [EnumeratorCancellation]CancellationToken cancellationToken = default) { - await foreach (var record in DoExecuteStatementAsync(statement, format, cancellationToken)) + await foreach (var record in DoExecuteStatementAsync(statement, format, cancellationToken).ConfigureAwait(false)) { yield return record; } @@ -181,7 +181,7 @@ private async IAsyncEnumerable DoExecuteStatementAsync( var stream = await _externalHttpClient.GetStreamAsync(chunkResponse.external_links[0].external_link, cancellationToken).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - await foreach (var row in strategy.ExecuteAsync(stream, response, cancellationToken)) + await foreach (var row in strategy.ExecuteAsync(stream, response, cancellationToken).ConfigureAwait(false)) { yield return row; } From 25d5d0a5ba8ef0a9389baa588e2a6d6d48d2802c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Mon, 11 Mar 2024 10:00:00 +0100 Subject: [PATCH 10/11] Add missing ConfigureAwait() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dan Stenrøjl <51327761+dstenroejl@users.noreply.github.com> --- .../DatabricksSqlWarehouseQueryExecutor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs index 7a91693ed..44866607b 100644 --- a/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs +++ b/source/Databricks/source/SqlStatementExecution/DatabricksSqlWarehouseQueryExecutor.cs @@ -146,7 +146,7 @@ public virtual async IAsyncEnumerable ExecuteStatementAsync( cancellationToken).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - await foreach (var row in strategy.ExecuteAsync(stream, cancellationToken)) + await foreach (var row in strategy.ExecuteAsync(stream, cancellationToken).ConfigureAwait(false)) { yield return row; } From 88ac84739f06bb3d53a5474b387bc0e342f2e71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20S=C3=B8ndergaard?= Date: Mon, 11 Mar 2024 10:05:36 +0100 Subject: [PATCH 11/11] Add missing .ConfigureAwait(false) --- .../source/SqlStatementExecution/Formats/JsonArrayFormat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs b/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs index 2daf7a5ef..00a0cc382 100644 --- a/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs +++ b/source/Databricks/source/SqlStatementExecution/Formats/JsonArrayFormat.cs @@ -38,7 +38,7 @@ public async IAsyncEnumerable ExecuteAsync( { await foreach (var record in JsonSerializer.DeserializeAsyncEnumerable( content, - cancellationToken: cancellationToken)) + cancellationToken: cancellationToken).ConfigureAwait(false)) { if (record == null) continue;