Spark Job Definition objects are supported now

Azure-Player · May 23, 2022 · 4860e39 · 4860e39
1 parent 0917d36
commit 4860e39
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -12,13 +12,13 @@ The deployment of these objects:
 - KQL script *
 - SQL script *
 - notebook *
+- Spark job definition *
 
 > \* via RestAPI only
 
 ## What is NOT yet supported
 The deployment of these objects:
 - credential
-- Spark job definition
 - 'AzResource' deployment method 
 - Apache Spark pools (BigDataPool - #11)
 

diff --git a/azure.synapse.tools.psd1 b/azure.synapse.tools.psd1
diff --git a/changelog.md b/changelog.md
@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 Replacing all properties environment-related fails
 
+## [0.20.0] - 2022-05-23
+* Spark Job Definition objects are supported now
+
 ## [0.19.0] - 2022-05-23
 * Pretending that Spark pool exists in order to deploy referring objects (#11)
 

diff --git a/private/!Synapse.class.ps1 b/private/!Synapse.class.ps1
@@ -27,13 +27,14 @@ class Synapse {
     [System.Collections.ArrayList] $KQLScripts = @{}
     [System.Collections.ArrayList] $SQLScripts = @{}
     [System.Collections.ArrayList] $Notebooks = @{}
+    [System.Collections.ArrayList] $SparkJobDefinitions = @{}
     [System.Collections.ArrayList] $SqlPool = @{}
     [string] $Location = ""
     [SynapsePublishOption] $PublishOptions
 
     [System.Collections.ArrayList] AllObjects()
     {
-        return $this.LinkedServices + $this.Pipelines + $this.DataSets + $this.DataFlows + $this.Triggers + $this.SQLScripts + $this.KQLScripts+ $this.Notebooks + $this.IntegrationRuntimes + $this.ManagedVirtualNetwork + $this.ManagedPrivateEndpoints
+        return $this.LinkedServices + $this.Pipelines + $this.DataSets + $this.DataFlows + $this.Triggers + $this.SQLScripts + $this.KQLScripts+ $this.Notebooks + $this.SparkJobDefinitions + $this.IntegrationRuntimes + $this.ManagedVirtualNetwork + $this.ManagedPrivateEndpoints
     }
 
     [hashtable] GetObjectsByFullName([string]$pattern)

diff --git a/private/!SynapseObject.class.ps1 b/private/!SynapseObject.class.ps1
@@ -83,7 +83,7 @@ class SynapseObject {
         return $ofn
     }
 
-    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'kqlscript', 'sqlscript', 'notebook', 'managedVirtualNetwork', 'managedPrivateEndpoint', 'sqlpool', 'BigDataPool')
+    static $AllowedTypes = @('integrationRuntime', 'pipeline', 'dataset', 'dataflow', 'linkedService', 'trigger', 'kqlscript', 'sqlscript', 'notebook', 'managedVirtualNetwork', 'managedPrivateEndpoint', 'sqlpool', 'BigDataPool', 'SparkJobDefinition')
 
     static AssertType ([string] $Type)
     {

diff --git a/private/Deploy-SynapseObjectOnly.ps1 b/private/Deploy-SynapseObjectOnly.ps1
@@ -29,7 +29,7 @@ function Deploy-SynapseObjectOnly {
     $json = $body | ConvertFrom-Json
 
     if ($script:PublishMethod -eq "AzResource") { $type = "AzResource" }
-    if ($obj.Type -in ('notebook', 'sqlscript', 'kqlscript')) { 
+    if ($obj.Type -in ('notebook', 'sqlscript', 'kqlscript', 'sparkJobDefinition')) { 
         $type = $obj.Type 
         Write-Warning "$($obj.Type)s are being deployed by Rest-API regardless of PublishMethod."
     }
@@ -152,6 +152,13 @@ function Deploy-SynapseObjectOnly {
             $r = Invoke-RestMethod -Method PUT -Uri $uri -Body $body -Headers $h
             Wait-CompleteOperation -SynapseWorkspaceName $SynapseWorkspaceName -requestHeader $h -operationId $r.operationId -operation 'notebookOperationResults' | Out-Null
         }
+        'sparkJobDefinition'
+        {
+            $h = Get-RequestHeader
+            $uri = "https://$SynapseWorkspaceName.dev.azuresynapse.net/sparkJobDefinitions/$($obj.Name)?api-version=2020-12-01"
+            $r = Invoke-RestMethod -Method PUT -Uri $uri -Body $body -Headers $h
+            Wait-CompleteOperation -SynapseWorkspaceName $SynapseWorkspaceName -requestHeader $h -operationId $r.operationId -operation 'operationResults' | Out-Null
+        }
         'AzResource'
         {
             $resType = Get-AzureResourceType $obj.Type

diff --git a/private/Get-AzureResourceType.ps1 b/private/Get-AzureResourceType.ps1
@@ -19,6 +19,7 @@ function Get-AzureResourceType {
         'trigger'               { $resType = 'Microsoft.Synapse/workspaces/triggers' }
         'sqlscript'             { $resType = 'Microsoft.Synapse/workspaces/sqlscripts' }
         'notebook'              { $resType = 'Microsoft.Synapse/workspaces/notebooks' }
+        'sparkJobDefinition'    { $resType = 'Microsoft.Synapse/workspaces/sparkJobDefinitions' }
         'managedVirtualNetwork' { $resType = 'Microsoft.Synapse/workspaces/managedVirtualNetworks' }
         'managedVirtualNetwork\default\managedPrivateEndpoint' { $resType = 'Microsoft.Synapse/workspaces/managedVirtualNetworks/managedPrivateEndpoints' }
         default                 { Write-Error "ASWT0016: Type '$Type' is not supported." }

diff --git a/public/Import-SynapseFromFolder.ps1 b/public/Import-SynapseFromFolder.ps1
@@ -57,6 +57,8 @@ function Import-SynapseFromFolder {
     Write-Host ("KqlScripts: {0} object(s) loaded." -f $synapse.KqlScripts.Count)
     Import-SynapseObjects -Synapse $synapse -All $synapse.Notebooks -RootFolder $RootFolder -SubFolder "notebook" | Out-Null
     Write-Host ("Notebooks: {0} object(s) loaded." -f $synapse.Notebooks.Count)
+    Import-SynapseObjects -Synapse $synapse -All $synapse.SparkJobDefinitions -RootFolder $RootFolder -SubFolder "sparkJobDefinition" | Out-Null
+    Write-Host ("SparkJobDefinitions: {0} object(s) loaded." -f $synapse.SparkJobDefinitions.Count)
     Import-SynapseObjects -Synapse $synapse -All $synapse.ManagedVirtualNetwork -RootFolder $RootFolder -SubFolder "managedVirtualNetwork" | Out-Null
     Write-Host ("Managed VNet: {0} object(s) loaded." -f $synapse.ManagedVirtualNetwork.Count)
     Import-SynapseObjects -Synapse $synapse -All $synapse.ManagedPrivateEndpoints -RootFolder $RootFolder -SubFolder "managedVirtualNetwork\default\managedPrivateEndpoint" | Out-Null