-
Notifications
You must be signed in to change notification settings - Fork 0
/
Storage-account-data-copy-setup.template.json
188 lines (185 loc) · 9.2 KB
/
Storage-account-data-copy-setup.template.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
{
"contentVersion": "1.0.0.0",
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"parameters": {
},
"variables": {
"dataFactoryName": "[concat('DataCopy',uniquestring(resourceGroup().id))]",
"containerName": "rspark",
"storageAccountForDataset": "dataforadfmrsdemo",
"publicFolderPath": "demofiles/",
"storageAccountName": "[concat('stgact',uniquestring(resourceGroup().id))]",
"apiVersion": "2015-10-01",
"storageLinkedServiceName": "AzureStorageLinkedService",
"publicStorageLinkedServiceName": "PublicDataSetAzureStorageLinkedService",
"blobInputPublicDataset": "AzureBlobInputPublic",
"blobOutputDataset": "AzureBlobOutput",
"defaultApiVersion": "2015-05-01-preview",
"clusterApiVersion": "2015-03-01-preview",
"publicStorageAccountSasURI": "https://dataforadfmrsdemo.blob.core.windows.net/demofiles?st=2016-08-23T18%3A04%3A00Z&se=2020-08-24T18%3A04%3A00Z&sp=rl&sv=2015-04-05&sr=c&sig=ZlGFdS5SusdmPGVsVGtAP9q8OhinwFidhZObdqi6y0M%3D"
},
"resources": [
{
"name": "[variables('storageAccountName')]",
"type": "Microsoft.Storage/storageAccounts",
"location": "[resourceGroup().location]",
"apiVersion": "[variables('defaultApiVersion')]",
"dependsOn": [],
"tags": {},
"properties": {
"accountType": "Standard_LRS"
}
},
{
"name": "[variables('dataFactoryName')]",
"apiVersion": "[variables('apiVersion')]",
"type": "Microsoft.DataFactory/datafactories",
"location": "westus",
"resources": [
{
"dependsOn": [
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'))]"
],
"type": "linkedservices",
"name": "[variables('storageLinkedServiceName')]",
"apiVersion": "[variables('apiVersion')]",
"properties": {
"type": "AzureStorage",
"typeProperties": {
"connectionString": "[concat('DefaultEndpointsProtocol=https;AccountName=',variables('storageAccountName'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('defaultApiVersion')).key1)]"
}
}
},
{
"dependsOn": [
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'))]"
],
"type": "linkedservices",
"name": "[variables('publicStorageLinkedServiceName')]",
"apiVersion": "[variables('apiVersion')]",
"properties": {
"type": "AzureStorageSas",
"typeProperties": {
"sasUri": "[variables('publicStorageAccountSasURI')]"
}
}
},
{
"dependsOn": [
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'))]",
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'), '/linkedServices/', variables('publicStorageLinkedServiceName'))]"
],
"type": "datasets",
"name": "[variables('blobInputPublicDataset')]",
"apiVersion": "[variables('apiVersion')]",
"properties": {
"type": "AzureBlob",
"linkedServiceName": "[variables('publicStorageLinkedServiceName')]",
"typeProperties": {
"folderPath": "[variables('publicFolderPath')]"
},
"availability": {
"frequency": "Month",
"interval": 1
},
"external": true,
"policy": {}
}
},
{
"dependsOn": [
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'))]",
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'), '/linkedServices/', variables('storageLinkedServiceName'))]"
],
"type": "datasets",
"name": "[variables('blobOutputDataset')]",
"apiVersion": "[variables('apiVersion')]",
"properties": {
"published": false,
"type": "AzureBlob",
"linkedServiceName": "[variables('storageLinkedServiceName')]",
"typeProperties": {
"folderPath": "[concat(variables('containerName'),'/')]"
},
"availability": {
"frequency": "Month",
"interval": 1
}
}
},
{
"dependsOn": [
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'))]",
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'), '/linkedServices/', variables('storageLinkedServiceName'))]",
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'), '/datasets/', variables('blobInputPublicDataset'))]",
"[concat('Microsoft.DataFactory/dataFactories/', variables('dataFactoryName'), '/datasets/', variables('blobOutputDataset'))]"
],
"type": "datapipelines",
"name": "[variables('dataFactoryName')]",
"apiVersion": "[variables('apiVersion')]",
"properties": {
"description": "My first Azure Data Factory pipeline",
"activities": [
{
"type": "Copy",
"typeProperties": {
"source": {
"type": "BlobSource",
"recursive": true
},
"sink": {
"type": "BlobSink",
"copyBehavior": "PreserveHierarchy",
"writeBatchSize": 0,
"writeBatchTimeout": "00:00:00"
}
},
"inputs": [
{
"name": "[variables('blobInputPublicDataset')]"
}
],
"outputs": [
{
"name": "[variables('blobOutputDataset')]"
}
],
"policy": {
"timeout": "02:00:00",
"concurrency": 1,
"executionPriorityOrder": "NewestFirst",
"style": "StartOfInterval",
"retry": 3,
"longRetry": 0,
"longRetryInterval": "00:00:00"
},
"scheduler": {
"frequency": "Month",
"interval": 1
},
"name": "CopyActivity-0"
}
],
"start": "2016-04-01T00:00:00Z",
"end": "2016-04-02T00:00:00Z",
"isPaused": false
}
}
]
}
],
"outputs": {
"parameterStorageAccountName": {
"type": "string",
"value": "[variables('storageAccountName')]"
},
"paramerterStorageAccountKey": {
"type": "string",
"value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), variables('defaultApiVersion')).key1]"
},
"parameterStorageContainerName": {
"type": "string",
"value": "[variables('containerName')]"
}
}
}