Skip to content

Commit

Permalink
15070 #Azure File Copy Task 4.*: Update to AzCopy 10.11 to support cu… (
Browse files Browse the repository at this point in the history 
#15594)

* 15070 #Azure File Copy Task 4.*: Update to AzCopy 10.11 to support custom file extension to content-type mapping

* AzureFileCopy V4 CleanTarget #14759 Azure File Copy Task 4.*: Update to AzCopy 10.11 to support custom file extension to content-type mappings #15070

* updated readme file

* Updated readme

* AzureFileCopyv5 PR review changes

Co-authored-by: Philipson Joseph V <77002750+PhilipsonJoseph@users.noreply.github.com>
Co-authored-by: v-nagarajku <v-nagarajku@microsoft.com>
Co-authored-by: v-nagarajku <96724708+v-nagarajku@users.noreply.github.com>
  • Loading branch information
@v-jkarri @PhilipsonJoseph
@nagarajku @v-nagarajku
4 people authored Feb 3, 2022
1 parent 78f1a9d commit bd84d4a
Show file tree
Hide file tree
Showing 56 changed files with 8,321 additions and 0 deletions.
268 changes: 268 additions & 0 deletions Tasks/AzureFileCopyV5/AzureFileCopy.ps1
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
[CmdletBinding()]
param()

Trace-VstsEnteringInvocation $MyInvocation

# Get inputs for the task
$sourcePath = Get-VstsInput -Name SourcePath -Require
$destination = Get-VstsInput -Name Destination -Require
$connectedServiceName = Get-VstsInput -Name ConnectedServiceNameARM -Require
$storageAccount = Get-VstsInput -Name StorageAccountRM
$containerName = Get-VstsInput -Name ContainerName
$blobPrefix = Get-VstsInput -Name BlobPrefix
$environmentName = Get-VstsInput -Name EnvironmentNameRM
$resourceFilteringMethod = Get-VstsInput -Name ResourceFilteringMethod
$machineNames = Get-VstsInput -Name MachineNames
$vmsAdminUserName = Get-VstsInput -Name VmsAdminUsername
$vmsAdminPassword = Get-VstsInput -Name VmsAdminPassword
$targetPath = Get-VstsInput -Name TargetPath
$additionalArgumentsForBlobCopy = Get-VstsInput -Name AdditionalArgumentsForBlobCopy
$additionalArgumentsForVMCopy = Get-VstsInput -Name AdditionalArgumentsForVMCopy
$cleanTargetBeforeCopy = Get-VstsInput -Name CleanTargetBeforeCopy -AsBool
$copyFilesInParallel = Get-VstsInput -Name CopyFilesInParallel -AsBool
$skipCACheck = Get-VstsInput -Name SkipCACheck -AsBool
$enableCopyPrerequisites = Get-VstsInput -Name EnableCopyPrerequisites -AsBool

$sasTokenTimeOutInMinutes = 240

if ($destination -eq "AzureBlob"){
$userGivenTimeOutInMinutes = Get-VstsInput -Name SasTokenTimeOutInMinutes
if($userGivenTimeOutInMinutes -ne ""){
$sasTokenTimeOutInMinutes = $userGivenTimeOutInMinutes
}
}

if ($destination -ne "AzureBlob")
{
$blobPrefix = ""
}

# Constants
$useHttpsProtocolOption = ''
$ErrorActionPreference = 'Stop'
$telemetrySet = $false
$isPremiumStorage = $false

$sourcePath = $sourcePath.Trim('"')
$storageAccount = $storageAccount.Trim()
$containerName = $containerName.Trim().ToLower()

$additionalArgumentsForBlobCopy = $additionalArgumentsForBlobCopy.Trim()
$additionalArgumentsForVMCopy = $additionalArgumentsForVMCopy.Trim()
$useDefaultArgumentsForBlobCopy = ($additionalArgumentsForBlobCopy -eq "")

# azcopy location on automation agent
$azCopyExeLocation = 'AzCopy\AzCopy.exe'
$azCopyLocation = [System.IO.Path]::GetDirectoryName($azCopyExeLocation)

# Import RemoteDeployer
Import-Module $PSScriptRoot\ps_modules\RemoteDeployer

# Initialize Azure.
Import-Module $PSScriptRoot\ps_modules\VstsAzureHelpers_

$endpoint = Get-VstsEndpoint -Name $connectedServiceName -Require

# Update PSModulePath for hosted agent
. "$PSScriptRoot\Utility.ps1"
CleanUp-PSModulePathForHostedAgent

if (Get-Module Az.Accounts -ListAvailable){
Initialize-AzModule -Endpoint $endpoint
}
else{
Write-Verbose "No module found with name: Az.Accounts"
throw ("Could not find the module Az.Accounts with given version. If the module was recently installed, retry after restarting the Azure Pipelines task agent.")
}

# Import the loc strings.
Import-VstsLocStrings -LiteralPath $PSScriptRoot/Task.json

# Load all dependent files for execution
. "$PSScriptRoot\AzureFileCopyRemoteJob.ps1"

# Enabling detailed logging only when system.debug is true
$enableDetailedLogging = ($env:system_debug -eq "true")

# Telemetry
Import-Module $PSScriptRoot\ps_modules\TelemetryHelper

#### MAIN EXECUTION OF AZURE FILE COPY TASK BEGINS HERE ####
try {
try
{
# Importing required version of azure cmdlets according to azureps installed on machine
$azureUtility = Get-AzureUtility

Write-Verbose -Verbose "Loading $azureUtility"
. "$PSScriptRoot/$azureUtility"

# Telemetry for endpoint id
$telemetryJsonContent = "{`"endpointId`":`"$connectedServiceName`"}"
Write-Host "##vso[telemetry.publish area=TaskEndpointId;feature=AzureFileCopy]$telemetryJsonContent"

# Getting storage key for the storage account
$storageKey = Get-StorageKey -storageAccountName $storageAccount -endpoint $endpoint

# creating storage context to be used while creating container, sas token, deleting container
$storageContext = Create-AzureStorageContext -StorageAccountName $storageAccount -StorageAccountKey $storageKey

# Geting Azure Storage Account type
$storageAccountType = Get-StorageAccountType -storageAccountName $storageAccount -endpoint $endpoint
Write-Verbose "Obtained Storage Account type: $storageAccountType"
if(-not [string]::IsNullOrEmpty($storageAccountType) -and $storageAccountType.Contains('Premium'))
{
$isPremiumStorage = $true
}

# creating temporary container for uploading files if no input is provided for container name
if([string]::IsNullOrEmpty($containerName) -or ($destination -ne "AzureBlob"))
{
$containerName = [guid]::NewGuid().ToString()
Write-Verbose "Container Name input not found. Creating Temporary container for uploading files."
Create-AzureContainer -containerName $containerName -storageContext $storageContext
}
else
{
#checking if the containerName provided exist or not
$containerPresent = Get-AzureContainer -containerName $containerName -storageContext $storageContext

#creating container if the containerName provided does not exist
if($containerPresent -eq $null)
{
Write-Verbose "Creating container if the containerName provided does not exist"
Create-AzureContainer -containerName $containerName -storageContext $storageContext
}
}


# Getting Azure Blob Storage Endpoint
$blobStorageEndpoint = Get-blobStorageEndpoint -storageAccountName $storageAccount -endpoint $endpoint

# Setting environment variable for tracking Azure Pipelines usage in AzCopy telemetry
$env:AZCOPY_USER_AGENT_PREFIX = "TFS_useragent"
}
catch
{
Write-Verbose $_.Exception.ToString()
Write-Telemetry "Task_InternalError" "TemporaryCopyingToBlobContainerFailed"
throw
}

# Set optional arguments for azcopy blob upload
if ($useDefaultArgumentsForBlobCopy)
{
# Adding default optional arguments:
# log-level: Defines the log verbosity for the log file. Default is INFO(all requests/responses)

Write-Verbose "Using default AzCopy arguments for uploading to blob storage"

$additionalArgumentsForBlobCopy = "--log-level=INFO"

# Add more arguments if required

# Premium storage accounts only support page blobs
if($isPremiumStorage)
{
Write-Verbose "Setting BlobType to page for Premium Storage account."
$additionalArgumentsForBlobCopy += " --blob-type=PageBlob"
}

# $root container does not support sub folders. So excluding recursive copy option for $root container.
if($containerName -ne '$root')
{
Write-Verbose "Adding argument for recursive copy"
$additionalArgumentsForBlobCopy += " --recursive"
}
}

Check-ContainerNameAndArgs -containerName $containerName -additionalArguments $additionalArgumentsForBlobCopy

# Uploading files to container
Upload-FilesToAzureContainer -sourcePath $sourcePath `
-endPoint $endpoint `
-storageAccountName $storageAccount `
-containerName $containerName `
-blobPrefix $blobPrefix `
-blobStorageEndpoint $blobStorageEndpoint `
-azCopyLocation $azCopyLocation `
-additionalArguments $additionalArgumentsForBlobCopy `
-destinationType $destination `
-useDefaultArguments $useDefaultArgumentsForBlobCopy `
-cleanTargetBeforeCopy $cleanTargetBeforeCopy `

# Complete the task if destination is azure blob
if ($destination -eq "AzureBlob")
{
# Get URI and SaSToken for output variable
$storageAccountContainerURI = $storageContext.BlobEndPoint + $containerName + "/"
Write-Host "##vso[task.setvariable variable=StorageContainerUri]$storageAccountContainerURI"


$storageContainerSaSToken = Generate-AzureStorageContainerSASToken -containerName $containerName -storageContext $storageContext -tokenTimeOutInMinutes $sasTokenTimeOutInMinutes
Write-Host "##vso[task.setvariable variable=StorageContainerSasToken]$storageContainerSasToken"

Remove-EndpointSecrets
Write-Verbose "Completed Azure File Copy Task for Azure Blob Destination"

return
}

# Copying files to Azure VMs
try
{
# Normalize admin username
if($vmsAdminUserName -and (-not $vmsAdminUserName.StartsWith(".\")) -and ($vmsAdminUserName.IndexOf("\") -eq -1) -and ($vmsAdminUserName.IndexOf("@") -eq -1))
{
$vmsAdminUserName = ".\" + $vmsAdminUserName
}
# getting azure vms properties(name, fqdn, winrmhttps port)
$azureVMResourcesProperties = Get-AzureVMResourcesProperties -resourceGroupName $environmentName `
-resourceFilteringMethod $resourceFilteringMethod -machineNames $machineNames -enableCopyPrerequisites $enableCopyPrerequisites -connectedServiceName $connectedServiceName

$azureVMsCredentials = Get-AzureVMsCredentials -vmsAdminUserName $vmsAdminUserName -vmsAdminPassword $vmsAdminPassword

# Get Invoke-RemoteScript parameters
$invokeRemoteScriptParams = Get-InvokeRemoteScriptParameters -azureVMResourcesProperties $azureVMResourcesProperties `
-networkCredentials $azureVMsCredentials `
-skipCACheck $skipCACheck

# generate container sas token with full permissions
$containerSasToken = Generate-AzureStorageContainerSASToken -containerName $containerName -storageContext $storageContext -tokenTimeOutInMinutes $sasTokenTimeOutInMinutes

# Copies files on azureVMs
Copy-FilesToAzureVMsFromStorageContainer -targetMachineNames $invokeRemoteScriptParams.targetMachineNames `
-credential $invokeRemoteScriptParams.credential `
-protocol $invokeRemoteScriptParams.protocol `
-sessionOption $invokeRemoteScriptParams.sessionOption `
-blobStorageEndpoint $blobStorageEndpoint `
-containerName $containerName `
-containerSasToken $containerSasToken `
-targetPath $targetPath `
-cleanTargetBeforeCopy $cleanTargetBeforeCopy `
-copyFilesInParallel $copyFilesInParallel `
-additionalArguments $additionalArgumentsForVMCopy `
-azCopyToolLocation $azCopyLocation `
-fileCopyJobScript $AzureFileCopyRemoteJob `
-enableDetailedLogging $enableDetailedLogging

Write-Output (Get-VstsLocString -Key "AFC_CopySuccessful" -ArgumentList $sourcePath, $environmentName)
}
catch
{
Write-Verbose $_.Exception.ToString()

Write-Telemetry "Task_InternalError" "CopyingToAzureVMFailed"
throw
}
finally
{
Remove-AzureContainer -containerName $containerName -storageContext $storageContext
Remove-EndpointSecrets
Write-Verbose "Completed Azure File Copy Task for Azure VMs Destination" -Verbose
Trace-VstsLeavingInvocation $MyInvocation
}
}
finally {
Disconnect-AzureAndClearContext -authScheme $endpoint.Auth.Scheme -ErrorAction SilentlyContinue
}
110 changes: 110 additions & 0 deletions Tasks/AzureFileCopyV5/AzureFileCopyRemoteJob.ps1
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
$AzureFileCopyRemoteJob = {
param(
[string]$containerURL,
[string]$targetPath,
[string]$containerSasToken,
[string]$additionalArguments,
[switch]$CleanTargetBeforeCopy,
[switch]$EnableDetailedLogging
)

function Write-DetailLogs
{
[CmdletBinding()]
param(
[string]$message
)

if($EnableDetailedLogging)
{
Write-Verbose $message
}
}

try
{
$useDefaultArguments = ($additionalArguments -eq "")

#argument to check whether azcopy.exe needs to be downloaded on VM or it is already present on VM
$shouldDownload = $false

if($CleanTargetBeforeCopy)
{
if (Test-Path $targetPath -PathType Container)
{
Get-ChildItem -Path $targetPath -Recurse -Force | Remove-Item -Force -Recurse
Write-DetailLogs "Destination location cleaned"
}
else
{
Write-DetailLogs "Folder at path $targetPath not found for cleanup."
}
}

try
{
$azCopyVersionCommand = azcopy --version
$azCopyVersion = $azCopyVersionCommand.split(' ')[2]
if([version]$azCopyVersion -lt [version]"10.12.2")
{
$shouldDownload = $true
}
}
catch
{
$shouldDownload = $true
}

if($shouldDownload)
{
try
{
$azCopyFolderName = "ADO_AzCopyV10"
$azCopyFolderPath = Join-Path -Path $env:systemdrive -ChildPath $azCopyFolderName

New-Item -ItemType Directory -Force -Path $azCopyFolderPath
$azCopyZipPath = Join-Path -Path $azCopyFolderPath -ChildPath "AzCopy.zip"

# Downloading AzCopy from URL and copying it in $azcopyZipPath
$webclient = New-Object System.Net.WebClient
$webclient.DownloadFile('https://vstsagenttools.blob.core.windows.net/tools/azcopy/10.12/AzCopy.zip',$azCopyZipPath)

#Unzipping the azcopy zip to $azcopyFolderPath
Expand-Archive $azCopyZipPath -DestinationPath $azCopyFolderPath -Force

$azCopyFolderEnvPath = Join-Path -Path $azCopyFolderPath -ChildPath "AzCopy"

#setting path at machine level so that when user again do copy on VM, there is no need to download the azcopy.exe again
[Environment]::SetEnvironmentVariable("Path", $azCopyFolderEnvPath + ';' + $env:Path, [System.EnvironmentVariableTarget]::Machine)

#setting $env:Path at user level to include azcopy.exe path as the above command used do set the path at machine level and not at user level
$env:Path = $azCopyFolderEnvPath + ';' + $env:Path
}
catch
{
$exceptionMessage = $_.Exception.Message.ToString()
throw "Failed while downloading azcopy.exe from the URL with exception $exceptionMessage. Please download azcopy.exe 10.12.2 and set this extracted path in env:Path"
}
}

if($useDefaultArguments)
{
# Adding default optional arguments:
# log-level: Defines the log verbosity for the log file. Default is INFO(all requests/responses)
# recursive: Recursive copy

Write-DetailLogs "Using default AzCopy arguments for dowloading to VM"
$additionalArguments = "--recursive --log-level=INFO"
}

Write-DetailLogs "##[command] & azcopy copy `"$containerURL*****`" `"$targetPath`" $additionalArguments"

$azCopyCommand = "& azcopy copy `"$containerURL/*$containerSasToken`" `"$targetPath`" $additionalArguments"
Invoke-Expression $azCopyCommand
}
catch
{
Write-Verbose "AzureFileCopyRemoteJob threw exception"
throw
}
}
Loading

0 comments on commit bd84d4a

Please sign in to comment.