Merge pull request #2456 from fmoehler/Integrate-AZCLI

Integrate bosh-azure-storage-cli into bosh

README.md

@@ -34,7 +34,7 @@ details.
 	- Stemcells: [bosh-linux-stemcell-builder](https://github.com/cloudfoundry/bosh-linux-stemcell-builder), [bosh-windows-stemcell-builder](https://github.com/cloudfoundry-incubator/bosh-windows-stemcell-builder), [aws-light-stemcell-builder](https://github.com/cloudfoundry-incubator/aws-light-stemcell-builder)
 	- CPIs: [AWS](https://github.com/cloudfoundry-incubator/bosh-aws-cpi-release), [Azure](https://github.com/cloudfoundry-incubator/bosh-azure-cpi-release), [Google](https://github.com/cloudfoundry-incubator/bosh-google-cpi-release), [OpenStack](https://github.com/cloudfoundry-incubator/bosh-openstack-cpi-release), [RackHD](https://github.com/cloudfoundry-incubator/bosh-rackhd-cpi-release), [SoftLayer](https://github.com/cloudfoundry-incubator/bosh-softlayer-cpi-release), [vSphere](https://github.com/cloudfoundry-incubator/bosh-vsphere-cpi-release), [vCloud](https://github.com/cloudfoundry-incubator/bosh-vcloud-cpi-release), [VirtualBox](https://github.com/cppforlife/bosh-virtualbox-cpi-release), [Warden](https://github.com/cppforlife/bosh-warden-cpi-release)
 	- [Agent (bosh-agent)](https://github.com/cloudfoundry/bosh-agent)
-	- Blobstores: [bosh-davcli](https://github.com/cloudfoundry/bosh-davcli), [bosh-s3cli](https://github.com/cloudfoundry/bosh-s3cli), [bosh-gcscli](https://github.com/cloudfoundry/bosh-gcscli)
+	- Blobstores: [bosh-davcli](https://github.com/cloudfoundry/bosh-davcli), [bosh-s3cli](https://github.com/cloudfoundry/bosh-s3cli), [bosh-gcscli](https://github.com/cloudfoundry/bosh-gcscli), [bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli)
 	- CPI libraries: [bosh-cpi-ruby](https://github.com/cloudfoundry/bosh-cpi-ruby), [bosh-cpi-go](https://github.com/cppforlife/bosh-cpi-go)
 	- [Go common packages (bosh-utils)](https://github.com/cloudfoundry/bosh-utils)
 

jobs/director/spec

@@ -49,6 +49,7 @@ packages:
 - mysql
 - director-ruby-3.2
 - s3cli
+- azure-storage-cli
 - davcli
 - bosh-gcscli
 - verify_multidigest
@@ -378,7 +379,7 @@ properties:
 
   # Blobstore
   blobstore.provider:
-    description: Provider of the blobstore used by director and agent (dav|simple|s3|gcs)
+    description: Provider of the blobstore used by director and agent (dav|simple|s3|gcs|azure-storage)
     default: 'dav'
   blobstore.s3_region:
     description: Region of the blobstore used by s3 blobstore plugin
@@ -435,6 +436,12 @@ properties:
     default: false
   blobstore.secret:
     description: Secret used for HMAC signature for pre-signed urls
+  blobstore.account_name:
+    description: account_name of azure storage account
+  blobstore.container_name:
+    description: container_name of azure storage account
+  blobstore.account_key:
+    description: account_key of azure storage account
 
   director.ignore_missing_gateway:
     description: Allow gateway to be omitted from subnet configuration. Boshlite vms(containers) do not require gateway.

jobs/director/templates/director.yml.erb

@@ -253,6 +253,12 @@ elsif p('blobstore.provider') == 'gcs'
   if_p('blobstore.encryption_key') do |encryption_key|
      blobstore_options['encryption_key'] = encryption_key
   end
+elsif p('blobstore.provider') == 'azure-storage'
+  blobstore_options = {
+    'account_name' => p('blobstore.account_name'),
+    'container_name' => p('blobstore.container_name'),
+    'account_key' => p('blobstore.account_key')
+  }
 else
   blobstore_options = {
     'endpoint' => "https://#{p('blobstore.address')}:#{p('blobstore.port')}",
@@ -290,6 +296,12 @@ if p('blobstore.provider') == "gcs"
    params['blobstore']['options']['gcscli_path'] = "/var/vcap/packages/bosh-gcscli/bin/bosh-gcscli"
 end
 
+if p('blobstore.provider') == "azure-storage"
+   params['blobstore']['provider'] = "azurestoragecli"
+   params['blobstore']['options']['azure_storage_cli_config_path'] = "/var/vcap/data/director/tmp"
+   params['blobstore']['options']['azure_storage_cli_path'] = "/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli"
+end
+
 if p('blobstore.provider') == "dav"
    params['blobstore']['provider'] = "davcli"
    params['blobstore']['options']['davcli_config_path'] = "/var/vcap/data/director/tmp"

packages/azure-storage-cli/README.md

@@ -0,0 +1,9 @@
+azure-storage-cli package
+============
+This package is used for communicating with azure storage account
+
+The blob is compiled from [bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli)
+and hosted in an [s3 bucket](https://bosh-azure-storage-cli-artifacts.s3.us-east-1.amazonaws.com/)
+
+(see the [concourse resource](https://bosh.ci.cloudfoundry.org/teams/main/pipelines/bosh-azure-storage-cli))
+

packages/azure-storage-cli/packaging

@@ -0,0 +1,5 @@
+set -e
+
+mkdir -p ${BOSH_INSTALL_TARGET}/bin
+mv azure-storage-cli/azure-storage-cli-*-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli
+chmod +x ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli

packages/azure-storage-cli/spec

@@ -0,0 +1,4 @@
+---
+name: azure-storage-cli
+files:
+- azure-storage-cli/azure-storage-cli-*-linux-amd64

src/bosh-director/lib/bosh/blobstore_client/azurestoragecli_blobstore_client.rb

@@ -0,0 +1,138 @@
+require 'openssl'
+require 'digest/sha1'
+require 'base64'
+require 'securerandom'
+require 'open3'
+require 'json'
+
+module Bosh::Blobstore
+  class AzurestoragecliBlobstoreClient < BaseClient
+    # Blobstore client for azure storage account, using azure-storage-cli Go version
+    # @param [Hash] options azure storage account connection options
+    # @option options [Symbol] account_name
+    #   key that is applied before the object is sent to azure storage account
+    # @option options [Symbol, optional] account_key
+    # @option options [Symbol, optional] container_name
+    # @option options [Symbol] azure_storage_cli_path
+    #   path to azure-storage-cli binary
+    # @option options [Symbol, optional] azure_storage_cli_config_path
+    #   path to store configuration files
+    def initialize(options)
+      super(options)
+
+      @azure_storage_cli_path = @options.fetch(:azure_storage_cli_path)
+
+      unless Kernel.system(@azure_storage_cli_path.to_s, '--v', out: '/dev/null', err: '/dev/null')
+        raise BlobstoreError, 'Cannot find azure-storage-cli executable. Please specify azure_storage_cli_path parameter'
+      end
+
+      @azure_storage_cli_options = {
+        "account-name": @options[:account_name],
+        "container-name": @options[:container_name],
+        "account-key": @options[:account_key]
+      }
+
+      @azure_storage_cli_options.reject! { |_k, v| v.nil? }
+
+      @config_file = write_config_file(@azure_storage_cli_options, @options.fetch(:azure_storage_cli_config_path, nil))
+    end
+
+    def redacted_credential_properties_list
+      %w[account_key]
+    end
+
+    def encryption_headers; end
+
+    def encryption?
+      false
+    end
+
+    protected
+
+    # @param [File] file file to store in az storage account
+    def create_file(object_id, file)
+      object_id ||= generate_object_id
+      path = file.path
+
+      store_in_azure_storage(path, full_oid_path(object_id))
+
+      object_id
+    end
+
+    # @param [String] object_id object id to retrieve
+    # @param [File] file file to store the retrieved object in
+    def get_file(object_id, file)
+      begin
+        out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'get', object_id.to_s, file.path.to_s)
+      rescue Exception => e
+        raise BlobstoreError, e.inspect
+      end
+      return if status.success?
+
+      raise NotFound, "Blobstore object '#{object_id}' not found" if err =~ /NoSuchKey/
+
+      raise BlobstoreError, "Failed to download azure storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'"
+    end
+
+    # @param [String] object_id object id to delete
+    def delete_object(object_id)
+      begin
+        out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'delete', object_id.to_s)
+      rescue Exception => e
+        raise BlobstoreError, e.inspect
+      end
+      raise BlobstoreError, "Failed to delete az storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+    end
+
+    def object_exists?(object_id)
+      begin
+        out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'exists', object_id.to_s)
+        return true if status.exitstatus.zero?
+        return false if status.exitstatus == 3
+      rescue Exception => e
+        raise BlobstoreError, e.inspect
+      end
+      raise BlobstoreError, "Failed to check existence of az storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+    end
+
+    def sign_url(object_id, verb, duration)
+      begin
+        out, err, status = Open3.capture3(
+          @azure_storage_cli_path.to_s,
+          '-c',
+          @config_file.to_s,
+          'sign',
+          object_id.to_s,
+          verb.to_s,
+          duration.to_s,
+        )
+      rescue Exception => e
+        raise BlobstoreError, e.inspect
+      end
+
+      return out if status.success?
+
+      raise BlobstoreError, "Failed to sign url, code #{status.exitstatus}, output: '#{out}', error: '#{err}'"
+    end
+
+    def required_credential_properties_list
+      %w[account_key]
+    end
+
+    # @param [String] path path to file which will be stored in az storage account
+    # @param [String] oid object id
+    # @return [void]
+    def store_in_azure_storage(path, oid)
+      begin
+        out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'put', path.to_s, oid.to_s)
+      rescue Exception => e
+        raise BlobstoreError, e.inspect
+      end
+      raise BlobstoreError, "Failed to create azure storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success?
+    end
+
+    def full_oid_path(object_id)
+      @options[:folder] ? @options[:folder] + '/' + object_id : object_id
+    end
+  end
+end

src/bosh-director/lib/bosh/blobstore_client/client.rb

@@ -1,7 +1,7 @@
 module Bosh
   module Blobstore
     class Client
-      PROVIDER_NAMES = %w[local s3cli gcscli davcli]
+      PROVIDER_NAMES = %w[local s3cli gcscli davcli azurestoragecli]
 
       def self.create(blobstore_provider, options = {})
         unless PROVIDER_NAMES.include?(blobstore_provider)

src/bosh-director/lib/bosh/director.rb

@@ -284,4 +284,5 @@ module Bosh::Director
 Bosh::Blobstore.autoload(:LocalClient, 'bosh/blobstore_client/local_client')
 Bosh::Blobstore.autoload(:DavcliBlobstoreClient, 'bosh/blobstore_client/davcli_blobstore_client')
 Bosh::Blobstore.autoload(:S3cliBlobstoreClient, 'bosh/blobstore_client/s3cli_blobstore_client')
+Bosh::Blobstore.autoload(:AzurestoragecliBlobstoreClient, 'bosh/blobstore_client/azurestoragecli_blobstore_client')
 Bosh::Blobstore.autoload(:GcscliBlobstoreClient, 'bosh/blobstore_client/gcscli_blobstore_client')