diff --git a/README.md b/README.md index 64ac89e6285..8522a4b3974 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ details. - Stemcells: [bosh-linux-stemcell-builder](https://github.com/cloudfoundry/bosh-linux-stemcell-builder), [bosh-windows-stemcell-builder](https://github.com/cloudfoundry-incubator/bosh-windows-stemcell-builder), [aws-light-stemcell-builder](https://github.com/cloudfoundry-incubator/aws-light-stemcell-builder) - CPIs: [AWS](https://github.com/cloudfoundry-incubator/bosh-aws-cpi-release), [Azure](https://github.com/cloudfoundry-incubator/bosh-azure-cpi-release), [Google](https://github.com/cloudfoundry-incubator/bosh-google-cpi-release), [OpenStack](https://github.com/cloudfoundry-incubator/bosh-openstack-cpi-release), [RackHD](https://github.com/cloudfoundry-incubator/bosh-rackhd-cpi-release), [SoftLayer](https://github.com/cloudfoundry-incubator/bosh-softlayer-cpi-release), [vSphere](https://github.com/cloudfoundry-incubator/bosh-vsphere-cpi-release), [vCloud](https://github.com/cloudfoundry-incubator/bosh-vcloud-cpi-release), [VirtualBox](https://github.com/cppforlife/bosh-virtualbox-cpi-release), [Warden](https://github.com/cppforlife/bosh-warden-cpi-release) - [Agent (bosh-agent)](https://github.com/cloudfoundry/bosh-agent) - - Blobstores: [bosh-davcli](https://github.com/cloudfoundry/bosh-davcli), [bosh-s3cli](https://github.com/cloudfoundry/bosh-s3cli), [bosh-gcscli](https://github.com/cloudfoundry/bosh-gcscli) + - Blobstores: [bosh-davcli](https://github.com/cloudfoundry/bosh-davcli), [bosh-s3cli](https://github.com/cloudfoundry/bosh-s3cli), [bosh-gcscli](https://github.com/cloudfoundry/bosh-gcscli), [bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli) - CPI libraries: [bosh-cpi-ruby](https://github.com/cloudfoundry/bosh-cpi-ruby), [bosh-cpi-go](https://github.com/cppforlife/bosh-cpi-go) - [Go common packages (bosh-utils)](https://github.com/cloudfoundry/bosh-utils) diff --git a/jobs/director/spec b/jobs/director/spec index 5041e31d15a..e7ae83a91ec 100644 --- a/jobs/director/spec +++ b/jobs/director/spec @@ -49,6 +49,7 @@ packages: - mysql - director-ruby-3.2 - s3cli +- azure-storage-cli - davcli - bosh-gcscli - verify_multidigest @@ -378,7 +379,7 @@ properties: # Blobstore blobstore.provider: - description: Provider of the blobstore used by director and agent (dav|simple|s3|gcs) + description: Provider of the blobstore used by director and agent (dav|simple|s3|gcs|azure-storage) default: 'dav' blobstore.s3_region: description: Region of the blobstore used by s3 blobstore plugin @@ -435,6 +436,12 @@ properties: default: false blobstore.secret: description: Secret used for HMAC signature for pre-signed urls + blobstore.account_name: + description: account_name of azure storage account + blobstore.container_name: + description: container_name of azure storage account + blobstore.account_key: + description: account_key of azure storage account director.ignore_missing_gateway: description: Allow gateway to be omitted from subnet configuration. Boshlite vms(containers) do not require gateway. diff --git a/jobs/director/templates/director.yml.erb b/jobs/director/templates/director.yml.erb index 984e133f7f2..14284c2f547 100644 --- a/jobs/director/templates/director.yml.erb +++ b/jobs/director/templates/director.yml.erb @@ -253,6 +253,12 @@ elsif p('blobstore.provider') == 'gcs' if_p('blobstore.encryption_key') do |encryption_key| blobstore_options['encryption_key'] = encryption_key end +elsif p('blobstore.provider') == 'azure-storage' + blobstore_options = { + 'account_name' => p('blobstore.account_name'), + 'container_name' => p('blobstore.container_name'), + 'account_key' => p('blobstore.account_key') + } else blobstore_options = { 'endpoint' => "https://#{p('blobstore.address')}:#{p('blobstore.port')}", @@ -290,6 +296,12 @@ if p('blobstore.provider') == "gcs" params['blobstore']['options']['gcscli_path'] = "/var/vcap/packages/bosh-gcscli/bin/bosh-gcscli" end +if p('blobstore.provider') == "azure-storage" + params['blobstore']['provider'] = "azurestoragecli" + params['blobstore']['options']['azure_storage_cli_config_path'] = "/var/vcap/data/director/tmp" + params['blobstore']['options']['azure_storage_cli_path'] = "/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli" +end + if p('blobstore.provider') == "dav" params['blobstore']['provider'] = "davcli" params['blobstore']['options']['davcli_config_path'] = "/var/vcap/data/director/tmp" diff --git a/packages/azure-storage-cli/README.md b/packages/azure-storage-cli/README.md new file mode 100644 index 00000000000..0fed880a8ad --- /dev/null +++ b/packages/azure-storage-cli/README.md @@ -0,0 +1,9 @@ +azure-storage-cli package +============ +This package is used for communicating with azure storage account + +The blob is compiled from [bosh-azure-storage-cli](https://github.com/cloudfoundry/bosh-azure-storage-cli) +and hosted in an [s3 bucket](https://bosh-azure-storage-cli-artifacts.s3.us-east-1.amazonaws.com/) + +(see the [concourse resource](https://bosh.ci.cloudfoundry.org/teams/main/pipelines/bosh-azure-storage-cli)) + diff --git a/packages/azure-storage-cli/packaging b/packages/azure-storage-cli/packaging new file mode 100644 index 00000000000..a1b1b2359d1 --- /dev/null +++ b/packages/azure-storage-cli/packaging @@ -0,0 +1,5 @@ +set -e + +mkdir -p ${BOSH_INSTALL_TARGET}/bin +mv azure-storage-cli/azure-storage-cli-*-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli +chmod +x ${BOSH_INSTALL_TARGET}/bin/azure-storage-cli diff --git a/packages/azure-storage-cli/spec b/packages/azure-storage-cli/spec new file mode 100644 index 00000000000..8890a50ddff --- /dev/null +++ b/packages/azure-storage-cli/spec @@ -0,0 +1,4 @@ +--- +name: azure-storage-cli +files: +- azure-storage-cli/azure-storage-cli-*-linux-amd64 diff --git a/src/bosh-director/lib/bosh/blobstore_client/azurestoragecli_blobstore_client.rb b/src/bosh-director/lib/bosh/blobstore_client/azurestoragecli_blobstore_client.rb new file mode 100644 index 00000000000..b0fe325fddf --- /dev/null +++ b/src/bosh-director/lib/bosh/blobstore_client/azurestoragecli_blobstore_client.rb @@ -0,0 +1,138 @@ +require 'openssl' +require 'digest/sha1' +require 'base64' +require 'securerandom' +require 'open3' +require 'json' + +module Bosh::Blobstore + class AzurestoragecliBlobstoreClient < BaseClient + # Blobstore client for azure storage account, using azure-storage-cli Go version + # @param [Hash] options azure storage account connection options + # @option options [Symbol] account_name + # key that is applied before the object is sent to azure storage account + # @option options [Symbol, optional] account_key + # @option options [Symbol, optional] container_name + # @option options [Symbol] azure_storage_cli_path + # path to azure-storage-cli binary + # @option options [Symbol, optional] azure_storage_cli_config_path + # path to store configuration files + def initialize(options) + super(options) + + @azure_storage_cli_path = @options.fetch(:azure_storage_cli_path) + + unless Kernel.system(@azure_storage_cli_path.to_s, '--v', out: '/dev/null', err: '/dev/null') + raise BlobstoreError, 'Cannot find azure-storage-cli executable. Please specify azure_storage_cli_path parameter' + end + + @azure_storage_cli_options = { + "account-name": @options[:account_name], + "container-name": @options[:container_name], + "account-key": @options[:account_key] + } + + @azure_storage_cli_options.reject! { |_k, v| v.nil? } + + @config_file = write_config_file(@azure_storage_cli_options, @options.fetch(:azure_storage_cli_config_path, nil)) + end + + def redacted_credential_properties_list + %w[account_key] + end + + def encryption_headers; end + + def encryption? + false + end + + protected + + # @param [File] file file to store in az storage account + def create_file(object_id, file) + object_id ||= generate_object_id + path = file.path + + store_in_azure_storage(path, full_oid_path(object_id)) + + object_id + end + + # @param [String] object_id object id to retrieve + # @param [File] file file to store the retrieved object in + def get_file(object_id, file) + begin + out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'get', object_id.to_s, file.path.to_s) + rescue Exception => e + raise BlobstoreError, e.inspect + end + return if status.success? + + raise NotFound, "Blobstore object '#{object_id}' not found" if err =~ /NoSuchKey/ + + raise BlobstoreError, "Failed to download azure storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" + end + + # @param [String] object_id object id to delete + def delete_object(object_id) + begin + out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'delete', object_id.to_s) + rescue Exception => e + raise BlobstoreError, e.inspect + end + raise BlobstoreError, "Failed to delete az storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success? + end + + def object_exists?(object_id) + begin + out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'exists', object_id.to_s) + return true if status.exitstatus.zero? + return false if status.exitstatus == 3 + rescue Exception => e + raise BlobstoreError, e.inspect + end + raise BlobstoreError, "Failed to check existence of az storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success? + end + + def sign_url(object_id, verb, duration) + begin + out, err, status = Open3.capture3( + @azure_storage_cli_path.to_s, + '-c', + @config_file.to_s, + 'sign', + object_id.to_s, + verb.to_s, + duration.to_s, + ) + rescue Exception => e + raise BlobstoreError, e.inspect + end + + return out if status.success? + + raise BlobstoreError, "Failed to sign url, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" + end + + def required_credential_properties_list + %w[account_key] + end + + # @param [String] path path to file which will be stored in az storage account + # @param [String] oid object id + # @return [void] + def store_in_azure_storage(path, oid) + begin + out, err, status = Open3.capture3(@azure_storage_cli_path.to_s, '-c', @config_file.to_s, 'put', path.to_s, oid.to_s) + rescue Exception => e + raise BlobstoreError, e.inspect + end + raise BlobstoreError, "Failed to create azure storage account object, code #{status.exitstatus}, output: '#{out}', error: '#{err}'" unless status.success? + end + + def full_oid_path(object_id) + @options[:folder] ? @options[:folder] + '/' + object_id : object_id + end + end +end diff --git a/src/bosh-director/lib/bosh/blobstore_client/client.rb b/src/bosh-director/lib/bosh/blobstore_client/client.rb index 4b264112250..317d642b091 100644 --- a/src/bosh-director/lib/bosh/blobstore_client/client.rb +++ b/src/bosh-director/lib/bosh/blobstore_client/client.rb @@ -1,7 +1,7 @@ module Bosh module Blobstore class Client - PROVIDER_NAMES = %w[local s3cli gcscli davcli] + PROVIDER_NAMES = %w[local s3cli gcscli davcli azurestoragecli] def self.create(blobstore_provider, options = {}) unless PROVIDER_NAMES.include?(blobstore_provider) diff --git a/src/bosh-director/lib/bosh/director.rb b/src/bosh-director/lib/bosh/director.rb index 1098fcd94e6..f9967aa07c2 100644 --- a/src/bosh-director/lib/bosh/director.rb +++ b/src/bosh-director/lib/bosh/director.rb @@ -284,4 +284,5 @@ module Bosh::Director Bosh::Blobstore.autoload(:LocalClient, 'bosh/blobstore_client/local_client') Bosh::Blobstore.autoload(:DavcliBlobstoreClient, 'bosh/blobstore_client/davcli_blobstore_client') Bosh::Blobstore.autoload(:S3cliBlobstoreClient, 'bosh/blobstore_client/s3cli_blobstore_client') +Bosh::Blobstore.autoload(:AzurestoragecliBlobstoreClient, 'bosh/blobstore_client/azurestoragecli_blobstore_client') Bosh::Blobstore.autoload(:GcscliBlobstoreClient, 'bosh/blobstore_client/gcscli_blobstore_client') \ No newline at end of file diff --git a/src/bosh-director/spec/unit/blobstore_client/azurestoragecli_blobstore_client_spec.rb b/src/bosh-director/spec/unit/blobstore_client/azurestoragecli_blobstore_client_spec.rb new file mode 100644 index 00000000000..44afca5c472 --- /dev/null +++ b/src/bosh-director/spec/unit/blobstore_client/azurestoragecli_blobstore_client_spec.rb @@ -0,0 +1,222 @@ +require 'spec_helper' +require 'json' + +module Bosh::Blobstore + describe AzurestoragecliBlobstoreClient do + subject(:client) { described_class.new(options) } + let!(:base_dir) { Dir.mktmpdir } + before do + allow(Dir).to receive(:tmpdir).and_return(base_dir) + allow(SecureRandom).to receive_messages(uuid: 'FAKE_UUID') + allow(Kernel).to receive(:system).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '--v', out: '/dev/null', err: '/dev/null').and_return(true) + end + + let(:options) do + { + container_name: 'test', + account_name: 'NAME', + account_key: 'SECRET', + azure_storage_cli_path: '/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', + } + end + let(:expected_config_file) { File.join(base_dir, 'blobstore-config') } + let(:success_exit_status) { instance_double('Process::Status', exitstatus: 0, success?: true) } + let(:not_existed_exit_status) { instance_double('Process::Status', exitstatus: 3, success?: true) } + let(:failure_exit_status) { instance_double('Process::Status', exitstatus: 1, success?: false) } + let(:object_id) { 'fo1' } + let(:file_path) { File.join(base_dir, 'temp-path-FAKE_UUID') } + + after { FileUtils.rm_rf(base_dir) } + + describe 'interface' do + it_implements_base_client_interface + end + + describe 'options' do + let(:expected_options) do + { + 'container-name' => 'test', + 'account-name' => 'NAME', + 'account-key' => 'SECRET' + } + end + let(:stored_config_file) { File.new(expected_config_file).readlines } + + context 'when there is no azure-storage-cli' do + it 'raises an error' do + allow(Kernel).to receive(:system).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '--v', out: '/dev/null', err: '/dev/null').and_return(false) + expect { described_class.new(options) }.to raise_error( + Bosh::Blobstore::BlobstoreError, 'Cannot find azure-storage-cli executable. Please specify azure_storage_cli_path parameter' + ) + end + end + + context 'when azure-storage-cli exists' do + before { described_class.new(options) } + + it 'should set default values to config file' do + expect(File.exist?(expected_config_file)).to eq(true) + expect(JSON.parse(stored_config_file[0])).to eq(expected_options) + end + + it 'should write the config file with reduced group and world permissions' do + expect(File.stat(expected_config_file).mode).to eq(0o100600) + end + end + + context 'when azure_storage_cli_config_path option is provided' do + let(:azure_storage_cli_config_path) { Dir.tmpdir } + let(:config_file_options) do + options.merge ( + { + azure_storage_cli_config_path: azure_storage_cli_config_path, + }) + end + + it 'creates config file with provided path' do + described_class.new(config_file_options) + expect(File.exist?(File.join(azure_storage_cli_config_path, 'blobstore-config'))).to eq(true) + end + end + end + + describe '#delete' do + it 'should delete an object' do + allow(Open3).to receive(:capture3).and_return([nil, nil, success_exit_status]) + expect(Open3).to receive(:capture3).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '-c', expected_config_file.to_s, 'delete', object_id.to_s) + client.delete(object_id) + end + + it 'should show an error from azure-storage-cli' do + allow(Open3).to receive(:capture3).and_return([nil, 'error', failure_exit_status]) + expect { client.delete(object_id) }.to raise_error( + BlobstoreError, /error: 'error'/ + ) + end + end + + describe '#exists?' do + it 'should return true if azure-storage-cli reported so' do + allow(Open3).to receive(:capture3).and_return([nil, nil, success_exit_status]) + expect(Open3).to receive(:capture3).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '-c', expected_config_file.to_s, 'exists', object_id.to_s) + + expect(client.exists?(object_id)).to eq(true) + end + + it 'should return false if azure-storage-cli reported so' do + allow(Open3).to receive(:capture3).and_return([nil, nil, not_existed_exit_status]) + expect(Open3).to receive(:capture3).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '-c', expected_config_file.to_s, 'exists', object_id.to_s) + expect(client.exists?(object_id)).to eq(false) + end + + it 'should show an error from azure-storage-cli' do + allow(Open3).to receive(:capture3).and_return([nil, 'error', failure_exit_status]) + expect { client.create(object_id) }.to raise_error( + BlobstoreError, /error: 'error'/ + ) + end + end + + describe '#get' do + it 'should raise on execution failure' do + allow(Open3).to receive(:capture3).and_raise(Exception.new('something bad happened')) + expect { client.get(object_id) }.to raise_error( + BlobstoreError, /something bad happened/ + ) + end + + it 'should have correct parameters' do + allow(Open3).to receive(:capture3).and_return([nil, nil, success_exit_status]) + expect(Open3).to receive(:capture3).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '-c', expected_config_file.to_s, 'get', object_id.to_s, file_path.to_s) + client.get(object_id) + end + + it 'should show an error from azure-storage-cli' do + allow(Open3).to receive(:capture3).and_return([nil, 'error', failure_exit_status]) + expect { client.get(object_id) }.to raise_error( + BlobstoreError, /Failed to download azure storage account object/ + ) + end + + it 'should raise a NotFound error if the key does not exist' do + allow(Open3).to receive(:capture3).and_return([nil, 'NoSuchKey', failure_exit_status]) + expect { client.get(object_id) }.to raise_error( + NotFound, "Blobstore object '#{object_id}' not found" + ) + end + end + + describe '#create' do + it 'should take a string as argument' do + expect(client).to receive(:store_in_azure_storage) + client.create('foobar') + end + + it 'should take a file as argument' do + expect(client).to receive(:store_in_azure_storage) + file = File.open(Tempfile.new('file')) + client.create(file) + end + + it 'should have correct parameters' do + allow(Open3).to receive(:capture3).and_return([nil, nil, success_exit_status]) + file = File.open(Tempfile.new('file')) + expect(Open3).to receive(:capture3).with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '-c', expected_config_file.to_s, 'put', file.path.to_s, 'FAKE_UUID') + client.create(file) + end + + it 'should show an error ' do + allow(Open3).to receive(:capture3).and_return([nil, nil, failure_exit_status]) + expect { client.create(object_id) }.to raise_error( + BlobstoreError, /Failed to create azure storage account object/ + ) + end + + it 'should show an error from azure-storage-cli' do + allow(Open3).to receive(:capture3).and_return([nil, 'error', failure_exit_status]) + expect { client.create(object_id) }.to raise_error( + BlobstoreError, /error: 'error'/ + ) + end + end + + describe '#sign_url' do + it 'should return the signed url' do + expect(Open3).to receive(:capture3) + .with('/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', '-c', expected_config_file.to_s, 'sign', object_id.to_s, 'get', '24h') + .and_return(['https://signed-url', nil, success_exit_status]) + expect(subject.sign(object_id, 'get')).to eq('https://signed-url') + end + + it 'should show an error from azure-storage-cli' do + allow(Open3).to receive(:capture3).and_return([nil, 'error', failure_exit_status]) + expect { subject.sign(object_id, 'get') }.to raise_error( + BlobstoreError, /error: 'error'/ + ) + end + + it 'provides properties to remove for agent settings' do + expect(subject.redacted_credential_properties_list).to eq(%w[account_key]) + end + end + + describe 'signed url encryption headers - not implemented' do + let(:options) do + { + container_name: 'test', + account_name: 'NAME', + account_key: 'SECRET', + azure_storage_cli_path: '/var/vcap/packages/azure-storage-cli/bin/azure-storage-cli', + } + end + + it 'should not produce headers based on encryption options' do + expect(subject.signed_url_encryption_headers).to be_nil + end + + it 'does not support encryption' do + expect(subject.encryption?).to eq(false) + end + end + end +end