From 040f31c6ef01e6d1aa2251aa09bc56633a9d3e2c Mon Sep 17 00:00:00 2001 From: Martin Strobel Date: Fri, 3 Mar 2017 13:38:42 -0800 Subject: [PATCH] Moving Storage to an Independent Repository (#556) * Updating CI to use Go 1.8. Go 1.8 was released on Thursday, February 16th. Many of our partners have adopted and we need to ensure that we support the most recent version. * Replacing storage folder with submodule. In order to allow for more independent growth of the storage package, we've given it its own repository. This change updates the main repository to use a submodule in order to prevent breaking changes for folks using storage today who do not want to migrate to the new repository. * Updating submodule target URL. There is an existing naming convention for Azure Storage repositories. the repo has been updated to match that convention, this updates the submodule to point to the new URL. * Responding to review feedback. * Updating submodule to ref v8.1.0 storage version. --- .gitmodules | 4 + .travis.yml | 8 +- Gododir/gen.go | 14 +- storage/README.md | 5 - storage/authorization.go | 223 ----- storage/authorization_test.go | 226 ----- storage/blob.go | 1539 ------------------------------- storage/blob_test.go | 1599 --------------------------------- storage/client.go | 469 ---------- storage/client_test.go | 244 ----- storage/directory.go | 217 ----- storage/directory_test.go | 112 --- storage/file.go | 360 -------- storage/file_test.go | 274 ------ storage/fileserviceclient.go | 360 -------- storage/queue.go | 346 ------- storage/queue_test.go | 142 --- storage/share.go | 186 ---- storage/share_test.go | 188 ---- storage/storagepolicy.go | 47 - storage/table.go | 258 ------ storage/table_entities.go | 354 -------- storage/table_test.go | 338 ------- storage/util.go | 85 -- storage/util_test.go | 86 -- storage/version.go | 5 - 26 files changed, 15 insertions(+), 7674 deletions(-) create mode 100644 .gitmodules delete mode 100644 storage/README.md delete mode 100644 storage/authorization.go delete mode 100644 storage/authorization_test.go delete mode 100644 storage/blob.go delete mode 100644 storage/blob_test.go delete mode 100644 storage/client.go delete mode 100644 storage/client_test.go delete mode 100644 storage/directory.go delete mode 100644 storage/directory_test.go delete mode 100644 storage/file.go delete mode 100644 storage/file_test.go delete mode 100644 storage/fileserviceclient.go delete mode 100644 storage/queue.go delete mode 100644 storage/queue_test.go delete mode 100644 storage/share.go delete mode 100644 storage/share_test.go delete mode 100644 storage/storagepolicy.go delete mode 100644 storage/table.go delete mode 100644 storage/table_entities.go delete mode 100644 storage/table_test.go delete mode 100644 storage/util.go delete mode 100644 storage/util_test.go delete mode 100644 storage/version.go diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..2bfaa0ebb403 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "storage"] + path = storage + url = https://github.com/Azure/azure-storage-go + branch = v0.1.0 diff --git a/.travis.yml b/.travis.yml index 231bfb3462b1..760841b0d820 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ sudo: false language: go -go: 1.7 +go: 1.8 install: - go get -u github.com/golang/lint/golint @@ -16,14 +16,10 @@ script: - bash rungas.sh - test -z "$(gofmt -s -l $(find ./arm/* -type d -print) | tee /dev/stderr)" - test -z "$(gofmt -s -l -w management | tee /dev/stderr)" - - test -z "$(gofmt -s -l -w storage | tee /dev/stderr)" - test -z "$(gofmt -s -l -w Gododir | tee /dev/stderr)" - - test -z "$(go build $(find ./* -type d -print | grep -v '^./vendor$') | tee /dev/stderr)" + - test -z "$(go build $(find ./* -type d -print | grep -v '^./vendor$' | grep -v '^./storage$') | tee /dev/stderr)" - test -z "$(go vet $(find ./arm/* -type d -print) | tee /dev/stderr)" - test -z "$(golint ./arm/... | tee /dev/stderr)" - - go test -v ./storage/... -check.v - - test -z "$(golint ./storage/... | tee /dev/stderr)" - - go vet ./storage/... - go test -v ./management/... - test -z "$(golint ./management/... | grep -v 'should have comment' | grep -v 'stutters' | tee /dev/stderr)" - go vet ./management/... diff --git a/Gododir/gen.go b/Gododir/gen.go index 956a0990eb94..6990afebf5e2 100644 --- a/Gododir/gen.go +++ b/Gododir/gen.go @@ -463,7 +463,7 @@ func initAndAddService(service *service, inputPrefix, plane string) { } func tasks(p *do.Project) { - p.Task("default", do.S{"setvars", "generate:all", "storage"}, nil) + p.Task("default", do.S{"setvars", "generate:all", "management"}, nil) p.Task("setvars", nil, setVars) p.Use("generate", generateTasks) p.Use("gofmt", formatTasks) @@ -471,7 +471,7 @@ func tasks(p *do.Project) { p.Use("golint", lintTasks) p.Use("govet", vetTasks) p.Use("delete", deleteTasks) - p.Task("storage", do.S{"setvars"}, storageVersion) + p.Task("management", do.S{"setvars"}, managementVersion) } func setVars(c *do.Context) { @@ -507,7 +507,7 @@ func generate(service *service) { "-OutputDirectory", service.Output, "-Modeler", "Swagger", "-pv", sdkVersion, - "-SkipValidation", + "-SkipValidation") autorest.Dir = filepath.Join(autorestDir, "autorest") err = runner(autorest) if err != nil { @@ -584,8 +584,12 @@ func vet(service *service) { } } -func storageVersion(c *do.Context) { - versionFile := "storage/version.go" +func managementVersion(c *do.Context) { + version("management") +} + +func version(packageName string) { + versionFile := filepath.Join(packageName, "version.go") os.Remove(versionFile) template := `package storage diff --git a/storage/README.md b/storage/README.md deleted file mode 100644 index 0ab099848bba..000000000000 --- a/storage/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Azure Storage SDK for Go - -The `github.com/Azure/azure-sdk-for-go/storage` package is used to perform operations in Azure Storage Service. To manage your storage accounts (Azure Resource Manager / ARM), use the [github.com/Azure/azure-sdk-for-go/arm/storage](../arm/storage) package. For your classic storage accounts (Azure Service Management / ASM), use [github.com/Azure/azure-sdk-for-go/management/storageservice](../management/storageservice) package. - -This package includes support for [Azure Storage Emulator](https://azure.microsoft.com/documentation/articles/storage-use-emulator/) \ No newline at end of file diff --git a/storage/authorization.go b/storage/authorization.go deleted file mode 100644 index 89a0d0b3cdf8..000000000000 --- a/storage/authorization.go +++ /dev/null @@ -1,223 +0,0 @@ -// Package storage provides clients for Microsoft Azure Storage Services. -package storage - -import ( - "bytes" - "fmt" - "net/url" - "sort" - "strings" -) - -// See: https://docs.microsoft.com/rest/api/storageservices/fileservices/authentication-for-the-azure-storage-services - -type authentication string - -const ( - sharedKey authentication = "sharedKey" - sharedKeyForTable authentication = "sharedKeyTable" - sharedKeyLite authentication = "sharedKeyLite" - sharedKeyLiteForTable authentication = "sharedKeyLiteTable" - - // headers - headerAuthorization = "Authorization" - headerContentLength = "Content-Length" - headerDate = "Date" - headerXmsDate = "x-ms-date" - headerXmsVersion = "x-ms-version" - headerContentEncoding = "Content-Encoding" - headerContentLanguage = "Content-Language" - headerContentType = "Content-Type" - headerContentMD5 = "Content-MD5" - headerIfModifiedSince = "If-Modified-Since" - headerIfMatch = "If-Match" - headerIfNoneMatch = "If-None-Match" - headerIfUnmodifiedSince = "If-Unmodified-Since" - headerRange = "Range" -) - -func (c *Client) addAuthorizationHeader(verb, url string, headers map[string]string, auth authentication) (map[string]string, error) { - authHeader, err := c.getSharedKey(verb, url, headers, auth) - if err != nil { - return nil, err - } - headers[headerAuthorization] = authHeader - return headers, nil -} - -func (c *Client) getSharedKey(verb, url string, headers map[string]string, auth authentication) (string, error) { - canRes, err := c.buildCanonicalizedResource(url, auth) - if err != nil { - return "", err - } - - canString, err := buildCanonicalizedString(verb, headers, canRes, auth) - if err != nil { - return "", err - } - return c.createAuthorizationHeader(canString, auth), nil -} - -func (c *Client) buildCanonicalizedResource(uri string, auth authentication) (string, error) { - errMsg := "buildCanonicalizedResource error: %s" - u, err := url.Parse(uri) - if err != nil { - return "", fmt.Errorf(errMsg, err.Error()) - } - - cr := bytes.NewBufferString("/") - cr.WriteString(c.getCanonicalizedAccountName()) - - if len(u.Path) > 0 { - // Any portion of the CanonicalizedResource string that is derived from - // the resource's URI should be encoded exactly as it is in the URI. - // -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx - cr.WriteString(u.EscapedPath()) - } - - params, err := url.ParseQuery(u.RawQuery) - if err != nil { - return "", fmt.Errorf(errMsg, err.Error()) - } - - // See https://github.com/Azure/azure-storage-net/blob/master/Lib/Common/Core/Util/AuthenticationUtility.cs#L277 - if auth == sharedKey { - if len(params) > 0 { - cr.WriteString("\n") - - keys := []string{} - for key := range params { - keys = append(keys, key) - } - sort.Strings(keys) - - completeParams := []string{} - for _, key := range keys { - if len(params[key]) > 1 { - sort.Strings(params[key]) - } - - completeParams = append(completeParams, fmt.Sprintf("%s:%s", key, strings.Join(params[key], ","))) - } - cr.WriteString(strings.Join(completeParams, "\n")) - } - } else { - // search for "comp" parameter, if exists then add it to canonicalizedresource - if v, ok := params["comp"]; ok { - cr.WriteString("?comp=" + v[0]) - } - } - - return string(cr.Bytes()), nil -} - -func (c *Client) getCanonicalizedAccountName() string { - // since we may be trying to access a secondary storage account, we need to - // remove the -secondary part of the storage name - return strings.TrimSuffix(c.accountName, "-secondary") -} - -func buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string, auth authentication) (string, error) { - contentLength := headers[headerContentLength] - if contentLength == "0" { - contentLength = "" - } - date := headers[headerDate] - if v, ok := headers[headerXmsDate]; ok { - if auth == sharedKey || auth == sharedKeyLite { - date = "" - } else { - date = v - } - } - var canString string - switch auth { - case sharedKey: - canString = strings.Join([]string{ - verb, - headers[headerContentEncoding], - headers[headerContentLanguage], - contentLength, - headers[headerContentMD5], - headers[headerContentType], - date, - headers[headerIfModifiedSince], - headers[headerIfMatch], - headers[headerIfNoneMatch], - headers[headerIfUnmodifiedSince], - headers[headerRange], - buildCanonicalizedHeader(headers), - canonicalizedResource, - }, "\n") - case sharedKeyForTable: - canString = strings.Join([]string{ - verb, - headers[headerContentMD5], - headers[headerContentType], - date, - canonicalizedResource, - }, "\n") - case sharedKeyLite: - canString = strings.Join([]string{ - verb, - headers[headerContentMD5], - headers[headerContentType], - date, - buildCanonicalizedHeader(headers), - canonicalizedResource, - }, "\n") - case sharedKeyLiteForTable: - canString = strings.Join([]string{ - date, - canonicalizedResource, - }, "\n") - default: - return "", fmt.Errorf("%s authentication is not supported yet", auth) - } - return canString, nil -} - -func buildCanonicalizedHeader(headers map[string]string) string { - cm := make(map[string]string) - - for k, v := range headers { - headerName := strings.TrimSpace(strings.ToLower(k)) - if strings.HasPrefix(headerName, "x-ms-") { - cm[headerName] = v - } - } - - if len(cm) == 0 { - return "" - } - - keys := []string{} - for key := range cm { - keys = append(keys, key) - } - - sort.Strings(keys) - - ch := bytes.NewBufferString("") - - for _, key := range keys { - ch.WriteString(key) - ch.WriteRune(':') - ch.WriteString(cm[key]) - ch.WriteRune('\n') - } - - return strings.TrimSuffix(string(ch.Bytes()), "\n") -} - -func (c *Client) createAuthorizationHeader(canonicalizedString string, auth authentication) string { - signature := c.computeHmac256(canonicalizedString) - var key string - switch auth { - case sharedKey, sharedKeyForTable: - key = "SharedKey" - case sharedKeyLite, sharedKeyLiteForTable: - key = "SharedKeyLite" - } - return fmt.Sprintf("%s %s:%s", key, c.getCanonicalizedAccountName(), signature) -} diff --git a/storage/authorization_test.go b/storage/authorization_test.go deleted file mode 100644 index 436a64fbf144..000000000000 --- a/storage/authorization_test.go +++ /dev/null @@ -1,226 +0,0 @@ -package storage - -import ( - "encoding/base64" - "net/http" - - chk "gopkg.in/check.v1" -) - -type AuthorizationSuite struct{} - -var _ = chk.Suite(&AuthorizationSuite{}) - -func (a *AuthorizationSuite) Test_addAuthorizationHeader(c *chk.C) { - cli, err := NewBasicClient("mindgotest", "zHDHGs7C+Di9pZSDMuarxJJz3xRBzAHBYaobxpLEc7kwTptR/hPEa9j93hIfb2Tbe9IA50MViGmjQ6nUF/OVvA==") - c.Assert(err, chk.IsNil) - cli.UseSharedKeyLite = true - tableCli := cli.GetTableService() - - headers := map[string]string{ - "Accept-Charset": "UTF-8", - headerContentType: "application/json", - headerXmsDate: "Wed, 23 Sep 2015 16:40:05 GMT", - headerContentLength: "0", - headerXmsVersion: "2015-02-21", - "Accept": "application/json;odata=nometadata", - } - url := "https://mindgotest.table.core.windows.net/tquery()" - headers, err = tableCli.client.addAuthorizationHeader("", url, headers, tableCli.auth) - c.Assert(err, chk.IsNil) - - c.Assert(headers[headerAuthorization], chk.Equals, "SharedKeyLite mindgotest:+32DTgsPUgXPo/O7RYaTs0DllA6FTXMj3uK4Qst8y/E=") -} - -func (a *AuthorizationSuite) Test_getSharedKey(c *chk.C) { - // Shared Key Lite for Tables - cli, err := NewBasicClient("mindgotest", "zHDHGs7C+Di9pZSDMuarxJJz3xRBzAHBYaobxpLEc7kwTptR/hPEa9j93hIfb2Tbe9IA50MViGmjQ6nUF/OVvA==") - c.Assert(err, chk.IsNil) - - headers := map[string]string{ - "Accept-Charset": "UTF-8", - headerContentType: "application/json", - headerXmsDate: "Wed, 23 Sep 2015 16:40:05 GMT", - headerContentLength: "0", - headerXmsVersion: "2015-02-21", - "Accept": "application/json;odata=nometadata", - } - url := "https://mindgotest.table.core.windows.net/tquery()" - - key, err := cli.getSharedKey("", url, headers, sharedKeyLiteForTable) - c.Assert(err, chk.IsNil) - c.Assert(key, chk.Equals, "SharedKeyLite mindgotest:+32DTgsPUgXPo/O7RYaTs0DllA6FTXMj3uK4Qst8y/E=") -} - -func (a *AuthorizationSuite) Test_buildCanonicalizedResource(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - - type test struct { - url string - auth authentication - expected string - } - tests := []test{ - // Shared Key - {"https://foo.blob.core.windows.net/path?a=b&c=d", sharedKey, "/foo/path\na:b\nc:d"}, - {"https://foo.blob.core.windows.net/?comp=list", sharedKey, "/foo/\ncomp:list"}, - {"https://foo.blob.core.windows.net/cnt/blob", sharedKey, "/foo/cnt/blob"}, - {"https://foo.blob.core.windows.net/cnt/bl ob", sharedKey, "/foo/cnt/bl%20ob"}, - {"https://foo.blob.core.windows.net/c nt/blob", sharedKey, "/foo/c%20nt/blob"}, - {"https://foo.blob.core.windows.net/cnt/blob%3F%23%5B%5D%21$&%27%28%29%2A blob", sharedKey, "/foo/cnt/blob%3F%23%5B%5D%21$&%27%28%29%2A%20blob"}, - {"https://foo.blob.core.windows.net/cnt/blob-._~:,@;+=blob", sharedKey, "/foo/cnt/blob-._~:,@;+=blob"}, - {"https://foo.blob.core.windows.net/c nt/blob-._~:%3F%23%5B%5D@%21$&%27%28%29%2A,;+=/blob", sharedKey, "/foo/c%20nt/blob-._~:%3F%23%5B%5D@%21$&%27%28%29%2A,;+=/blob"}, - // Shared Key Lite for Table - {"https://foo.table.core.windows.net/mytable", sharedKeyLiteForTable, "/foo/mytable"}, - {"https://foo.table.core.windows.net/mytable?comp=acl", sharedKeyLiteForTable, "/foo/mytable?comp=acl"}, - {"https://foo.table.core.windows.net/mytable?comp=acl&timeout=10", sharedKeyForTable, "/foo/mytable?comp=acl"}, - {"https://foo.table.core.windows.net/mytable(PartitionKey='pkey',RowKey='rowkey%3D')", sharedKeyForTable, "/foo/mytable(PartitionKey='pkey',RowKey='rowkey%3D')"}, - } - - for _, t := range tests { - out, err := cli.buildCanonicalizedResource(t.url, t.auth) - c.Assert(err, chk.IsNil) - c.Assert(out, chk.Equals, t.expected) - } -} - -func (a *AuthorizationSuite) Test_buildCanonicalizedString(c *chk.C) { - var tests = []struct { - verb string - headers map[string]string - canonicalizedResource string - auth authentication - out string - }{ - { - // Shared Key - verb: http.MethodGet, - headers: map[string]string{ - headerXmsDate: "Sun, 11 Oct 2009 21:49:13 GMT", - headerXmsVersion: "2009-09-19", - }, - canonicalizedResource: "/myaccount/ mycontainer\ncomp:metadata\nrestype:container\ntimeout:20", - auth: sharedKey, - out: "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:Sun, 11 Oct 2009 21:49:13 GMT\nx-ms-version:2009-09-19\n/myaccount/ mycontainer\ncomp:metadata\nrestype:container\ntimeout:20", - }, - { - // Shared Key for Tables - verb: http.MethodPut, - headers: map[string]string{ - headerContentType: "text/plain; charset=UTF-8", - headerDate: "Sun, 11 Oct 2009 19:52:39 GMT", - }, - canonicalizedResource: "/testaccount1/Tables", - auth: sharedKeyForTable, - out: "PUT\n\ntext/plain; charset=UTF-8\nSun, 11 Oct 2009 19:52:39 GMT\n/testaccount1/Tables", - }, - { - // Shared Key Lite - verb: http.MethodPut, - headers: map[string]string{ - headerContentType: "text/plain; charset=UTF-8", - headerXmsDate: "Sun, 20 Sep 2009 20:36:40 GMT", - "x-ms-meta-m1": "v1", - "x-ms-meta-m2": "v2", - }, - canonicalizedResource: "/testaccount1/mycontainer/hello.txt", - auth: sharedKeyLite, - out: "PUT\n\ntext/plain; charset=UTF-8\n\nx-ms-date:Sun, 20 Sep 2009 20:36:40 GMT\nx-ms-meta-m1:v1\nx-ms-meta-m2:v2\n/testaccount1/mycontainer/hello.txt", - }, - { - // Shared Key Lite for Tables - verb: "", - headers: map[string]string{ - headerDate: "Sun, 11 Oct 2009 19:52:39 GMT", - }, - canonicalizedResource: "/testaccount1/Tables", - auth: sharedKeyLiteForTable, - out: "Sun, 11 Oct 2009 19:52:39 GMT\n/testaccount1/Tables", - }, - } - - for _, t := range tests { - canonicalizedString, err := buildCanonicalizedString(t.verb, t.headers, t.canonicalizedResource, t.auth) - c.Assert(err, chk.IsNil) - c.Assert(canonicalizedString, chk.Equals, t.out) - } -} - -func (a *AuthorizationSuite) Test_buildCanonicalizedHeader(c *chk.C) { - type test struct { - headers map[string]string - expected string - } - tests := []test{ - {map[string]string{}, - ""}, - {map[string]string{ - "x-ms-foo": "bar"}, - "x-ms-foo:bar"}, - {map[string]string{ - "foo:": "bar"}, - ""}, - {map[string]string{ - "foo:": "bar", - "x-ms-foo": "bar"}, - "x-ms-foo:bar"}, - {map[string]string{ - "x-ms-version": "9999-99-99", - "x-ms-blob-type": "BlockBlob"}, - "x-ms-blob-type:BlockBlob\nx-ms-version:9999-99-99"}} - - for _, i := range tests { - c.Assert(buildCanonicalizedHeader(i.headers), chk.Equals, i.expected) - } -} - -func (a *AuthorizationSuite) Test_createAuthorizationHeader(c *chk.C) { - cli, err := NewBasicClient("foo", base64.StdEncoding.EncodeToString([]byte("bar"))) - c.Assert(err, chk.IsNil) - - canonicalizedString := `foobarzoo` - - c.Assert(cli.createAuthorizationHeader(canonicalizedString, sharedKey), - chk.Equals, `SharedKey foo:h5U0ATVX6SpbFX1H6GNuxIMeXXCILLoIvhflPtuQZ30=`) - c.Assert(cli.createAuthorizationHeader(canonicalizedString, sharedKeyLite), - chk.Equals, `SharedKeyLite foo:h5U0ATVX6SpbFX1H6GNuxIMeXXCILLoIvhflPtuQZ30=`) -} - -func (a *AuthorizationSuite) Test_allSharedKeys(c *chk.C) { - cli := getBasicClient(c) - - blobCli := cli.GetBlobService() - tableCli := cli.GetTableService() - - cnt1 := randContainer() - cnt2 := randContainer() - - tn1 := AzureTable(randTable()) - tn2 := AzureTable(randTable()) - - // Shared Key - c.Assert(blobCli.auth, chk.Equals, sharedKey) - c.Assert(blobCli.CreateContainer(cnt1, ContainerAccessTypePrivate), chk.IsNil) - c.Assert(blobCli.DeleteContainer(cnt1), chk.IsNil) - - // Shared Key for Tables - c.Assert(tableCli.auth, chk.Equals, sharedKeyForTable) - c.Assert(tableCli.CreateTable(tn1), chk.IsNil) - c.Assert(tableCli.DeleteTable(tn1), chk.IsNil) - - // Change to Lite - cli.UseSharedKeyLite = true - blobCli = cli.GetBlobService() - tableCli = cli.GetTableService() - - // Shared Key Lite - c.Assert(blobCli.auth, chk.Equals, sharedKeyLite) - c.Assert(blobCli.CreateContainer(cnt2, ContainerAccessTypeBlob), chk.IsNil) - c.Assert(blobCli.DeleteContainer(cnt2), chk.IsNil) - - // Shared Key Lite for Tables - c.Assert(tableCli.auth, chk.Equals, sharedKeyLiteForTable) - c.Assert(tableCli.CreateTable(tn2), chk.IsNil) - c.Assert(tableCli.DeleteTable(tn2), chk.IsNil) -} diff --git a/storage/blob.go b/storage/blob.go deleted file mode 100644 index e33de1031754..000000000000 --- a/storage/blob.go +++ /dev/null @@ -1,1539 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/xml" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - "time" -) - -// BlobStorageClient contains operations for Microsoft Azure Blob Storage -// Service. -type BlobStorageClient struct { - client Client - auth authentication -} - -// A Container is an entry in ContainerListResponse. -type Container struct { - Name string `xml:"Name"` - Properties ContainerProperties `xml:"Properties"` - // TODO (ahmetalpbalkan) Metadata -} - -// ContainerProperties contains various properties of a container returned from -// various endpoints like ListContainers. -type ContainerProperties struct { - LastModified string `xml:"Last-Modified"` - Etag string `xml:"Etag"` - LeaseStatus string `xml:"LeaseStatus"` - LeaseState string `xml:"LeaseState"` - LeaseDuration string `xml:"LeaseDuration"` - // TODO (ahmetalpbalkan) remaining fields -} - -// ContainerListResponse contains the response fields from -// ListContainers call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx -type ContainerListResponse struct { - XMLName xml.Name `xml:"EnumerationResults"` - Xmlns string `xml:"xmlns,attr"` - Prefix string `xml:"Prefix"` - Marker string `xml:"Marker"` - NextMarker string `xml:"NextMarker"` - MaxResults int64 `xml:"MaxResults"` - Containers []Container `xml:"Containers>Container"` -} - -// A Blob is an entry in BlobListResponse. -type Blob struct { - Name string `xml:"Name"` - Properties BlobProperties `xml:"Properties"` - Metadata BlobMetadata `xml:"Metadata"` -} - -// BlobMetadata is a set of custom name/value pairs. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179404.aspx -type BlobMetadata map[string]string - -type blobMetadataEntries struct { - Entries []blobMetadataEntry `xml:",any"` -} -type blobMetadataEntry struct { - XMLName xml.Name - Value string `xml:",chardata"` -} - -// UnmarshalXML converts the xml:Metadata into Metadata map -func (bm *BlobMetadata) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - var entries blobMetadataEntries - if err := d.DecodeElement(&entries, &start); err != nil { - return err - } - for _, entry := range entries.Entries { - if *bm == nil { - *bm = make(BlobMetadata) - } - (*bm)[strings.ToLower(entry.XMLName.Local)] = entry.Value - } - return nil -} - -// MarshalXML implements the xml.Marshaler interface. It encodes -// metadata name/value pairs as they would appear in an Azure -// ListBlobs response. -func (bm BlobMetadata) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { - entries := make([]blobMetadataEntry, 0, len(bm)) - for k, v := range bm { - entries = append(entries, blobMetadataEntry{ - XMLName: xml.Name{Local: http.CanonicalHeaderKey(k)}, - Value: v, - }) - } - return enc.EncodeElement(blobMetadataEntries{ - Entries: entries, - }, start) -} - -// BlobProperties contains various properties of a blob -// returned in various endpoints like ListBlobs or GetBlobProperties. -type BlobProperties struct { - LastModified string `xml:"Last-Modified"` - Etag string `xml:"Etag"` - ContentMD5 string `xml:"Content-MD5"` - ContentLength int64 `xml:"Content-Length"` - ContentType string `xml:"Content-Type"` - ContentEncoding string `xml:"Content-Encoding"` - CacheControl string `xml:"Cache-Control"` - ContentLanguage string `xml:"Cache-Language"` - BlobType BlobType `xml:"x-ms-blob-blob-type"` - SequenceNumber int64 `xml:"x-ms-blob-sequence-number"` - CopyID string `xml:"CopyId"` - CopyStatus string `xml:"CopyStatus"` - CopySource string `xml:"CopySource"` - CopyProgress string `xml:"CopyProgress"` - CopyCompletionTime string `xml:"CopyCompletionTime"` - CopyStatusDescription string `xml:"CopyStatusDescription"` - LeaseStatus string `xml:"LeaseStatus"` - LeaseState string `xml:"LeaseState"` -} - -// BlobHeaders contains various properties of a blob and is an entry -// in SetBlobProperties -type BlobHeaders struct { - ContentMD5 string `header:"x-ms-blob-content-md5"` - ContentLanguage string `header:"x-ms-blob-content-language"` - ContentEncoding string `header:"x-ms-blob-content-encoding"` - ContentType string `header:"x-ms-blob-content-type"` - CacheControl string `header:"x-ms-blob-cache-control"` -} - -// BlobListResponse contains the response fields from ListBlobs call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx -type BlobListResponse struct { - XMLName xml.Name `xml:"EnumerationResults"` - Xmlns string `xml:"xmlns,attr"` - Prefix string `xml:"Prefix"` - Marker string `xml:"Marker"` - NextMarker string `xml:"NextMarker"` - MaxResults int64 `xml:"MaxResults"` - Blobs []Blob `xml:"Blobs>Blob"` - - // BlobPrefix is used to traverse blobs as if it were a file system. - // It is returned if ListBlobsParameters.Delimiter is specified. - // The list here can be thought of as "folders" that may contain - // other folders or blobs. - BlobPrefixes []string `xml:"Blobs>BlobPrefix>Name"` - - // Delimiter is used to traverse blobs as if it were a file system. - // It is returned if ListBlobsParameters.Delimiter is specified. - Delimiter string `xml:"Delimiter"` -} - -// ListContainersParameters defines the set of customizable parameters to make a -// List Containers call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx -type ListContainersParameters struct { - Prefix string - Marker string - Include string - MaxResults uint - Timeout uint -} - -func (p ListContainersParameters) getParameters() url.Values { - out := url.Values{} - - if p.Prefix != "" { - out.Set("prefix", p.Prefix) - } - if p.Marker != "" { - out.Set("marker", p.Marker) - } - if p.Include != "" { - out.Set("include", p.Include) - } - if p.MaxResults != 0 { - out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults)) - } - if p.Timeout != 0 { - out.Set("timeout", fmt.Sprintf("%v", p.Timeout)) - } - - return out -} - -// ListBlobsParameters defines the set of customizable -// parameters to make a List Blobs call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx -type ListBlobsParameters struct { - Prefix string - Delimiter string - Marker string - Include string - MaxResults uint - Timeout uint -} - -func (p ListBlobsParameters) getParameters() url.Values { - out := url.Values{} - - if p.Prefix != "" { - out.Set("prefix", p.Prefix) - } - if p.Delimiter != "" { - out.Set("delimiter", p.Delimiter) - } - if p.Marker != "" { - out.Set("marker", p.Marker) - } - if p.Include != "" { - out.Set("include", p.Include) - } - if p.MaxResults != 0 { - out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults)) - } - if p.Timeout != 0 { - out.Set("timeout", fmt.Sprintf("%v", p.Timeout)) - } - - return out -} - -// BlobType defines the type of the Azure Blob. -type BlobType string - -// Types of page blobs -const ( - BlobTypeBlock BlobType = "BlockBlob" - BlobTypePage BlobType = "PageBlob" - BlobTypeAppend BlobType = "AppendBlob" -) - -// PageWriteType defines the type updates that are going to be -// done on the page blob. -type PageWriteType string - -// Types of operations on page blobs -const ( - PageWriteTypeUpdate PageWriteType = "update" - PageWriteTypeClear PageWriteType = "clear" -) - -const ( - blobCopyStatusPending = "pending" - blobCopyStatusSuccess = "success" - blobCopyStatusAborted = "aborted" - blobCopyStatusFailed = "failed" -) - -// lease constants. -const ( - leaseHeaderPrefix = "x-ms-lease-" - headerLeaseID = "x-ms-lease-id" - leaseAction = "x-ms-lease-action" - leaseBreakPeriod = "x-ms-lease-break-period" - leaseDuration = "x-ms-lease-duration" - leaseProposedID = "x-ms-proposed-lease-id" - leaseTime = "x-ms-lease-time" - - acquireLease = "acquire" - renewLease = "renew" - changeLease = "change" - releaseLease = "release" - breakLease = "break" -) - -// BlockListType is used to filter out types of blocks in a Get Blocks List call -// for a block blob. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx for all -// block types. -type BlockListType string - -// Filters for listing blocks in block blobs -const ( - BlockListTypeAll BlockListType = "all" - BlockListTypeCommitted BlockListType = "committed" - BlockListTypeUncommitted BlockListType = "uncommitted" -) - -// ContainerAccessType defines the access level to the container from a public -// request. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx and "x-ms- -// blob-public-access" header. -type ContainerAccessType string - -// Access options for containers -const ( - ContainerAccessTypePrivate ContainerAccessType = "" - ContainerAccessTypeBlob ContainerAccessType = "blob" - ContainerAccessTypeContainer ContainerAccessType = "container" -) - -// ContainerAccessPolicyDetails are used for SETTING container policies -type ContainerAccessPolicyDetails struct { - ID string - StartTime time.Time - ExpiryTime time.Time - CanRead bool - CanWrite bool - CanDelete bool -} - -// ContainerPermissions is used when setting permissions and Access Policies for containers. -type ContainerPermissions struct { - AccessType ContainerAccessType - AccessPolicies []ContainerAccessPolicyDetails -} - -// ContainerAccessHeader references header used when setting/getting container ACL -const ( - ContainerAccessHeader string = "x-ms-blob-public-access" -) - -// Maximum sizes (per REST API) for various concepts -const ( - MaxBlobBlockSize = 4 * 1024 * 1024 - MaxBlobPageSize = 4 * 1024 * 1024 -) - -// BlockStatus defines states a block for a block blob can -// be in. -type BlockStatus string - -// List of statuses that can be used to refer to a block in a block list -const ( - BlockStatusUncommitted BlockStatus = "Uncommitted" - BlockStatusCommitted BlockStatus = "Committed" - BlockStatusLatest BlockStatus = "Latest" -) - -// Block is used to create Block entities for Put Block List -// call. -type Block struct { - ID string - Status BlockStatus -} - -// BlockListResponse contains the response fields from Get Block List call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx -type BlockListResponse struct { - XMLName xml.Name `xml:"BlockList"` - CommittedBlocks []BlockResponse `xml:"CommittedBlocks>Block"` - UncommittedBlocks []BlockResponse `xml:"UncommittedBlocks>Block"` -} - -// BlockResponse contains the block information returned -// in the GetBlockListCall. -type BlockResponse struct { - Name string `xml:"Name"` - Size int64 `xml:"Size"` -} - -// GetPageRangesResponse contains the response fields from -// Get Page Ranges call. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx -type GetPageRangesResponse struct { - XMLName xml.Name `xml:"PageList"` - PageList []PageRange `xml:"PageRange"` -} - -// PageRange contains information about a page of a page blob from -// Get Pages Range call. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx -type PageRange struct { - Start int64 `xml:"Start"` - End int64 `xml:"End"` -} - -var ( - errBlobCopyAborted = errors.New("storage: blob copy is aborted") - errBlobCopyIDMismatch = errors.New("storage: blob copy id is a mismatch") -) - -// ListContainers returns the list of containers in a storage account along with -// pagination token and other response details. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx -func (b BlobStorageClient) ListContainers(params ListContainersParameters) (ContainerListResponse, error) { - q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}}) - uri := b.client.getEndpoint(blobServiceName, "", q) - headers := b.client.getStandardHeaders() - - var out ContainerListResponse - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return out, err - } - defer resp.body.Close() - - err = xmlUnmarshal(resp.body, &out) - return out, err -} - -// CreateContainer creates a blob container within the storage account -// with given name and access level. Returns error if container already exists. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx -func (b BlobStorageClient) CreateContainer(name string, access ContainerAccessType) error { - resp, err := b.createContainer(name, access) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// CreateContainerIfNotExists creates a blob container if it does not exist. Returns -// true if container is newly created or false if container already exists. -func (b BlobStorageClient) CreateContainerIfNotExists(name string, access ContainerAccessType) (bool, error) { - resp, err := b.createContainer(name, access) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict { - return resp.statusCode == http.StatusCreated, nil - } - } - return false, err -} - -func (b BlobStorageClient) createContainer(name string, access ContainerAccessType) (*storageResponse, error) { - uri := b.client.getEndpoint(blobServiceName, pathForContainer(name), url.Values{"restype": {"container"}}) - - headers := b.client.getStandardHeaders() - if access != "" { - headers[ContainerAccessHeader] = string(access) - } - return b.client.exec(http.MethodPut, uri, headers, nil, b.auth) -} - -// ContainerExists returns true if a container with given name exists -// on the storage account, otherwise returns false. -func (b BlobStorageClient) ContainerExists(name string) (bool, error) { - uri := b.client.getEndpoint(blobServiceName, pathForContainer(name), url.Values{"restype": {"container"}}) - headers := b.client.getStandardHeaders() - - resp, err := b.client.exec(http.MethodHead, uri, headers, nil, b.auth) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusOK, nil - } - } - return false, err -} - -// SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391.aspx -func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions, timeout int, leaseID string) (err error) { - params := url.Values{ - "restype": {"container"}, - "comp": {"acl"}, - } - - if timeout > 0 { - params.Add("timeout", strconv.Itoa(timeout)) - } - - uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) - headers := b.client.getStandardHeaders() - if containerPermissions.AccessType != "" { - headers[ContainerAccessHeader] = string(containerPermissions.AccessType) - } - - if leaseID != "" { - headers[headerLeaseID] = leaseID - } - - body, length, err := generateContainerACLpayload(containerPermissions.AccessPolicies) - headers["Content-Length"] = strconv.Itoa(length) - resp, err := b.client.exec(http.MethodPut, uri, headers, body, b.auth) - - if err != nil { - return err - } - - if resp != nil { - defer resp.body.Close() - - if resp.statusCode != http.StatusOK { - return errors.New("Unable to set permissions") - } - } - return nil -} - -// GetContainerPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx -// If timeout is 0 then it will not be passed to Azure -// leaseID will only be passed to Azure if populated -// Returns permissionResponse which is combined permissions and AccessPolicy -func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (*ContainerPermissions, error) { - params := url.Values{"restype": {"container"}, - "comp": {"acl"}} - - if timeout > 0 { - params.Add("timeout", strconv.Itoa(timeout)) - } - - uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) - headers := b.client.getStandardHeaders() - - if leaseID != "" { - headers[headerLeaseID] = leaseID - } - - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - var out AccessPolicy - err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList) - if err != nil { - return nil, err - } - - permissionResponse := updateContainerAccessPolicy(out, &resp.headers) - return &permissionResponse, nil -} - -func updateContainerAccessPolicy(ap AccessPolicy, headers *http.Header) ContainerPermissions { - // containerAccess. Blob, Container, empty - containerAccess := headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader)) - - var cp ContainerPermissions - cp.AccessType = ContainerAccessType(containerAccess) - for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers { - capd := ContainerAccessPolicyDetails{ - ID: policy.ID, - StartTime: policy.AccessPolicy.StartTime, - ExpiryTime: policy.AccessPolicy.ExpiryTime, - } - capd.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r") - capd.CanWrite = updatePermissions(policy.AccessPolicy.Permission, "w") - capd.CanDelete = updatePermissions(policy.AccessPolicy.Permission, "d") - - cp.AccessPolicies = append(cp.AccessPolicies, capd) - } - - return cp -} - -// DeleteContainer deletes the container with given name on the storage -// account. If the container does not exist returns error. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx -func (b BlobStorageClient) DeleteContainer(name string) error { - resp, err := b.deleteContainer(name) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) -} - -// DeleteContainerIfExists deletes the container with given name on the storage -// account if it exists. Returns true if container is deleted with this call, or -// false if the container did not exist at the time of the Delete Container -// operation. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx -func (b BlobStorageClient) DeleteContainerIfExists(name string) (bool, error) { - resp, err := b.deleteContainer(name) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusAccepted, nil - } - } - return false, err -} - -func (b BlobStorageClient) deleteContainer(name string) (*storageResponse, error) { - uri := b.client.getEndpoint(blobServiceName, pathForContainer(name), url.Values{"restype": {"container"}}) - - headers := b.client.getStandardHeaders() - return b.client.exec(http.MethodDelete, uri, headers, nil, b.auth) -} - -// ListBlobs returns an object that contains list of blobs in the container, -// pagination token and other information in the response of List Blobs call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx -func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameters) (BlobListResponse, error) { - q := mergeParams(params.getParameters(), url.Values{ - "restype": {"container"}, - "comp": {"list"}}) - uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), q) - headers := b.client.getStandardHeaders() - - var out BlobListResponse - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return out, err - } - defer resp.body.Close() - - err = xmlUnmarshal(resp.body, &out) - return out, err -} - -// BlobExists returns true if a blob with given name exists on the specified -// container of the storage account. -func (b BlobStorageClient) BlobExists(container, name string) (bool, error) { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - headers := b.client.getStandardHeaders() - resp, err := b.client.exec(http.MethodHead, uri, headers, nil, b.auth) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusOK, nil - } - } - return false, err -} - -// GetBlobURL gets the canonical URL to the blob with the specified name in the -// specified container. This method does not create a publicly accessible URL if -// the blob or container is private and this method does not check if the blob -// exists. -func (b BlobStorageClient) GetBlobURL(container, name string) string { - if container == "" { - container = "$root" - } - return b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) -} - -// GetBlob returns a stream to read the blob. Caller must call Close() the -// reader to close on the underlying connection. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx -func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error) { - resp, err := b.getBlobRange(container, name, "", nil) - if err != nil { - return nil, err - } - - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } - return resp.body, nil -} - -// GetBlobRange reads the specified range of a blob to a stream. The bytesRange -// string must be in a format like "0-", "10-100" as defined in HTTP 1.1 spec. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx -func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string, extraHeaders map[string]string) (io.ReadCloser, error) { - resp, err := b.getBlobRange(container, name, bytesRange, extraHeaders) - if err != nil { - return nil, err - } - - if err := checkRespCode(resp.statusCode, []int{http.StatusPartialContent}); err != nil { - return nil, err - } - return resp.body, nil -} - -func (b BlobStorageClient) getBlobRange(container, name, bytesRange string, extraHeaders map[string]string) (*storageResponse, error) { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - if bytesRange != "" { - headers["Range"] = fmt.Sprintf("bytes=%s", bytesRange) - } - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return nil, err - } - return resp, err -} - -// leasePut is common PUT code for the various acquire/release/break etc functions. -func (b BlobStorageClient) leaseCommonPut(container string, name string, headers map[string]string, expectedStatus int) (http.Header, error) { - params := url.Values{"comp": {"lease"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{expectedStatus}); err != nil { - return nil, err - } - - return resp.headers, nil -} - -// SnapshotBlob creates a snapshot for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691971.aspx -func (b BlobStorageClient) SnapshotBlob(container string, name string, timeout int, extraHeaders map[string]string) (snapshotTimestamp *time.Time, err error) { - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - params := url.Values{"comp": {"snapshot"}} - - if timeout > 0 { - params.Add("timeout", strconv.Itoa(timeout)) - } - - for k, v := range extraHeaders { - headers[k] = v - } - - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return nil, err - } - - if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil { - return nil, err - } - - snapshotResponse := resp.headers.Get(http.CanonicalHeaderKey("x-ms-snapshot")) - if snapshotResponse != "" { - snapshotTimestamp, err := time.Parse(time.RFC3339, snapshotResponse) - if err != nil { - return nil, err - } - - return &snapshotTimestamp, nil - } - - return nil, errors.New("Snapshot not created") -} - -// AcquireLease creates a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -// returns leaseID acquired -func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (returnedLeaseID string, err error) { - headers := b.client.getStandardHeaders() - headers[leaseAction] = acquireLease - - if leaseTimeInSeconds > 0 { - headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds) - } - - if proposedLeaseID != "" { - headers[leaseProposedID] = proposedLeaseID - } - - respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusCreated) - if err != nil { - return "", err - } - - returnedLeaseID = respHeaders.Get(http.CanonicalHeaderKey(headerLeaseID)) - - if returnedLeaseID != "" { - return returnedLeaseID, nil - } - - return "", errors.New("LeaseID not returned") -} - -// BreakLease breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -// Returns the timeout remaining in the lease in seconds -func (b BlobStorageClient) BreakLease(container string, name string) (breakTimeout int, err error) { - headers := b.client.getStandardHeaders() - headers[leaseAction] = breakLease - return b.breakLeaseCommon(container, name, headers) -} - -// BreakLeaseWithBreakPeriod breaks the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -// breakPeriodInSeconds is used to determine how long until new lease can be created. -// Returns the timeout remaining in the lease in seconds -func (b BlobStorageClient) BreakLeaseWithBreakPeriod(container string, name string, breakPeriodInSeconds int) (breakTimeout int, err error) { - headers := b.client.getStandardHeaders() - headers[leaseAction] = breakLease - headers[leaseBreakPeriod] = strconv.Itoa(breakPeriodInSeconds) - return b.breakLeaseCommon(container, name, headers) -} - -// breakLeaseCommon is common code for both version of BreakLease (with and without break period) -func (b BlobStorageClient) breakLeaseCommon(container string, name string, headers map[string]string) (breakTimeout int, err error) { - - respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusAccepted) - if err != nil { - return 0, err - } - - breakTimeoutStr := respHeaders.Get(http.CanonicalHeaderKey(leaseTime)) - if breakTimeoutStr != "" { - breakTimeout, err = strconv.Atoi(breakTimeoutStr) - if err != nil { - return 0, err - } - } - - return breakTimeout, nil -} - -// ChangeLease changes a lease ID for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -// Returns the new LeaseID acquired -func (b BlobStorageClient) ChangeLease(container string, name string, currentLeaseID string, proposedLeaseID string) (newLeaseID string, err error) { - headers := b.client.getStandardHeaders() - headers[leaseAction] = changeLease - headers[headerLeaseID] = currentLeaseID - headers[leaseProposedID] = proposedLeaseID - - respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusOK) - if err != nil { - return "", err - } - - newLeaseID = respHeaders.Get(http.CanonicalHeaderKey(headerLeaseID)) - if newLeaseID != "" { - return newLeaseID, nil - } - - return "", errors.New("LeaseID not returned") -} - -// ReleaseLease releases the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) ReleaseLease(container string, name string, currentLeaseID string) error { - headers := b.client.getStandardHeaders() - headers[leaseAction] = releaseLease - headers[headerLeaseID] = currentLeaseID - - _, err := b.leaseCommonPut(container, name, headers, http.StatusOK) - if err != nil { - return err - } - - return nil -} - -// RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx -func (b BlobStorageClient) RenewLease(container string, name string, currentLeaseID string) error { - headers := b.client.getStandardHeaders() - headers[leaseAction] = renewLease - headers[headerLeaseID] = currentLeaseID - - _, err := b.leaseCommonPut(container, name, headers, http.StatusOK) - if err != nil { - return err - } - - return nil -} - -// GetBlobProperties provides various information about the specified -// blob. See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx -func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobProperties, error) { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - - headers := b.client.getStandardHeaders() - resp, err := b.client.exec(http.MethodHead, uri, headers, nil, b.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } - - var contentLength int64 - contentLengthStr := resp.headers.Get("Content-Length") - if contentLengthStr != "" { - contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64) - if err != nil { - return nil, err - } - } - - var sequenceNum int64 - sequenceNumStr := resp.headers.Get("x-ms-blob-sequence-number") - if sequenceNumStr != "" { - sequenceNum, err = strconv.ParseInt(sequenceNumStr, 0, 64) - if err != nil { - return nil, err - } - } - - return &BlobProperties{ - LastModified: resp.headers.Get("Last-Modified"), - Etag: resp.headers.Get("Etag"), - ContentMD5: resp.headers.Get("Content-MD5"), - ContentLength: contentLength, - ContentEncoding: resp.headers.Get("Content-Encoding"), - ContentType: resp.headers.Get("Content-Type"), - CacheControl: resp.headers.Get("Cache-Control"), - ContentLanguage: resp.headers.Get("Content-Language"), - SequenceNumber: sequenceNum, - CopyCompletionTime: resp.headers.Get("x-ms-copy-completion-time"), - CopyStatusDescription: resp.headers.Get("x-ms-copy-status-description"), - CopyID: resp.headers.Get("x-ms-copy-id"), - CopyProgress: resp.headers.Get("x-ms-copy-progress"), - CopySource: resp.headers.Get("x-ms-copy-source"), - CopyStatus: resp.headers.Get("x-ms-copy-status"), - BlobType: BlobType(resp.headers.Get("x-ms-blob-type")), - LeaseStatus: resp.headers.Get("x-ms-lease-status"), - LeaseState: resp.headers.Get("x-ms-lease-state"), - }, nil -} - -// SetBlobProperties replaces the BlobHeaders for the specified blob. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by GetBlobProperties. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee691966.aspx -func (b BlobStorageClient) SetBlobProperties(container, name string, blobHeaders BlobHeaders) error { - params := url.Values{"comp": {"properties"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - headers := b.client.getStandardHeaders() - - extraHeaders := headersFromStruct(blobHeaders) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusOK}) -} - -// SetBlobMetadata replaces the metadata for the specified blob. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by GetBlobMetadata. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx -func (b BlobStorageClient) SetBlobMetadata(container, name string, metadata map[string]string, extraHeaders map[string]string) error { - params := url.Values{"comp": {"metadata"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - metadata = b.client.protectUserAgent(metadata) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - for k, v := range metadata { - headers[userDefinedMetadataHeaderPrefix+k] = v - } - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusOK}) -} - -// GetBlobMetadata returns all user-defined metadata for the specified blob. -// -// All metadata keys will be returned in lower case. (HTTP header -// names are case-insensitive.) -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx -func (b BlobStorageClient) GetBlobMetadata(container, name string) (map[string]string, error) { - params := url.Values{"comp": {"metadata"}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - headers := b.client.getStandardHeaders() - - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } - - metadata := make(map[string]string) - for k, v := range resp.headers { - // Can't trust CanonicalHeaderKey() to munge case - // reliably. "_" is allowed in identifiers: - // https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx - // https://msdn.microsoft.com/library/aa664670(VS.71).aspx - // http://tools.ietf.org/html/rfc7230#section-3.2 - // ...but "_" is considered invalid by - // CanonicalMIMEHeaderKey in - // https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542 - // so k can be "X-Ms-Meta-Foo" or "x-ms-meta-foo_bar". - k = strings.ToLower(k) - if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) { - continue - } - // metadata["foo"] = content of the last X-Ms-Meta-Foo header - k = k[len(userDefinedMetadataHeaderPrefix):] - metadata[k] = v[len(v)-1] - } - return metadata, nil -} - -// CreateBlockBlob initializes an empty block blob with no blocks. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx -func (b BlobStorageClient) CreateBlockBlob(container, name string) error { - return b.CreateBlockBlobFromReader(container, name, 0, nil, nil) -} - -// CreateBlockBlobFromReader initializes a block blob using data from -// reader. Size must be the number of bytes read from reader. To -// create an empty blob, use size==0 and reader==nil. -// -// The API rejects requests with size > 64 MiB (but this limit is not -// checked by the SDK). To write a larger blob, use CreateBlockBlob, -// PutBlock, and PutBlockList. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx -func (b BlobStorageClient) CreateBlockBlobFromReader(container, name string, size uint64, blob io.Reader, extraHeaders map[string]string) error { - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypeBlock) - headers["Content-Length"] = fmt.Sprintf("%d", size) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, blob, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// PutBlock saves the given data chunk to the specified block blob with -// given ID. -// -// The API rejects chunks larger than 4 MiB (but this limit is not -// checked by the SDK). -// -// See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx -func (b BlobStorageClient) PutBlock(container, name, blockID string, chunk []byte) error { - return b.PutBlockWithLength(container, name, blockID, uint64(len(chunk)), bytes.NewReader(chunk), nil) -} - -// PutBlockWithLength saves the given data stream of exactly specified size to -// the block blob with given ID. It is an alternative to PutBlocks where data -// comes as stream but the length is known in advance. -// -// The API rejects requests with size > 4 MiB (but this limit is not -// checked by the SDK). -// -// See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx -func (b BlobStorageClient) PutBlockWithLength(container, name, blockID string, size uint64, blob io.Reader, extraHeaders map[string]string) error { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{"comp": {"block"}, "blockid": {blockID}}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypeBlock) - headers["Content-Length"] = fmt.Sprintf("%v", size) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, blob, b.auth) - if err != nil { - return err - } - - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// PutBlockList saves list of blocks to the specified block blob. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179467.aspx -func (b BlobStorageClient) PutBlockList(container, name string, blocks []Block) error { - blockListXML := prepareBlockListRequest(blocks) - - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{"comp": {"blocklist"}}) - headers := b.client.getStandardHeaders() - headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXML)) - - resp, err := b.client.exec(http.MethodPut, uri, headers, strings.NewReader(blockListXML), b.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// GetBlockList retrieves list of blocks in the specified block blob. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx -func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockListType) (BlockListResponse, error) { - params := url.Values{"comp": {"blocklist"}, "blocklisttype": {string(blockType)}} - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - headers := b.client.getStandardHeaders() - - var out BlockListResponse - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return out, err - } - defer resp.body.Close() - - err = xmlUnmarshal(resp.body, &out) - return out, err -} - -// PutPageBlob initializes an empty page blob with specified name and maximum -// size in bytes (size must be aligned to a 512-byte boundary). A page blob must -// be created using this method before writing pages. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx -func (b BlobStorageClient) PutPageBlob(container, name string, size int64, extraHeaders map[string]string) error { - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypePage) - headers["x-ms-blob-content-length"] = fmt.Sprintf("%v", size) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// PutPage writes a range of pages to a page blob or clears the given range. -// In case of 'clear' writes, given chunk is discarded. Ranges must be aligned -// with 512-byte boundaries and chunk must be of size multiplies by 512. -// -// See https://msdn.microsoft.com/en-us/library/ee691975.aspx -func (b BlobStorageClient) PutPage(container, name string, startByte, endByte int64, writeType PageWriteType, chunk []byte, extraHeaders map[string]string) error { - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"page"}}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypePage) - headers["x-ms-page-write"] = string(writeType) - headers["x-ms-range"] = fmt.Sprintf("bytes=%v-%v", startByte, endByte) - for k, v := range extraHeaders { - headers[k] = v - } - var contentLength int64 - var data io.Reader - if writeType == PageWriteTypeClear { - contentLength = 0 - data = bytes.NewReader([]byte{}) - } else { - contentLength = int64(len(chunk)) - data = bytes.NewReader(chunk) - } - headers["Content-Length"] = fmt.Sprintf("%v", contentLength) - - resp, err := b.client.exec(http.MethodPut, uri, headers, data, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// GetPageRanges returns the list of valid page ranges for a page blob. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx -func (b BlobStorageClient) GetPageRanges(container, name string) (GetPageRangesResponse, error) { - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"pagelist"}}) - headers := b.client.getStandardHeaders() - - var out GetPageRangesResponse - resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth) - if err != nil { - return out, err - } - defer resp.body.Close() - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return out, err - } - err = xmlUnmarshal(resp.body, &out) - return out, err -} - -// PutAppendBlob initializes an empty append blob with specified name. An -// append blob must be created using this method before appending blocks. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx -func (b BlobStorageClient) PutAppendBlob(container, name string, extraHeaders map[string]string) error { - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypeAppend) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// AppendBlock appends a block to an append blob. -// -// See https://msdn.microsoft.com/en-us/library/azure/mt427365.aspx -func (b BlobStorageClient) AppendBlock(container, name string, chunk []byte, extraHeaders map[string]string) error { - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"appendblock"}}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypeAppend) - headers["Content-Length"] = fmt.Sprintf("%v", len(chunk)) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, bytes.NewReader(chunk), b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// CopyBlob starts a blob copy operation and waits for the operation to -// complete. sourceBlob parameter must be a canonical URL to the blob (can be -// obtained using GetBlobURL method.) There is no SLA on blob copy and therefore -// this helper method works faster on smaller files. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx -func (b BlobStorageClient) CopyBlob(container, name, sourceBlob string) error { - copyID, err := b.StartBlobCopy(container, name, sourceBlob) - if err != nil { - return err - } - - return b.WaitForBlobCopy(container, name, copyID) -} - -// StartBlobCopy starts a blob copy operation. -// sourceBlob parameter must be a canonical URL to the blob (can be -// obtained using GetBlobURL method.) -// -// See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx -func (b BlobStorageClient) StartBlobCopy(container, name, sourceBlob string) (string, error) { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - - headers := b.client.getStandardHeaders() - headers["x-ms-copy-source"] = sourceBlob - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return "", err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusAccepted, http.StatusCreated}); err != nil { - return "", err - } - - copyID := resp.headers.Get("x-ms-copy-id") - if copyID == "" { - return "", errors.New("Got empty copy id header") - } - return copyID, nil -} - -// AbortBlobCopy aborts a BlobCopy which has already been triggered by the StartBlobCopy function. -// copyID is generated from StartBlobCopy function. -// currentLeaseID is required IF the destination blob has an active lease on it. -// As defined in https://msdn.microsoft.com/en-us/library/azure/jj159098.aspx -func (b BlobStorageClient) AbortBlobCopy(container, name, copyID, currentLeaseID string, timeout int) error { - params := url.Values{"comp": {"copy"}, "copyid": {copyID}} - if timeout > 0 { - params.Add("timeout", strconv.Itoa(timeout)) - } - - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) - headers := b.client.getStandardHeaders() - headers["x-ms-copy-action"] = "abort" - - if currentLeaseID != "" { - headers[headerLeaseID] = currentLeaseID - } - - resp, err := b.client.exec(http.MethodPut, uri, headers, nil, b.auth) - if err != nil { - return err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { - return err - } - - return nil -} - -// WaitForBlobCopy loops until a BlobCopy operation is completed (or fails with error) -func (b BlobStorageClient) WaitForBlobCopy(container, name, copyID string) error { - for { - props, err := b.GetBlobProperties(container, name) - if err != nil { - return err - } - - if props.CopyID != copyID { - return errBlobCopyIDMismatch - } - - switch props.CopyStatus { - case blobCopyStatusSuccess: - return nil - case blobCopyStatusPending: - continue - case blobCopyStatusAborted: - return errBlobCopyAborted - case blobCopyStatusFailed: - return fmt.Errorf("storage: blob copy failed. Id=%s Description=%s", props.CopyID, props.CopyStatusDescription) - default: - return fmt.Errorf("storage: unhandled blob copy status: '%s'", props.CopyStatus) - } - } -} - -// DeleteBlob deletes the given blob from the specified container. -// If the blob does not exists at the time of the Delete Blob operation, it -// returns error. See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx -func (b BlobStorageClient) DeleteBlob(container, name string, extraHeaders map[string]string) error { - resp, err := b.deleteBlob(container, name, extraHeaders) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) -} - -// DeleteBlobIfExists deletes the given blob from the specified container If the -// blob is deleted with this call, returns true. Otherwise returns false. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx -func (b BlobStorageClient) DeleteBlobIfExists(container, name string, extraHeaders map[string]string) (bool, error) { - resp, err := b.deleteBlob(container, name, extraHeaders) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusAccepted, nil - } - } - return false, err -} - -func (b BlobStorageClient) deleteBlob(container, name string, extraHeaders map[string]string) (*storageResponse, error) { - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - extraHeaders = b.client.protectUserAgent(extraHeaders) - headers := b.client.getStandardHeaders() - for k, v := range extraHeaders { - headers[k] = v - } - - return b.client.exec(http.MethodDelete, uri, headers, nil, b.auth) -} - -// helper method to construct the path to a container given its name -func pathForContainer(name string) string { - return fmt.Sprintf("/%s", name) -} - -// helper method to construct the path to a blob given its container and blob -// name -func pathForBlob(container, name string) string { - return fmt.Sprintf("/%s/%s", container, name) -} - -// GetBlobSASURIWithSignedIPAndProtocol creates an URL to the specified blob which contains the Shared -// Access Signature with specified permissions and expiration time. Also includes signedIPRange and allowed protocols. -// If old API version is used but no signedIP is passed (ie empty string) then this should still work. -// We only populate the signedIP when it non-empty. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx -func (b BlobStorageClient) GetBlobSASURIWithSignedIPAndProtocol(container, name string, expiry time.Time, permissions string, signedIPRange string, HTTPSOnly bool) (string, error) { - var ( - signedPermissions = permissions - blobURL = b.GetBlobURL(container, name) - ) - canonicalizedResource, err := b.client.buildCanonicalizedResource(blobURL, b.auth) - if err != nil { - return "", err - } - - // "The canonicalizedresouce portion of the string is a canonical path to the signed resource. - // It must include the service name (blob, table, queue or file) for version 2015-02-21 or - // later, the storage account name, and the resource name, and must be URL-decoded. - // -- https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx - - // We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component). - canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1) - canonicalizedResource, err = url.QueryUnescape(canonicalizedResource) - if err != nil { - return "", err - } - - signedExpiry := expiry.UTC().Format(time.RFC3339) - signedResource := "b" - - protocols := "https,http" - if HTTPSOnly { - protocols = "https" - } - stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions, signedIPRange, protocols) - if err != nil { - return "", err - } - - sig := b.client.computeHmac256(stringToSign) - sasParams := url.Values{ - "sv": {b.client.apiVersion}, - "se": {signedExpiry}, - "sr": {signedResource}, - "sp": {signedPermissions}, - "sig": {sig}, - } - - if b.client.apiVersion >= "2015-04-05" { - sasParams.Add("spr", protocols) - if signedIPRange != "" { - sasParams.Add("sip", signedIPRange) - } - } - - sasURL, err := url.Parse(blobURL) - if err != nil { - return "", err - } - sasURL.RawQuery = sasParams.Encode() - return sasURL.String(), nil -} - -// GetBlobSASURI creates an URL to the specified blob which contains the Shared -// Access Signature with specified permissions and expiration time. -// -// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx -func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) { - url, err := b.GetBlobSASURIWithSignedIPAndProtocol(container, name, expiry, permissions, "", false) - return url, err -} - -func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string, signedIP string, protocols string) (string, error) { - var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string - - if signedVersion >= "2015-02-21" { - canonicalizedResource = "/blob" + canonicalizedResource - } - - // https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12 - if signedVersion >= "2015-04-05" { - return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion, rscc, rscd, rsce, rscl, rsct), nil - } - - // reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx - if signedVersion >= "2013-08-15" { - return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil - } - - return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") -} - -func generateContainerACLpayload(policies []ContainerAccessPolicyDetails) (io.Reader, int, error) { - sil := SignedIdentifiers{ - SignedIdentifiers: []SignedIdentifier{}, - } - for _, capd := range policies { - permission := capd.generateContainerPermissions() - signedIdentifier := convertAccessPolicyToXMLStructs(capd.ID, capd.StartTime, capd.ExpiryTime, permission) - sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier) - } - return xmlMarshal(sil) -} - -func (capd *ContainerAccessPolicyDetails) generateContainerPermissions() (permissions string) { - // generate the permissions string (rwd). - // still want the end user API to have bool flags. - permissions = "" - - if capd.CanRead { - permissions += "r" - } - - if capd.CanWrite { - permissions += "w" - } - - if capd.CanDelete { - permissions += "d" - } - - return permissions -} diff --git a/storage/blob_test.go b/storage/blob_test.go deleted file mode 100644 index 1e84942d9cae..000000000000 --- a/storage/blob_test.go +++ /dev/null @@ -1,1599 +0,0 @@ -package storage - -import ( - "bytes" - "crypto/rand" - "encoding/base64" - "encoding/xml" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "sort" - "sync" - "testing" - "time" - - chk "gopkg.in/check.v1" -) - -type StorageBlobSuite struct{} - -var _ = chk.Suite(&StorageBlobSuite{}) - -const testContainerPrefix = "zzzztest-" - -func getBlobClient(c *chk.C) BlobStorageClient { - return getBasicClient(c).GetBlobService() -} - -func (s *StorageBlobSuite) Test_pathForContainer(c *chk.C) { - c.Assert(pathForContainer("foo"), chk.Equals, "/foo") -} - -func (s *StorageBlobSuite) Test_pathForBlob(c *chk.C) { - c.Assert(pathForBlob("foo", "blob"), chk.Equals, "/foo/blob") -} - -func (s *StorageBlobSuite) Test_blobSASStringToSign(c *chk.C) { - _, err := blobSASStringToSign("2012-02-12", "CS", "SE", "SP", "", "") - c.Assert(err, chk.NotNil) // not implemented SAS for versions earlier than 2013-08-15 - - out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP", "", "") - c.Assert(err, chk.IsNil) - c.Assert(out, chk.Equals, "SP\n\nSE\nCS\n\n2013-08-15\n\n\n\n\n") - - // check format for 2015-04-05 version - out, err = blobSASStringToSign("2015-04-05", "CS", "SE", "SP", "127.0.0.1", "https,http") - c.Assert(err, chk.IsNil) - c.Assert(out, chk.Equals, "SP\n\nSE\n/blobCS\n\n127.0.0.1\nhttps,http\n2015-04-05\n\n\n\n\n") -} - -func (s *StorageBlobSuite) TestGetBlobSASURI(c *chk.C) { - api, err := NewClient("foo", "YmFy", DefaultBaseURL, "2013-08-15", true) - c.Assert(err, chk.IsNil) - cli := api.GetBlobService() - expiry := time.Time{} - - expectedParts := url.URL{ - Scheme: "https", - Host: "foo.blob.core.windows.net", - Path: "container/name", - RawQuery: url.Values{ - "sv": {"2013-08-15"}, - "sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="}, - "sr": {"b"}, - "sp": {"r"}, - "se": {"0001-01-01T00:00:00Z"}, - }.Encode()} - - u, err := cli.GetBlobSASURI("container", "name", expiry, "r") - c.Assert(err, chk.IsNil) - sasParts, err := url.Parse(u) - c.Assert(err, chk.IsNil) - c.Assert(expectedParts.String(), chk.Equals, sasParts.String()) - c.Assert(expectedParts.Query(), chk.DeepEquals, sasParts.Query()) -} - -func (s *StorageBlobSuite) TestGetBlobSASURIWithSignedIPAndProtocolValidAPIVersionPassed(c *chk.C) { - api, err := NewClient("foo", "YmFy", DefaultBaseURL, "2015-04-05", true) - c.Assert(err, chk.IsNil) - cli := api.GetBlobService() - expiry := time.Time{} - - expectedParts := url.URL{ - Scheme: "https", - Host: "foo.blob.core.windows.net", - Path: "/container/name", - RawQuery: url.Values{ - "sv": {"2015-04-05"}, - "sig": {"VBOYJmt89UuBRXrxNzmsCMoC+8PXX2yklV71QcL1BfM="}, - "sr": {"b"}, - "sip": {"127.0.0.1"}, - "sp": {"r"}, - "se": {"0001-01-01T00:00:00Z"}, - "spr": {"https"}, - }.Encode()} - - u, err := cli.GetBlobSASURIWithSignedIPAndProtocol("container", "name", expiry, "r", "127.0.0.1", true) - c.Assert(err, chk.IsNil) - sasParts, err := url.Parse(u) - c.Assert(err, chk.IsNil) - c.Assert(sasParts.Query(), chk.DeepEquals, expectedParts.Query()) -} - -// Trying to use SignedIP and Protocol but using an older version of the API. -// Should ignore the signedIP/protocol and just use what the older version requires. -func (s *StorageBlobSuite) TestGetBlobSASURIWithSignedIPAndProtocolUsingOldAPIVersion(c *chk.C) { - api, err := NewClient("foo", "YmFy", DefaultBaseURL, "2013-08-15", true) - c.Assert(err, chk.IsNil) - cli := api.GetBlobService() - expiry := time.Time{} - - expectedParts := url.URL{ - Scheme: "https", - Host: "foo.blob.core.windows.net", - Path: "/container/name", - RawQuery: url.Values{ - "sv": {"2013-08-15"}, - "sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="}, - "sr": {"b"}, - "sp": {"r"}, - "se": {"0001-01-01T00:00:00Z"}, - }.Encode()} - - u, err := cli.GetBlobSASURIWithSignedIPAndProtocol("container", "name", expiry, "r", "", true) - c.Assert(err, chk.IsNil) - sasParts, err := url.Parse(u) - c.Assert(err, chk.IsNil) - c.Assert(expectedParts.String(), chk.Equals, sasParts.String()) - c.Assert(expectedParts.Query(), chk.DeepEquals, sasParts.Query()) -} - -func (s *StorageBlobSuite) TestBlobSASURICorrectness(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - blob := randNameWithSpecialChars(5) - body := []byte(randString(100)) - expiry := now.UTC().Add(time.Hour) - permissions := "r" - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.DeleteContainer(cnt) - - c.Assert(cli.putSingleBlockBlob(cnt, blob, body), chk.IsNil) - - sasURI, err := cli.GetBlobSASURI(cnt, blob, expiry, permissions) - c.Assert(err, chk.IsNil) - - resp, err := http.Get(sasURI) - c.Assert(err, chk.IsNil) - - blobResp, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - c.Assert(err, chk.IsNil) - - c.Assert(resp.StatusCode, chk.Equals, http.StatusOK) - c.Assert(len(blobResp), chk.Equals, len(body)) -} - -func (s *StorageBlobSuite) TestListContainersPagination(c *chk.C) { - cli := getBlobClient(c) - c.Assert(deleteTestContainers(cli), chk.IsNil) - - const n = 5 - const pageSize = 2 - - // Create test containers - created := []string{} - for i := 0; i < n; i++ { - name := randContainer() - c.Assert(cli.CreateContainer(name, ContainerAccessTypePrivate), chk.IsNil) - created = append(created, name) - } - sort.Strings(created) - - // Defer test container deletions - defer func() { - var wg sync.WaitGroup - for _, cnt := range created { - wg.Add(1) - go func(name string) { - c.Assert(cli.DeleteContainer(name), chk.IsNil) - wg.Done() - }(cnt) - } - wg.Wait() - }() - - // Paginate results - seen := []string{} - marker := "" - for { - resp, err := cli.ListContainers(ListContainersParameters{ - Prefix: testContainerPrefix, - MaxResults: pageSize, - Marker: marker}) - c.Assert(err, chk.IsNil) - - containers := resp.Containers - if len(containers) > pageSize { - c.Fatalf("Got a bigger page. Expected: %d, got: %d", pageSize, len(containers)) - } - - for _, c := range containers { - seen = append(seen, c.Name) - } - - marker = resp.NextMarker - if marker == "" || len(containers) == 0 { - break - } - } - - c.Assert(seen, chk.DeepEquals, created) -} - -func (s *StorageBlobSuite) TestContainerExists(c *chk.C) { - cnt := randContainer() - cli := getBlobClient(c) - ok, err := cli.ContainerExists(cnt) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil) - defer cli.DeleteContainer(cnt) - - ok, err = cli.ContainerExists(cnt) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) -} - -func (s *StorageBlobSuite) TestCreateContainerDeleteContainer(c *chk.C) { - cnt := randContainer() - cli := getBlobClient(c) - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - c.Assert(cli.DeleteContainer(cnt), chk.IsNil) -} - -func (s *StorageBlobSuite) TestCreateContainerIfNotExists(c *chk.C) { - cnt := randContainer() - cli := getBlobClient(c) - defer cli.DeleteContainer(cnt) - - // First create - ok, err := cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) - - // Second create, should not give errors - ok, err = cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) -} - -func (s *StorageBlobSuite) TestDeleteContainerIfExists(c *chk.C) { - cnt := randContainer() - cli := getBlobClient(c) - - // Nonexisting container - c.Assert(cli.DeleteContainer(cnt), chk.NotNil) - - ok, err := cli.DeleteContainerIfExists(cnt) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - // Existing container - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - ok, err = cli.DeleteContainerIfExists(cnt) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) -} - -func (s *StorageBlobSuite) TestBlobExists(c *chk.C) { - cnt := randContainer() - blob := randName(5) - cli := getBlobClient(c) - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil) - defer cli.DeleteContainer(cnt) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte("Hello!")), chk.IsNil) - defer cli.DeleteBlob(cnt, blob, nil) - - ok, err := cli.BlobExists(cnt, blob+".foo") - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - ok, err = cli.BlobExists(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) -} - -func (s *StorageBlobSuite) TestGetBlobURL(c *chk.C) { - api, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - cli := api.GetBlobService() - - c.Assert(cli.GetBlobURL("c", "nested/blob"), chk.Equals, "https://foo.blob.core.windows.net/c/nested/blob") - c.Assert(cli.GetBlobURL("", "blob"), chk.Equals, "https://foo.blob.core.windows.net/$root/blob") - c.Assert(cli.GetBlobURL("", "nested/blob"), chk.Equals, "https://foo.blob.core.windows.net/$root/nested/blob") -} - -func (s *StorageBlobSuite) TestBlobCopy(c *chk.C) { - if testing.Short() { - c.Skip("skipping blob copy in short mode, no SLA on async operation") - } - - cli := getBlobClient(c) - cnt := randContainer() - src := randName(5) - dst := randName(5) - body := []byte(randString(1024)) - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - c.Assert(cli.putSingleBlockBlob(cnt, src, body), chk.IsNil) - defer cli.DeleteBlob(cnt, src, nil) - - c.Assert(cli.CopyBlob(cnt, dst, cli.GetBlobURL(cnt, src)), chk.IsNil) - defer cli.DeleteBlob(cnt, dst, nil) - - blobBody, err := cli.GetBlob(cnt, dst) - c.Assert(err, chk.IsNil) - - b, err := ioutil.ReadAll(blobBody) - defer blobBody.Close() - c.Assert(err, chk.IsNil) - c.Assert(b, chk.DeepEquals, body) -} - -func (s *StorageBlobSuite) TestStartBlobCopy(c *chk.C) { - if testing.Short() { - c.Skip("skipping blob copy in short mode, no SLA on async operation") - } - - cli := getBlobClient(c) - cnt := randContainer() - src := randName(5) - dst := randName(5) - body := []byte(randString(1024)) - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - c.Assert(cli.putSingleBlockBlob(cnt, src, body), chk.IsNil) - defer cli.DeleteBlob(cnt, src, nil) - - // given we dont know when it will start, can we even test destination creation? - // will just test that an error wasn't thrown for now. - copyID, err := cli.StartBlobCopy(cnt, dst, cli.GetBlobURL(cnt, src)) - c.Assert(copyID, chk.NotNil) - c.Assert(err, chk.IsNil) -} - -// Tests abort of blobcopy. Given the blobcopy is usually over before we can actually trigger an abort -// it is agreed that we perform a copy then try and perform an abort. It should result in a HTTP status of 409. -// So basically we're testing negative scenario (as good as we can do for now) -func (s *StorageBlobSuite) TestAbortBlobCopy(c *chk.C) { - if testing.Short() { - c.Skip("skipping blob copy in short mode, no SLA on async operation") - } - - cli := getBlobClient(c) - cnt := randContainer() - src := randName(5) - dst := randName(5) - body := []byte(randString(1024)) - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - c.Assert(cli.putSingleBlockBlob(cnt, src, body), chk.IsNil) - defer cli.DeleteBlob(cnt, src, nil) - - // given we dont know when it will start, can we even test destination creation? - // will just test that an error wasn't thrown for now. - copyID, err := cli.StartBlobCopy(cnt, dst, cli.GetBlobURL(cnt, src)) - c.Assert(copyID, chk.NotNil) - c.Assert(err, chk.IsNil) - - err = cli.WaitForBlobCopy(cnt, dst, copyID) - c.Assert(err, chk.IsNil) - - // abort abort abort, but we *know* its already completed. - err = cli.AbortBlobCopy(cnt, dst, copyID, "", 0) - - // abort should fail (over already) - c.Assert(err.(AzureStorageServiceError).StatusCode, chk.Equals, http.StatusConflict) -} - -func (s *StorageBlobSuite) TestDeleteBlobIfExists(c *chk.C) { - cnt := randContainer() - blob := randName(5) - - cli := getBlobClient(c) - c.Assert(cli.DeleteBlob(cnt, blob, nil), chk.NotNil) - - ok, err := cli.DeleteBlobIfExists(cnt, blob, nil) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) -} - -func (s *StorageBlobSuite) TestDeleteBlobWithConditions(c *chk.C) { - cnt := randContainer() - blob := randName(5) - - cli := getBlobClient(c) - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - c.Assert(cli.CreateBlockBlob(cnt, blob), chk.IsNil) - oldProps, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - - // Update metadata, so Etag changes - c.Assert(cli.SetBlobMetadata(cnt, blob, map[string]string{}, nil), chk.IsNil) - newProps, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - - // "Delete if matches old Etag" should fail without deleting. - err = cli.DeleteBlob(cnt, blob, map[string]string{ - "If-Match": oldProps.Etag, - }) - c.Assert(err, chk.FitsTypeOf, AzureStorageServiceError{}) - c.Assert(err.(AzureStorageServiceError).StatusCode, chk.Equals, http.StatusPreconditionFailed) - _, err = cli.GetBlob(cnt, blob) - c.Assert(err, chk.IsNil) - - // "Delete if matches new Etag" should succeed. - err = cli.DeleteBlob(cnt, blob, map[string]string{ - "If-Match": newProps.Etag, - }) - c.Assert(err, chk.IsNil) - _, err = cli.GetBlob(cnt, blob) - c.Assert(err, chk.Not(chk.IsNil)) -} - -func (s *StorageBlobSuite) TestGetBlobProperties(c *chk.C) { - cnt := randContainer() - blob := randName(5) - contents := randString(64) - - cli := getBlobClient(c) - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.DeleteContainer(cnt) - - // Nonexisting blob - _, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.NotNil) - - // Put the blob - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte(contents)), chk.IsNil) - - // Get blob properties - props, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - - c.Assert(props.ContentLength, chk.Equals, int64(len(contents))) - c.Assert(props.ContentType, chk.Equals, "application/octet-stream") - c.Assert(props.BlobType, chk.Equals, BlobTypeBlock) -} - -func (s *StorageBlobSuite) TestListBlobsPagination(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.DeleteContainer(cnt) - - blobs := []string{} - const n = 5 - const pageSize = 2 - for i := 0; i < n; i++ { - name := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, name, []byte("Hello, world!")), chk.IsNil) - blobs = append(blobs, name) - } - sort.Strings(blobs) - - // Paginate - seen := []string{} - marker := "" - for { - resp, err := cli.ListBlobs(cnt, ListBlobsParameters{ - MaxResults: pageSize, - Marker: marker}) - c.Assert(err, chk.IsNil) - - for _, v := range resp.Blobs { - seen = append(seen, v.Name) - } - - marker = resp.NextMarker - if marker == "" || len(resp.Blobs) == 0 { - break - } - } - - // Compare - c.Assert(seen, chk.DeepEquals, blobs) -} - -// listBlobsAsFiles is a helper function to list blobs as "folders" and "files". -func listBlobsAsFiles(cli BlobStorageClient, cnt string, parentDir string) (folders []string, files []string, err error) { - var blobParams ListBlobsParameters - var blobListResponse BlobListResponse - - // Top level "folders" - blobParams = ListBlobsParameters{ - Delimiter: "/", - Prefix: parentDir, - } - - blobListResponse, err = cli.ListBlobs(cnt, blobParams) - if err != nil { - return nil, nil, err - } - - // These are treated as "folders" under the parentDir. - folders = blobListResponse.BlobPrefixes - - // "Files"" are blobs which are under the parentDir. - files = make([]string, len(blobListResponse.Blobs)) - for i := range blobListResponse.Blobs { - files[i] = blobListResponse.Blobs[i].Name - } - - return folders, files, nil -} - -// TestListBlobsTraversal tests that we can correctly traverse -// blobs in blob storage as if it were a file system by using -// a combination of Prefix, Delimiter, and BlobPrefixes. -// -// Blob storage is flat, but we can *simulate* the file -// system with folders and files using conventions in naming. -// With the blob namedd "/usr/bin/ls", when we use delimiter '/', -// the "ls" would be a "file"; with "/", /usr" and "/usr/bin" being -// the "folders" -// -// NOTE: The use of delimiter (eg forward slash) is extremely fiddly -// and difficult to get right so some discipline in naming and rules -// when using the API is required to get everything to work as expected. -// -// Assuming our delimiter is a forward slash, the rules are: -// -// - Do use a leading forward slash in blob names to make things -// consistent and simpler (see further). -// Note that doing so will show "" as the only top-level -// folder in the container in Azure portal, which may look strange. -// -// - The "folder names" are returned *with trailing forward slash* as per MSDN. -// -// - The "folder names" will be "absolue paths", e.g. listing things under "/usr/" -// will return folder names "/usr/bin/". -// -// - The "file names" are returned as full blob names, e.g. when listing -// things under "/usr/bin/", the file names will be "/usr/bin/ls" and -// "/usr/bin/cat". -// -// - Everything is returned with case-sensitive order as expected in real file system -// as per MSDN. -// -// - To list things under a "folder" always use trailing forward slash. -// -// Example: to list top level folders we use root folder named "" with -// trailing forward slash, so we use "/". -// -// Example: to list folders under "/usr", we again append forward slash and -// so we use "/usr/". -// -// Because we use leading forward slash we don't need to have different -// treatment of "get top-level folders" and "get non-top-level folders" -// scenarios. -func (s *StorageBlobSuite) TestListBlobsTraversal(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.DeleteContainer(cnt) - - // Note use of leading forward slash as per naming rules. - blobsToCreate := []string{ - "/usr/bin/ls", - "/usr/bin/cat", - "/usr/lib64/libc.so", - "/etc/hosts", - "/etc/init.d/iptables", - } - - // Create the above blobs - for _, blobName := range blobsToCreate { - err := cli.CreateBlockBlob(cnt, blobName) - c.Assert(err, chk.IsNil) - } - - var folders []string - var files []string - var err error - - // Top level folders and files. - folders, files, err = listBlobsAsFiles(cli, cnt, "/") - c.Assert(err, chk.IsNil) - c.Assert(folders, chk.DeepEquals, []string{"/etc/", "/usr/"}) - c.Assert(files, chk.DeepEquals, []string{}) - - // Things under /etc/. Note use of trailing forward slash here as per rules. - folders, files, err = listBlobsAsFiles(cli, cnt, "/etc/") - c.Assert(err, chk.IsNil) - c.Assert(folders, chk.DeepEquals, []string{"/etc/init.d/"}) - c.Assert(files, chk.DeepEquals, []string{"/etc/hosts"}) - - // Things under /etc/init.d/ - folders, files, err = listBlobsAsFiles(cli, cnt, "/etc/init.d/") - c.Assert(err, chk.IsNil) - c.Assert(folders, chk.DeepEquals, []string(nil)) - c.Assert(files, chk.DeepEquals, []string{"/etc/init.d/iptables"}) - - // Things under /usr/ - folders, files, err = listBlobsAsFiles(cli, cnt, "/usr/") - c.Assert(err, chk.IsNil) - c.Assert(folders, chk.DeepEquals, []string{"/usr/bin/", "/usr/lib64/"}) - c.Assert(files, chk.DeepEquals, []string{}) - - // Things under /usr/bin/ - folders, files, err = listBlobsAsFiles(cli, cnt, "/usr/bin/") - c.Assert(err, chk.IsNil) - c.Assert(folders, chk.DeepEquals, []string(nil)) - c.Assert(files, chk.DeepEquals, []string{"/usr/bin/cat", "/usr/bin/ls"}) -} - -func (s *StorageBlobSuite) TestListBlobsWithMetadata(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - expectMeta := make(map[string]BlobMetadata) - - // Put 4 blobs with metadata - for i := 0; i < 4; i++ { - name := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, name, []byte("Hello, world!")), chk.IsNil) - c.Assert(cli.SetBlobMetadata(cnt, name, map[string]string{ - "Foo": name, - "Bar_BAZ": "Waz Qux", - }, nil), chk.IsNil) - expectMeta[name] = BlobMetadata{ - "foo": name, - "bar_baz": "Waz Qux", - } - } - - // Put one more blob with no metadata - blobWithoutMetadata := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blobWithoutMetadata, []byte("Hello, world!")), chk.IsNil) - expectMeta[blobWithoutMetadata] = nil - - // Get ListBlobs with include:"metadata" - resp, err := cli.ListBlobs(cnt, ListBlobsParameters{ - MaxResults: 5, - Include: "metadata"}) - c.Assert(err, chk.IsNil) - - respBlobs := make(map[string]Blob) - for _, v := range resp.Blobs { - respBlobs[v.Name] = v - } - - // Verify the metadata is as expected - for name := range expectMeta { - c.Check(respBlobs[name].Metadata, chk.DeepEquals, expectMeta[name]) - } -} - -// Ensure it's possible to generate a ListBlobs response with -// metadata, e.g., for a stub server. -func (s *StorageBlobSuite) TestMarshalBlobMetadata(c *chk.C) { - buf, err := xml.Marshal(Blob{ - Name: randName(5), - Properties: BlobProperties{}, - Metadata: BlobMetadata{"foo": "baz < waz"}, - }) - c.Assert(err, chk.IsNil) - c.Assert(string(buf), chk.Matches, `.*baz < waz.*`) -} - -func (s *StorageBlobSuite) TestGetAndSetMetadata(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - m, err := cli.GetBlobMetadata(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(m, chk.Not(chk.Equals), nil) - c.Assert(len(m), chk.Equals, 0) - - mPut := map[string]string{ - "foo": "bar", - "bar_baz": "waz qux", - } - - err = cli.SetBlobMetadata(cnt, blob, mPut, nil) - c.Assert(err, chk.IsNil) - - m, err = cli.GetBlobMetadata(cnt, blob) - c.Assert(err, chk.IsNil) - c.Check(m, chk.DeepEquals, mPut) - - // Case munging - - mPutUpper := map[string]string{ - "Foo": "different bar", - "bar_BAZ": "different waz qux", - } - mExpectLower := map[string]string{ - "foo": "different bar", - "bar_baz": "different waz qux", - } - - err = cli.SetBlobMetadata(cnt, blob, mPutUpper, nil) - c.Assert(err, chk.IsNil) - - m, err = cli.GetBlobMetadata(cnt, blob) - c.Assert(err, chk.IsNil) - c.Check(m, chk.DeepEquals, mExpectLower) -} - -func (s *StorageBlobSuite) TestSetMetadataWithExtraHeaders(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - mPut := map[string]string{ - "foo": "bar", - "bar_baz": "waz qux", - } - - extraHeaders := map[string]string{ - "If-Match": "incorrect-etag", - } - - // Set with incorrect If-Match in extra headers should result in error - err := cli.SetBlobMetadata(cnt, blob, mPut, extraHeaders) - c.Assert(err, chk.NotNil) - - props, err := cli.GetBlobProperties(cnt, blob) - extraHeaders = map[string]string{ - "If-Match": props.Etag, - } - - // Set with matching If-Match in extra headers should succeed - err = cli.SetBlobMetadata(cnt, blob, mPut, extraHeaders) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestSetBlobProperties(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - mPut := BlobHeaders{ - CacheControl: "private, max-age=0, no-cache", - ContentMD5: "oBATU+oaDduHWbVZLuzIJw==", - ContentType: "application/json", - ContentEncoding: "gzip", - ContentLanguage: "de-DE", - } - - err := cli.SetBlobProperties(cnt, blob, mPut) - c.Assert(err, chk.IsNil) - - props, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - - c.Check(mPut.CacheControl, chk.Equals, props.CacheControl) - c.Check(mPut.ContentType, chk.Equals, props.ContentType) - c.Check(mPut.ContentMD5, chk.Equals, props.ContentMD5) - c.Check(mPut.ContentEncoding, chk.Equals, props.ContentEncoding) - c.Check(mPut.ContentLanguage, chk.Equals, props.ContentLanguage) -} - -func appendContainerPermission(perms ContainerPermissions, accessType ContainerAccessType, - ID string, start time.Time, expiry time.Time, - canRead bool, canWrite bool, canDelete bool) ContainerPermissions { - - perms.AccessType = accessType - - if ID != "" { - capd := ContainerAccessPolicyDetails{ - ID: ID, - StartTime: start, - ExpiryTime: expiry, - CanRead: canRead, - CanWrite: canWrite, - CanDelete: canDelete, - } - perms.AccessPolicies = append(perms.AccessPolicies, capd) - } - return perms -} - -func (s *StorageBlobSuite) TestSetContainerPermissionsWithTimeoutSuccessfully(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - perms := ContainerPermissions{} - perms = appendContainerPermission(perms, ContainerAccessTypeBlob, "GolangRocksOnAzure", now, now.Add(10*time.Hour), true, true, true) - - err := cli.SetContainerPermissions(cnt, perms, 30, "") - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestSetContainerPermissionsSuccessfully(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - perms := ContainerPermissions{} - perms = appendContainerPermission(perms, ContainerAccessTypeBlob, "GolangRocksOnAzure", now, now.Add(10*time.Hour), true, true, true) - - err := cli.SetContainerPermissions(cnt, perms, 0, "") - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsSuccessfully(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - perms := ContainerPermissions{} - perms = appendContainerPermission(perms, ContainerAccessTypeBlob, "AutoRestIsSuperCool", now, now.Add(10*time.Hour), true, true, true) - perms = appendContainerPermission(perms, ContainerAccessTypeBlob, "GolangRocksOnAzure", now.Add(20*time.Hour), now.Add(30*time.Hour), true, false, false) - - err := cli.SetContainerPermissions(cnt, perms, 0, "") - c.Assert(err, chk.IsNil) - - returnedPerms, err := cli.GetContainerPermissions(cnt, 0, "") - c.Assert(err, chk.IsNil) - - // check container permissions itself. - c.Assert(returnedPerms.AccessType, chk.Equals, perms.AccessType) - - // now check policy set. - c.Assert(returnedPerms.AccessPolicies, chk.HasLen, 2) - - for i := range perms.AccessPolicies { - c.Assert(returnedPerms.AccessPolicies[i].ID, chk.Equals, perms.AccessPolicies[i].ID) - - // test timestamps down the second - // rounding start/expiry time original perms since the returned perms would have been rounded. - // so need rounded vs rounded. - c.Assert(returnedPerms.AccessPolicies[i].StartTime.UTC().Round(time.Second).Format(time.RFC1123), - chk.Equals, perms.AccessPolicies[i].StartTime.UTC().Round(time.Second).Format(time.RFC1123)) - - c.Assert(returnedPerms.AccessPolicies[i].ExpiryTime.UTC().Round(time.Second).Format(time.RFC1123), - chk.Equals, perms.AccessPolicies[i].ExpiryTime.UTC().Round(time.Second).Format(time.RFC1123)) - - c.Assert(returnedPerms.AccessPolicies[i].CanRead, chk.Equals, perms.AccessPolicies[i].CanRead) - c.Assert(returnedPerms.AccessPolicies[i].CanWrite, chk.Equals, perms.AccessPolicies[i].CanWrite) - c.Assert(returnedPerms.AccessPolicies[i].CanDelete, chk.Equals, perms.AccessPolicies[i].CanDelete) - } -} - -func (s *StorageBlobSuite) TestSetContainerPermissionsOnlySuccessfully(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - perms := ContainerPermissions{} - perms = appendContainerPermission(perms, ContainerAccessTypeBlob, "GolangRocksOnAzure", now, now.Add(10*time.Hour), true, true, true) - - err := cli.SetContainerPermissions(cnt, perms, 0, "") - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestSetThenGetContainerPermissionsOnlySuccessfully(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - perms := ContainerPermissions{} - perms = appendContainerPermission(perms, ContainerAccessTypeBlob, "", now, now.Add(10*time.Hour), true, true, true) - - err := cli.SetContainerPermissions(cnt, perms, 0, "") - c.Assert(err, chk.IsNil) - - returnedPerms, err := cli.GetContainerPermissions(cnt, 0, "") - c.Assert(err, chk.IsNil) - - // check container permissions itself. - c.Assert(returnedPerms.AccessType, chk.Equals, perms.AccessType) - - // now check there are NO policies set - c.Assert(returnedPerms.AccessPolicies, chk.HasLen, 0) -} - -func (s *StorageBlobSuite) TestSnapshotBlob(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - snapshotTime, err := cli.SnapshotBlob(cnt, blob, 0, nil) - c.Assert(err, chk.IsNil) - c.Assert(snapshotTime, chk.NotNil) -} - -func (s *StorageBlobSuite) TestSnapshotBlobWithTimeout(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - snapshotTime, err := cli.SnapshotBlob(cnt, blob, 30, nil) - c.Assert(err, chk.IsNil) - c.Assert(snapshotTime, chk.NotNil) -} - -func (s *StorageBlobSuite) TestSnapshotBlobWithValidLease(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - // generate lease. - currentLeaseID, err := cli.AcquireLease(cnt, blob, 30, "") - c.Assert(err, chk.IsNil) - - extraHeaders := map[string]string{ - headerLeaseID: currentLeaseID, - } - - snapshotTime, err := cli.SnapshotBlob(cnt, blob, 0, extraHeaders) - c.Assert(err, chk.IsNil) - c.Assert(snapshotTime, chk.NotNil) -} - -func (s *StorageBlobSuite) TestSnapshotBlobWithInvalidLease(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - // generate lease. - leaseID, err := cli.AcquireLease(cnt, blob, 30, "") - c.Assert(err, chk.IsNil) - c.Assert(leaseID, chk.Not(chk.Equals), "") - - extraHeaders := map[string]string{ - headerLeaseID: "GolangRocksOnAzure", - } - - snapshotTime, err := cli.SnapshotBlob(cnt, blob, 0, extraHeaders) - c.Assert(err, chk.NotNil) - c.Assert(snapshotTime, chk.IsNil) -} - -func (s *StorageBlobSuite) TestAcquireLeaseWithNoProposedLeaseID(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - _, err := cli.AcquireLease(cnt, blob, 30, "") - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestAcquireLeaseWithProposedLeaseID(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - c.Assert(leaseID, chk.Equals, proposedLeaseID) -} - -func (s *StorageBlobSuite) TestAcquireLeaseWithBadProposedLeaseID(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - proposedLeaseID := "badbadbad" - _, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.NotNil) -} - -func (s *StorageBlobSuite) TestRenewLeaseSuccessful(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - - err = cli.RenewLease(cnt, blob, leaseID) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestRenewLeaseAgainstNoCurrentLease(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - badLeaseID := "1f812371-a41d-49e6-b123-f4b542e85144" - err := cli.RenewLease(cnt, blob, badLeaseID) - c.Assert(err, chk.NotNil) -} - -func (s *StorageBlobSuite) TestChangeLeaseSuccessful(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - - newProposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fbb" - newLeaseID, err := cli.ChangeLease(cnt, blob, leaseID, newProposedLeaseID) - c.Assert(err, chk.IsNil) - c.Assert(newLeaseID, chk.Equals, newProposedLeaseID) -} - -func (s *StorageBlobSuite) TestChangeLeaseNotSuccessfulbadProposedLeaseID(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - - newProposedLeaseID := "1f812371-a41d-49e6-b123-f4b542e" - _, err = cli.ChangeLease(cnt, blob, leaseID, newProposedLeaseID) - c.Assert(err, chk.NotNil) -} - -func (s *StorageBlobSuite) TestReleaseLeaseSuccessful(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - leaseID, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - - err = cli.ReleaseLease(cnt, blob, leaseID) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestReleaseLeaseNotSuccessfulBadLeaseID(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - _, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - - err = cli.ReleaseLease(cnt, blob, "badleaseid") - c.Assert(err, chk.NotNil) -} - -func (s *StorageBlobSuite) TestBreakLeaseSuccessful(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - proposedLeaseID := "dfe6dde8-68d5-4910-9248-c97c61768fea" - _, err := cli.AcquireLease(cnt, blob, 30, proposedLeaseID) - c.Assert(err, chk.IsNil) - - _, err = cli.BreakLease(cnt, blob) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestPutEmptyBlockBlob(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte{}), chk.IsNil) - - props, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(props.ContentLength, chk.Not(chk.Equals), 0) -} - -func (s *StorageBlobSuite) TestGetBlobRange(c *chk.C) { - cnt := randContainer() - blob := randName(5) - body := "0123456789" - - cli := getBlobClient(c) - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypeBlob), chk.IsNil) - defer cli.DeleteContainer(cnt) - - c.Assert(cli.putSingleBlockBlob(cnt, blob, []byte(body)), chk.IsNil) - defer cli.DeleteBlob(cnt, blob, nil) - - // Read 1-3 - for _, r := range []struct { - rangeStr string - expected string - }{ - {"0-", body}, - {"1-3", body[1 : 3+1]}, - {"3-", body[3:]}, - } { - resp, err := cli.GetBlobRange(cnt, blob, r.rangeStr, nil) - c.Assert(err, chk.IsNil) - blobBody, err := ioutil.ReadAll(resp) - c.Assert(err, chk.IsNil) - - str := string(blobBody) - c.Assert(str, chk.Equals, r.expected) - } -} - -func (s *StorageBlobSuite) TestCreateBlockBlobFromReader(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - name := randName(5) - data := randBytes(8888) - c.Assert(cli.CreateBlockBlobFromReader(cnt, name, uint64(len(data)), bytes.NewReader(data), nil), chk.IsNil) - - body, err := cli.GetBlob(cnt, name) - c.Assert(err, chk.IsNil) - gotData, err := ioutil.ReadAll(body) - body.Close() - - c.Assert(err, chk.IsNil) - c.Assert(gotData, chk.DeepEquals, data) -} - -func (s *StorageBlobSuite) TestCreateBlockBlobFromReaderWithShortData(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - name := randName(5) - data := randBytes(8888) - err := cli.CreateBlockBlobFromReader(cnt, name, 9999, bytes.NewReader(data), nil) - c.Assert(err, chk.Not(chk.IsNil)) - - _, err = cli.GetBlob(cnt, name) - // Upload was incomplete: blob should not have been created. - c.Assert(err, chk.Not(chk.IsNil)) -} - -func (s *StorageBlobSuite) TestPutBlock(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - chunk := []byte(randString(1024)) - blockID := base64.StdEncoding.EncodeToString([]byte("foo")) - c.Assert(cli.PutBlock(cnt, blob, blockID, chunk), chk.IsNil) -} - -func (s *StorageBlobSuite) TestGetBlockList_PutBlockList(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - chunk := []byte(randString(1024)) - blockID := base64.StdEncoding.EncodeToString([]byte("foo")) - - // Put one block - c.Assert(cli.PutBlock(cnt, blob, blockID, chunk), chk.IsNil) - defer cli.deleteBlob(cnt, blob, nil) - - // Get committed blocks - committed, err := cli.GetBlockList(cnt, blob, BlockListTypeCommitted) - c.Assert(err, chk.IsNil) - - if len(committed.CommittedBlocks) > 0 { - c.Fatal("There are committed blocks") - } - - // Get uncommitted blocks - uncommitted, err := cli.GetBlockList(cnt, blob, BlockListTypeUncommitted) - c.Assert(err, chk.IsNil) - - c.Assert(len(uncommitted.UncommittedBlocks), chk.Equals, 1) - // Commit block list - c.Assert(cli.PutBlockList(cnt, blob, []Block{{blockID, BlockStatusUncommitted}}), chk.IsNil) - - // Get all blocks - all, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) - c.Assert(err, chk.IsNil) - c.Assert(len(all.CommittedBlocks), chk.Equals, 1) - c.Assert(len(all.UncommittedBlocks), chk.Equals, 0) - - // Verify the block - thatBlock := all.CommittedBlocks[0] - c.Assert(thatBlock.Name, chk.Equals, blockID) - c.Assert(thatBlock.Size, chk.Equals, int64(len(chunk))) -} - -func (s *StorageBlobSuite) TestCreateBlockBlob(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.CreateBlockBlob(cnt, blob), chk.IsNil) - - // Verify - blocks, err := cli.GetBlockList(cnt, blob, BlockListTypeAll) - c.Assert(err, chk.IsNil) - c.Assert(len(blocks.CommittedBlocks), chk.Equals, 0) - c.Assert(len(blocks.UncommittedBlocks), chk.Equals, 0) -} - -func (s *StorageBlobSuite) TestPutPageBlob(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - size := int64(10 * 1024 * 1024) - c.Assert(cli.PutPageBlob(cnt, blob, size, nil), chk.IsNil) - - // Verify - props, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(props.ContentLength, chk.Equals, size) - c.Assert(props.BlobType, chk.Equals, BlobTypePage) -} - -func (s *StorageBlobSuite) TestPutPagesUpdate(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - size := int64(10 * 1024 * 1024) // larger than we'll use - c.Assert(cli.PutPageBlob(cnt, blob, size, nil), chk.IsNil) - - chunk1 := []byte(randString(1024)) - chunk2 := []byte(randString(512)) - - // Append chunks - c.Assert(cli.PutPage(cnt, blob, 0, int64(len(chunk1)-1), PageWriteTypeUpdate, chunk1, nil), chk.IsNil) - c.Assert(cli.PutPage(cnt, blob, int64(len(chunk1)), int64(len(chunk1)+len(chunk2)-1), PageWriteTypeUpdate, chunk2, nil), chk.IsNil) - - // Verify contents - out, err := cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1), nil) - c.Assert(err, chk.IsNil) - defer out.Close() - blobContents, err := ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - c.Assert(blobContents, chk.DeepEquals, append(chunk1, chunk2...)) - out.Close() - - // Overwrite first half of chunk1 - chunk0 := []byte(randString(512)) - c.Assert(cli.PutPage(cnt, blob, 0, int64(len(chunk0)-1), PageWriteTypeUpdate, chunk0, nil), chk.IsNil) - - // Verify contents - out, err = cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1), nil) - c.Assert(err, chk.IsNil) - defer out.Close() - blobContents, err = ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - c.Assert(blobContents, chk.DeepEquals, append(append(chunk0, chunk1[512:]...), chunk2...)) -} - -func (s *StorageBlobSuite) TestPutPagesClear(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - size := int64(10 * 1024 * 1024) // larger than we'll use - c.Assert(cli.PutPageBlob(cnt, blob, size, nil), chk.IsNil) - - // Put 0-2047 - chunk := []byte(randString(2048)) - c.Assert(cli.PutPage(cnt, blob, 0, 2047, PageWriteTypeUpdate, chunk, nil), chk.IsNil) - - // Clear 512-1023 - c.Assert(cli.PutPage(cnt, blob, 512, 1023, PageWriteTypeClear, nil, nil), chk.IsNil) - - // Verify contents - out, err := cli.GetBlobRange(cnt, blob, "0-2047", nil) - c.Assert(err, chk.IsNil) - contents, err := ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - defer out.Close() - c.Assert(contents, chk.DeepEquals, append(append(chunk[:512], make([]byte, 512)...), chunk[1024:]...)) -} - -func (s *StorageBlobSuite) TestGetPageRanges(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - size := int64(10 * 1024 * 1024) // larger than we'll use - c.Assert(cli.PutPageBlob(cnt, blob, size, nil), chk.IsNil) - - // Get page ranges on empty blob - out, err := cli.GetPageRanges(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(len(out.PageList), chk.Equals, 0) - - // Add 0-512 page - c.Assert(cli.PutPage(cnt, blob, 0, 511, PageWriteTypeUpdate, []byte(randString(512)), nil), chk.IsNil) - - out, err = cli.GetPageRanges(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(len(out.PageList), chk.Equals, 1) - - // Add 1024-2048 - c.Assert(cli.PutPage(cnt, blob, 1024, 2047, PageWriteTypeUpdate, []byte(randString(1024)), nil), chk.IsNil) - - out, err = cli.GetPageRanges(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(len(out.PageList), chk.Equals, 2) -} - -func (s *StorageBlobSuite) TestPutAppendBlob(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.PutAppendBlob(cnt, blob, nil), chk.IsNil) - - // Verify - props, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(props.ContentLength, chk.Equals, int64(0)) - c.Assert(props.BlobType, chk.Equals, BlobTypeAppend) -} - -func (s *StorageBlobSuite) TestPutAppendBlobAppendBlocks(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randName(5) - c.Assert(cli.PutAppendBlob(cnt, blob, nil), chk.IsNil) - - chunk1 := []byte(randString(1024)) - chunk2 := []byte(randString(512)) - - // Append first block - c.Assert(cli.AppendBlock(cnt, blob, chunk1, nil), chk.IsNil) - - // Verify contents - out, err := cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)-1), nil) - c.Assert(err, chk.IsNil) - defer out.Close() - blobContents, err := ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - c.Assert(blobContents, chk.DeepEquals, chunk1) - out.Close() - - // Append second block - c.Assert(cli.AppendBlock(cnt, blob, chunk2, nil), chk.IsNil) - - // Verify contents - out, err = cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1), nil) - c.Assert(err, chk.IsNil) - defer out.Close() - blobContents, err = ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - c.Assert(blobContents, chk.DeepEquals, append(chunk1, chunk2...)) - out.Close() -} - -func deleteTestContainers(cli BlobStorageClient) error { - for { - resp, err := cli.ListContainers(ListContainersParameters{Prefix: testContainerPrefix}) - if err != nil { - return err - } - if len(resp.Containers) == 0 { - break - } - for _, c := range resp.Containers { - err = cli.DeleteContainer(c.Name) - if err != nil { - return err - } - } - } - return nil -} - -func (b BlobStorageClient) putSingleBlockBlob(container, name string, chunk []byte) error { - if len(chunk) > MaxBlobBlockSize { - return fmt.Errorf("storage: provided chunk (%d bytes) cannot fit into single-block blob (max %d bytes)", len(chunk), MaxBlobBlockSize) - } - - uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) - headers := b.client.getStandardHeaders() - headers["x-ms-blob-type"] = string(BlobTypeBlock) - headers["Content-Length"] = fmt.Sprintf("%v", len(chunk)) - - resp, err := b.client.exec(http.MethodPut, uri, headers, bytes.NewReader(chunk), b.auth) - if err != nil { - return err - } - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -func (s *StorageBlobSuite) TestPutAppendBlobSpecialChars(c *chk.C) { - cli := getBlobClient(c) - cnt := randContainer() - c.Assert(cli.CreateContainer(cnt, ContainerAccessTypePrivate), chk.IsNil) - defer cli.deleteContainer(cnt) - - blob := randNameWithSpecialChars(5) - c.Assert(cli.PutAppendBlob(cnt, blob, nil), chk.IsNil) - - // Verify metadata - props, err := cli.GetBlobProperties(cnt, blob) - c.Assert(err, chk.IsNil) - c.Assert(props.ContentLength, chk.Equals, int64(0)) - c.Assert(props.BlobType, chk.Equals, BlobTypeAppend) - - chunk1 := []byte(randString(1024)) - chunk2 := []byte(randString(512)) - - // Append first block - c.Assert(cli.AppendBlock(cnt, blob, chunk1, nil), chk.IsNil) - - // Verify contents - out, err := cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)-1), nil) - c.Assert(err, chk.IsNil) - defer out.Close() - blobContents, err := ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - c.Assert(blobContents, chk.DeepEquals, chunk1) - out.Close() - - // Append second block - c.Assert(cli.AppendBlock(cnt, blob, chunk2, nil), chk.IsNil) - - // Verify contents - out, err = cli.GetBlobRange(cnt, blob, fmt.Sprintf("%v-%v", 0, len(chunk1)+len(chunk2)-1), nil) - c.Assert(err, chk.IsNil) - defer out.Close() - blobContents, err = ioutil.ReadAll(out) - c.Assert(err, chk.IsNil) - c.Assert(blobContents, chk.DeepEquals, append(chunk1, chunk2...)) - out.Close() -} - -func randContainer() string { - return testContainerPrefix + randString(32-len(testContainerPrefix)) -} - -func randString(n int) string { - if n <= 0 { - panic("negative number") - } - const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz" - var bytes = make([]byte, n) - rand.Read(bytes) - for i, b := range bytes { - bytes[i] = alphanum[b%byte(len(alphanum))] - } - return string(bytes) -} - -func randBytes(n int) []byte { - data := make([]byte, n) - if _, err := io.ReadFull(rand.Reader, data); err != nil { - panic(err) - } - return data -} - -func randName(n int) string { - name := randString(n) + "/" + randString(n) - return name -} - -func randNameWithSpecialChars(n int) string { - name := randString(n) + "/" + randString(n) + "-._~:?#[]@!$&'()*,;+= " + randString(n) - return name -} diff --git a/storage/client.go b/storage/client.go deleted file mode 100644 index 817f934c5cdd..000000000000 --- a/storage/client.go +++ /dev/null @@ -1,469 +0,0 @@ -// Package storage provides clients for Microsoft Azure Storage Services. -package storage - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "encoding/xml" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "runtime" - "strconv" - "strings" -) - -const ( - // DefaultBaseURL is the domain name used for storage requests when a - // default client is created. - DefaultBaseURL = "core.windows.net" - - // DefaultAPIVersion is the Azure Storage API version string used when a - // basic client is created. - DefaultAPIVersion = "2015-02-21" - - defaultUseHTTPS = true - - // StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator - StorageEmulatorAccountName = "devstoreaccount1" - - // StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator - StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - - blobServiceName = "blob" - tableServiceName = "table" - queueServiceName = "queue" - fileServiceName = "file" - - storageEmulatorBlob = "127.0.0.1:10000" - storageEmulatorTable = "127.0.0.1:10002" - storageEmulatorQueue = "127.0.0.1:10001" - - userAgentHeader = "User-Agent" -) - -// Client is the object that needs to be constructed to perform -// operations on the storage account. -type Client struct { - // HTTPClient is the http.Client used to initiate API - // requests. If it is nil, http.DefaultClient is used. - HTTPClient *http.Client - - accountName string - accountKey []byte - useHTTPS bool - UseSharedKeyLite bool - baseURL string - apiVersion string - userAgent string -} - -type storageResponse struct { - statusCode int - headers http.Header - body io.ReadCloser -} - -type odataResponse struct { - storageResponse - odata odataErrorMessage -} - -// AzureStorageServiceError contains fields of the error response from -// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx -// Some fields might be specific to certain calls. -type AzureStorageServiceError struct { - Code string `xml:"Code"` - Message string `xml:"Message"` - AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"` - QueryParameterName string `xml:"QueryParameterName"` - QueryParameterValue string `xml:"QueryParameterValue"` - Reason string `xml:"Reason"` - StatusCode int - RequestID string -} - -type odataErrorMessageMessage struct { - Lang string `json:"lang"` - Value string `json:"value"` -} - -type odataErrorMessageInternal struct { - Code string `json:"code"` - Message odataErrorMessageMessage `json:"message"` -} - -type odataErrorMessage struct { - Err odataErrorMessageInternal `json:"odata.error"` -} - -// UnexpectedStatusCodeError is returned when a storage service responds with neither an error -// nor with an HTTP status code indicating success. -type UnexpectedStatusCodeError struct { - allowed []int - got int -} - -func (e UnexpectedStatusCodeError) Error() string { - s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) } - - got := s(e.got) - expected := []string{} - for _, v := range e.allowed { - expected = append(expected, s(v)) - } - return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or ")) -} - -// Got is the actual status code returned by Azure. -func (e UnexpectedStatusCodeError) Got() int { - return e.got -} - -// NewBasicClient constructs a Client with given storage service name and -// key. -func NewBasicClient(accountName, accountKey string) (Client, error) { - if accountName == StorageEmulatorAccountName { - return NewEmulatorClient() - } - return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS) - -} - -//NewEmulatorClient contructs a Client intended to only work with Azure -//Storage Emulator -func NewEmulatorClient() (Client, error) { - return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false) -} - -// NewClient constructs a Client. This should be used if the caller wants -// to specify whether to use HTTPS, a specific REST API version or a custom -// storage endpoint than Azure Public Cloud. -func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) { - var c Client - if accountName == "" { - return c, fmt.Errorf("azure: account name required") - } else if accountKey == "" { - return c, fmt.Errorf("azure: account key required") - } else if blobServiceBaseURL == "" { - return c, fmt.Errorf("azure: base storage service url required") - } - - key, err := base64.StdEncoding.DecodeString(accountKey) - if err != nil { - return c, fmt.Errorf("azure: malformed storage account key: %v", err) - } - - c = Client{ - accountName: accountName, - accountKey: key, - useHTTPS: useHTTPS, - baseURL: blobServiceBaseURL, - apiVersion: apiVersion, - UseSharedKeyLite: false, - } - c.userAgent = c.getDefaultUserAgent() - return c, nil -} - -func (c Client) getDefaultUserAgent() string { - return fmt.Sprintf("Go/%s (%s-%s) Azure-SDK-For-Go/%s storage-dataplane/%s", - runtime.Version(), - runtime.GOARCH, - runtime.GOOS, - sdkVersion, - c.apiVersion, - ) -} - -// AddToUserAgent adds an extension to the current user agent -func (c *Client) AddToUserAgent(extension string) error { - if extension != "" { - c.userAgent = fmt.Sprintf("%s %s", c.userAgent, extension) - return nil - } - return fmt.Errorf("Extension was empty, User Agent stayed as %s", c.userAgent) -} - -// protectUserAgent is used in funcs that include extraheaders as a parameter. -// It prevents the User-Agent header to be overwritten, instead if it happens to -// be present, it gets added to the current User-Agent. Use it before getStandardHeaders -func (c *Client) protectUserAgent(extraheaders map[string]string) map[string]string { - if v, ok := extraheaders[userAgentHeader]; ok { - c.AddToUserAgent(v) - delete(extraheaders, userAgentHeader) - } - return extraheaders -} - -func (c Client) getBaseURL(service string) string { - scheme := "http" - if c.useHTTPS { - scheme = "https" - } - host := "" - if c.accountName == StorageEmulatorAccountName { - switch service { - case blobServiceName: - host = storageEmulatorBlob - case tableServiceName: - host = storageEmulatorTable - case queueServiceName: - host = storageEmulatorQueue - } - } else { - host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL) - } - - u := &url.URL{ - Scheme: scheme, - Host: host} - return u.String() -} - -func (c Client) getEndpoint(service, path string, params url.Values) string { - u, err := url.Parse(c.getBaseURL(service)) - if err != nil { - // really should not be happening - panic(err) - } - - // API doesn't accept path segments not starting with '/' - if !strings.HasPrefix(path, "/") { - path = fmt.Sprintf("/%v", path) - } - - if c.accountName == StorageEmulatorAccountName { - path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path) - } - - u.Path = path - u.RawQuery = params.Encode() - return u.String() -} - -// GetBlobService returns a BlobStorageClient which can operate on the blob -// service of the storage account. -func (c Client) GetBlobService() BlobStorageClient { - b := BlobStorageClient{ - client: c, - } - b.client.AddToUserAgent(blobServiceName) - b.auth = sharedKey - if c.UseSharedKeyLite { - b.auth = sharedKeyLite - } - return b -} - -// GetQueueService returns a QueueServiceClient which can operate on the queue -// service of the storage account. -func (c Client) GetQueueService() QueueServiceClient { - q := QueueServiceClient{ - client: c, - } - q.client.AddToUserAgent(queueServiceName) - q.auth = sharedKey - if c.UseSharedKeyLite { - q.auth = sharedKeyLite - } - return q -} - -// GetTableService returns a TableServiceClient which can operate on the table -// service of the storage account. -func (c Client) GetTableService() TableServiceClient { - t := TableServiceClient{ - client: c, - } - t.client.AddToUserAgent(tableServiceName) - t.auth = sharedKeyForTable - if c.UseSharedKeyLite { - t.auth = sharedKeyLiteForTable - } - return t -} - -// GetFileService returns a FileServiceClient which can operate on the file -// service of the storage account. -func (c Client) GetFileService() FileServiceClient { - f := FileServiceClient{ - client: c, - } - f.client.AddToUserAgent(fileServiceName) - f.auth = sharedKey - if c.UseSharedKeyLite { - f.auth = sharedKeyLite - } - return f -} - -func (c Client) getStandardHeaders() map[string]string { - return map[string]string{ - userAgentHeader: c.userAgent, - "x-ms-version": c.apiVersion, - "x-ms-date": currentTimeRfc1123Formatted(), - } -} - -func (c Client) exec(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*storageResponse, error) { - headers, err := c.addAuthorizationHeader(verb, url, headers, auth) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(verb, url, body) - if err != nil { - return nil, errors.New("azure/storage: error creating request: " + err.Error()) - } - - if clstr, ok := headers["Content-Length"]; ok { - // content length header is being signed, but completely ignored by golang. - // instead we have to use the ContentLength property on the request struct - // (see https://golang.org/src/net/http/request.go?s=18140:18370#L536 and - // https://golang.org/src/net/http/transfer.go?s=1739:2467#L49) - req.ContentLength, err = strconv.ParseInt(clstr, 10, 64) - if err != nil { - return nil, err - } - } - for k, v := range headers { - req.Header.Add(k, v) - } - - httpClient := c.HTTPClient - if httpClient == nil { - httpClient = http.DefaultClient - } - resp, err := httpClient.Do(req) - if err != nil { - return nil, err - } - - statusCode := resp.StatusCode - if statusCode >= 400 && statusCode <= 505 { - var respBody []byte - respBody, err = readResponseBody(resp) - if err != nil { - return nil, err - } - - requestID := resp.Header.Get("x-ms-request-id") - if len(respBody) == 0 { - // no error in response body, might happen in HEAD requests - err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID) - } else { - // response contains storage service error object, unmarshal - storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, requestID) - if err != nil { // error unmarshaling the error response - err = errIn - } - err = storageErr - } - return &storageResponse{ - statusCode: resp.StatusCode, - headers: resp.Header, - body: ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */ - }, err - } - - return &storageResponse{ - statusCode: resp.StatusCode, - headers: resp.Header, - body: resp.Body}, nil -} - -func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) { - headers, err := c.addAuthorizationHeader(verb, url, headers, auth) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(verb, url, body) - for k, v := range headers { - req.Header.Add(k, v) - } - - httpClient := c.HTTPClient - if httpClient == nil { - httpClient = http.DefaultClient - } - - resp, err := httpClient.Do(req) - if err != nil { - return nil, err - } - - respToRet := &odataResponse{} - respToRet.body = resp.Body - respToRet.statusCode = resp.StatusCode - respToRet.headers = resp.Header - - statusCode := resp.StatusCode - if statusCode >= 400 && statusCode <= 505 { - var respBody []byte - respBody, err = readResponseBody(resp) - if err != nil { - return nil, err - } - - if len(respBody) == 0 { - // no error in response body, might happen in HEAD requests - err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, resp.Header.Get("x-ms-request-id")) - return respToRet, err - } - // try unmarshal as odata.error json - err = json.Unmarshal(respBody, &respToRet.odata) - return respToRet, err - } - - return respToRet, nil -} - -func readResponseBody(resp *http.Response) ([]byte, error) { - defer resp.Body.Close() - out, err := ioutil.ReadAll(resp.Body) - if err == io.EOF { - err = nil - } - return out, err -} - -func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStorageServiceError, error) { - var storageErr AzureStorageServiceError - if err := xml.Unmarshal(body, &storageErr); err != nil { - return storageErr, err - } - storageErr.StatusCode = statusCode - storageErr.RequestID = requestID - return storageErr, nil -} - -func serviceErrFromStatusCode(code int, status string, requestID string) AzureStorageServiceError { - return AzureStorageServiceError{ - StatusCode: code, - Code: status, - RequestID: requestID, - Message: "no response body was available for error status code", - } -} - -func (e AzureStorageServiceError) Error() string { - return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s, QueryParameterName=%s, QueryParameterValue=%s", - e.StatusCode, e.Code, e.Message, e.RequestID, e.QueryParameterName, e.QueryParameterValue) -} - -// checkRespCode returns UnexpectedStatusError if the given response code is not -// one of the allowed status codes; otherwise nil. -func checkRespCode(respCode int, allowed []int) error { - for _, v := range allowed { - if respCode == v { - return nil - } - } - return UnexpectedStatusCodeError{allowed, respCode} -} diff --git a/storage/client_test.go b/storage/client_test.go deleted file mode 100644 index 04ea80bbb31b..000000000000 --- a/storage/client_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package storage - -import ( - "encoding/base64" - "net/http" - "net/url" - "os" - "testing" - "time" - - chk "gopkg.in/check.v1" -) - -// Hook up gocheck to testing -func Test(t *testing.T) { chk.TestingT(t) } - -type StorageClientSuite struct{} - -var _ = chk.Suite(&StorageClientSuite{}) - -var now = time.Now() - -// getBasicClient returns a test client from storage credentials in the env -func getBasicClient(c *chk.C) Client { - name := os.Getenv("ACCOUNT_NAME") - if name == "" { - c.Fatal("ACCOUNT_NAME not set, need an empty storage account to test") - } - key := os.Getenv("ACCOUNT_KEY") - if key == "" { - c.Fatal("ACCOUNT_KEY not set") - } - cli, err := NewBasicClient(name, key) - c.Assert(err, chk.IsNil) - return cli -} - -//getEmulatorClient returns a test client for Azure Storeage Emulator -func getEmulatorClient(c *chk.C) Client { - cli, err := NewBasicClient(StorageEmulatorAccountName, "") - c.Assert(err, chk.IsNil) - return cli -} - -func (s *StorageClientSuite) TestNewEmulatorClient(c *chk.C) { - cli, err := NewBasicClient(StorageEmulatorAccountName, "") - c.Assert(err, chk.IsNil) - c.Assert(cli.accountName, chk.Equals, StorageEmulatorAccountName) - expectedKey, err := base64.StdEncoding.DecodeString(StorageEmulatorAccountKey) - c.Assert(err, chk.IsNil) - c.Assert(cli.accountKey, chk.DeepEquals, expectedKey) -} - -func (s *StorageClientSuite) TestMalformedKeyError(c *chk.C) { - _, err := NewBasicClient("foo", "malformed") - c.Assert(err, chk.ErrorMatches, "azure: malformed storage account key: .*") -} - -func (s *StorageClientSuite) TestGetBaseURL_Basic_Https(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - c.Assert(cli.apiVersion, chk.Equals, DefaultAPIVersion) - c.Assert(err, chk.IsNil) - c.Assert(cli.getBaseURL("table"), chk.Equals, "https://foo.table.core.windows.net") -} - -func (s *StorageClientSuite) TestGetBaseURL_Custom_NoHttps(c *chk.C) { - apiVersion := "2015-01-01" // a non existing one - cli, err := NewClient("foo", "YmFy", "core.chinacloudapi.cn", apiVersion, false) - c.Assert(err, chk.IsNil) - c.Assert(cli.apiVersion, chk.Equals, apiVersion) - c.Assert(cli.getBaseURL("table"), chk.Equals, "http://foo.table.core.chinacloudapi.cn") -} - -func (s *StorageClientSuite) TestGetBaseURL_StorageEmulator(c *chk.C) { - cli, err := NewBasicClient(StorageEmulatorAccountName, StorageEmulatorAccountKey) - c.Assert(err, chk.IsNil) - - type test struct{ service, expected string } - tests := []test{ - {blobServiceName, "http://127.0.0.1:10000"}, - {tableServiceName, "http://127.0.0.1:10002"}, - {queueServiceName, "http://127.0.0.1:10001"}, - } - for _, i := range tests { - baseURL := cli.getBaseURL(i.service) - c.Assert(baseURL, chk.Equals, i.expected) - } -} - -func (s *StorageClientSuite) TestGetEndpoint_None(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - output := cli.getEndpoint(blobServiceName, "", url.Values{}) - c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/") -} - -func (s *StorageClientSuite) TestGetEndpoint_PathOnly(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - output := cli.getEndpoint(blobServiceName, "path", url.Values{}) - c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/path") -} - -func (s *StorageClientSuite) TestGetEndpoint_ParamsOnly(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - params := url.Values{} - params.Set("a", "b") - params.Set("c", "d") - output := cli.getEndpoint(blobServiceName, "", params) - c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/?a=b&c=d") -} - -func (s *StorageClientSuite) TestGetEndpoint_Mixed(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - params := url.Values{} - params.Set("a", "b") - params.Set("c", "d") - output := cli.getEndpoint(blobServiceName, "path", params) - c.Assert(output, chk.Equals, "https://foo.blob.core.windows.net/path?a=b&c=d") -} - -func (s *StorageClientSuite) TestGetEndpoint_StorageEmulator(c *chk.C) { - cli, err := NewBasicClient(StorageEmulatorAccountName, StorageEmulatorAccountKey) - c.Assert(err, chk.IsNil) - - type test struct{ service, expected string } - tests := []test{ - {blobServiceName, "http://127.0.0.1:10000/devstoreaccount1/"}, - {tableServiceName, "http://127.0.0.1:10002/devstoreaccount1/"}, - {queueServiceName, "http://127.0.0.1:10001/devstoreaccount1/"}, - } - for _, i := range tests { - endpoint := cli.getEndpoint(i.service, "", url.Values{}) - c.Assert(endpoint, chk.Equals, i.expected) - } -} - -func (s *StorageClientSuite) Test_getStandardHeaders(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - - headers := cli.getStandardHeaders() - c.Assert(len(headers), chk.Equals, 3) - c.Assert(headers["x-ms-version"], chk.Equals, cli.apiVersion) - if _, ok := headers["x-ms-date"]; !ok { - c.Fatal("Missing date header") - } - c.Assert(headers[userAgentHeader], chk.Equals, cli.getDefaultUserAgent()) -} - -func (s *StorageClientSuite) TestReturnsStorageServiceError(c *chk.C) { - // attempt to delete a nonexisting container - _, err := getBlobClient(c).deleteContainer(randContainer()) - c.Assert(err, chk.NotNil) - - v, ok := err.(AzureStorageServiceError) - c.Check(ok, chk.Equals, true) - c.Assert(v.StatusCode, chk.Equals, 404) - c.Assert(v.Code, chk.Equals, "ContainerNotFound") - c.Assert(v.Code, chk.Not(chk.Equals), "") - c.Assert(v.RequestID, chk.Not(chk.Equals), "") -} - -func (s *StorageClientSuite) TestReturnsStorageServiceError_withoutResponseBody(c *chk.C) { - // HEAD on non-existing blob - _, err := getBlobClient(c).GetBlobProperties("non-existing-blob", "non-existing-container") - - c.Assert(err, chk.NotNil) - c.Assert(err, chk.FitsTypeOf, AzureStorageServiceError{}) - - v, ok := err.(AzureStorageServiceError) - c.Check(ok, chk.Equals, true) - c.Assert(v.StatusCode, chk.Equals, http.StatusNotFound) - c.Assert(v.Code, chk.Equals, "404 The specified container does not exist.") - c.Assert(v.RequestID, chk.Not(chk.Equals), "") - c.Assert(v.Message, chk.Equals, "no response body was available for error status code") -} - -func (s *StorageClientSuite) Test_createServiceClients(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - - ua := cli.getDefaultUserAgent() - - headers := cli.getStandardHeaders() - c.Assert(headers[userAgentHeader], chk.Equals, ua) - c.Assert(cli.userAgent, chk.Equals, ua) - - b := cli.GetBlobService() - c.Assert(b.client.userAgent, chk.Equals, ua+" "+blobServiceName) - c.Assert(cli.userAgent, chk.Equals, ua) - - t := cli.GetTableService() - c.Assert(t.client.userAgent, chk.Equals, ua+" "+tableServiceName) - c.Assert(cli.userAgent, chk.Equals, ua) - - q := cli.GetQueueService() - c.Assert(q.client.userAgent, chk.Equals, ua+" "+queueServiceName) - c.Assert(cli.userAgent, chk.Equals, ua) - - f := cli.GetFileService() - c.Assert(f.client.userAgent, chk.Equals, ua+" "+fileServiceName) - c.Assert(cli.userAgent, chk.Equals, ua) -} - -func (s *StorageClientSuite) TestAddToUserAgent(c *chk.C) { - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - - ua := cli.getDefaultUserAgent() - - err = cli.AddToUserAgent("bar") - c.Assert(err, chk.IsNil) - c.Assert(cli.userAgent, chk.Equals, ua+" bar") - - err = cli.AddToUserAgent("") - c.Assert(err, chk.NotNil) -} - -func (s *StorageClientSuite) Test_protectUserAgent(c *chk.C) { - extraheaders := map[string]string{ - "1": "one", - "2": "two", - "3": "three", - userAgentHeader: "four", - } - - cli, err := NewBasicClient("foo", "YmFy") - c.Assert(err, chk.IsNil) - - ua := cli.getDefaultUserAgent() - - got := cli.protectUserAgent(extraheaders) - c.Assert(cli.userAgent, chk.Equals, ua+" four") - c.Assert(got, chk.HasLen, 3) - c.Assert(got, chk.DeepEquals, map[string]string{ - "1": "one", - "2": "two", - "3": "three", - }) -} diff --git a/storage/directory.go b/storage/directory.go deleted file mode 100644 index d07e0af1cc88..000000000000 --- a/storage/directory.go +++ /dev/null @@ -1,217 +0,0 @@ -package storage - -import ( - "encoding/xml" - "net/http" - "net/url" -) - -// Directory represents a directory on a share. -type Directory struct { - fsc *FileServiceClient - Metadata map[string]string - Name string `xml:"Name"` - parent *Directory - Properties DirectoryProperties - share *Share -} - -// DirectoryProperties contains various properties of a directory. -type DirectoryProperties struct { - LastModified string `xml:"Last-Modified"` - Etag string `xml:"Etag"` -} - -// ListDirsAndFilesParameters defines the set of customizable parameters to -// make a List Files and Directories call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166980.aspx -type ListDirsAndFilesParameters struct { - Marker string - MaxResults uint - Timeout uint -} - -// DirsAndFilesListResponse contains the response fields from -// a List Files and Directories call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166980.aspx -type DirsAndFilesListResponse struct { - XMLName xml.Name `xml:"EnumerationResults"` - Xmlns string `xml:"xmlns,attr"` - Marker string `xml:"Marker"` - MaxResults int64 `xml:"MaxResults"` - Directories []Directory `xml:"Entries>Directory"` - Files []File `xml:"Entries>File"` - NextMarker string `xml:"NextMarker"` -} - -// builds the complete directory path for this directory object. -func (d *Directory) buildPath() string { - path := "" - current := d - for current.Name != "" { - path = "/" + current.Name + path - current = current.parent - } - return d.share.buildPath() + path -} - -// Create this directory in the associated share. -// If a directory with the same name already exists, the operation fails. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166993.aspx -func (d *Directory) Create() error { - // if this is the root directory exit early - if d.parent == nil { - return nil - } - - headers, err := d.fsc.createResource(d.buildPath(), resourceDirectory, mergeMDIntoExtraHeaders(d.Metadata, nil)) - if err != nil { - return err - } - - d.updateEtagAndLastModified(headers) - return nil -} - -// CreateIfNotExists creates this directory under the associated share if the -// directory does not exists. Returns true if the directory is newly created or -// false if the directory already exists. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166993.aspx -func (d *Directory) CreateIfNotExists() (bool, error) { - // if this is the root directory exit early - if d.parent == nil { - return false, nil - } - - resp, err := d.fsc.createResourceNoClose(d.buildPath(), resourceDirectory, nil) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict { - if resp.statusCode == http.StatusCreated { - d.updateEtagAndLastModified(resp.headers) - return true, nil - } - - return false, d.FetchAttributes() - } - } - - return false, err -} - -// Delete removes this directory. It must be empty in order to be deleted. -// If the directory does not exist the operation fails. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166969.aspx -func (d *Directory) Delete() error { - return d.fsc.deleteResource(d.buildPath(), resourceDirectory) -} - -// DeleteIfExists removes this directory if it exists. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166969.aspx -func (d *Directory) DeleteIfExists() (bool, error) { - resp, err := d.fsc.deleteResourceNoClose(d.buildPath(), resourceDirectory) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusAccepted, nil - } - } - return false, err -} - -// Exists returns true if this directory exists. -func (d *Directory) Exists() (bool, error) { - exists, headers, err := d.fsc.resourceExists(d.buildPath(), resourceDirectory) - if exists { - d.updateEtagAndLastModified(headers) - } - return exists, err -} - -// FetchAttributes retrieves metadata for this directory. -func (d *Directory) FetchAttributes() error { - headers, err := d.fsc.getResourceHeaders(d.buildPath(), compNone, resourceDirectory, http.MethodHead) - if err != nil { - return err - } - - d.updateEtagAndLastModified(headers) - d.Metadata = getMetadataFromHeaders(headers) - - return nil -} - -// GetDirectoryReference returns a child Directory object for this directory. -func (d *Directory) GetDirectoryReference(name string) *Directory { - return &Directory{ - fsc: d.fsc, - Name: name, - parent: d, - share: d.share, - } -} - -// GetFileReference returns a child File object for this directory. -func (d *Directory) GetFileReference(name string) *File { - return &File{ - fsc: d.fsc, - Name: name, - parent: d, - share: d.share, - } -} - -// ListDirsAndFiles returns a list of files and directories under this directory. -// It also contains a pagination token and other response details. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166980.aspx -func (d *Directory) ListDirsAndFiles(params ListDirsAndFilesParameters) (*DirsAndFilesListResponse, error) { - q := mergeParams(params.getParameters(), getURLInitValues(compList, resourceDirectory)) - - resp, err := d.fsc.listContent(d.buildPath(), q, nil) - if err != nil { - return nil, err - } - - defer resp.body.Close() - var out DirsAndFilesListResponse - err = xmlUnmarshal(resp.body, &out) - return &out, err -} - -// SetMetadata replaces the metadata for this directory. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by GetDirectoryMetadata. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/mt427370.aspx -func (d *Directory) SetMetadata() error { - headers, err := d.fsc.setResourceHeaders(d.buildPath(), compMetadata, resourceDirectory, mergeMDIntoExtraHeaders(d.Metadata, nil)) - if err != nil { - return err - } - - d.updateEtagAndLastModified(headers) - return nil -} - -// updates Etag and last modified date -func (d *Directory) updateEtagAndLastModified(headers http.Header) { - d.Properties.Etag = headers.Get("Etag") - d.Properties.LastModified = headers.Get("Last-Modified") -} - -// URL gets the canonical URL to this directory. -// This method does not create a publicly accessible URL if the directory -// is private and this method does not check if the directory exists. -func (d *Directory) URL() string { - return d.fsc.client.getEndpoint(fileServiceName, d.buildPath(), url.Values{}) -} diff --git a/storage/directory_test.go b/storage/directory_test.go deleted file mode 100644 index 221348dba04b..000000000000 --- a/storage/directory_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package storage - -import chk "gopkg.in/check.v1" - -type StorageDirSuite struct{} - -var _ = chk.Suite(&StorageDirSuite{}) - -func (s *StorageDirSuite) TestListDirsAndFiles(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - // list contents, should be empty - resp, err := root.ListDirsAndFiles(ListDirsAndFilesParameters{}) - c.Assert(err, chk.IsNil) - c.Assert(resp.Directories, chk.IsNil) - c.Assert(resp.Files, chk.IsNil) - - // create a directory and a file - dir := root.GetDirectoryReference("SomeDirectory") - file := root.GetFileReference("foo.file") - c.Assert(dir.Create(), chk.IsNil) - c.Assert(file.Create(512), chk.IsNil) - - // list contents - resp, err = root.ListDirsAndFiles(ListDirsAndFilesParameters{}) - c.Assert(err, chk.IsNil) - c.Assert(len(resp.Directories), chk.Equals, 1) - c.Assert(len(resp.Files), chk.Equals, 1) - c.Assert(resp.Directories[0].Name, chk.Equals, dir.Name) - c.Assert(resp.Files[0].Name, chk.Equals, file.Name) - - // delete file - del, err := file.DeleteIfExists() - c.Assert(err, chk.IsNil) - c.Assert(del, chk.Equals, true) - - // attempt to delete again - del, err = file.DeleteIfExists() - c.Assert(err, chk.IsNil) - c.Assert(del, chk.Equals, false) -} - -func (s *StorageDirSuite) TestCreateDirectory(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - // directory shouldn't exist - dir := root.GetDirectoryReference("SomeDirectory") - exists, err := dir.Exists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, false) - - // create directory - exists, err = dir.CreateIfNotExists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, true) - - // try to create again, should fail - c.Assert(dir.Create(), chk.NotNil) - exists, err = dir.CreateIfNotExists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, false) - - // check properties - c.Assert(dir.Properties.Etag, chk.Not(chk.Equals), "") - c.Assert(dir.Properties.LastModified, chk.Not(chk.Equals), "") - - // delete directory and verify - c.Assert(dir.Delete(), chk.IsNil) - exists, err = dir.Exists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, false) -} - -func (s *StorageDirSuite) TestDirectoryMetadata(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - dir := root.GetDirectoryReference("testdir") - c.Assert(dir.Create(), chk.IsNil) - - // get metadata, shouldn't be any - c.Assert(dir.Metadata, chk.IsNil) - - // set some custom metadata - md := map[string]string{ - "something": "somethingvalue", - "another": "anothervalue", - } - dir.Metadata = md - c.Assert(dir.SetMetadata(), chk.IsNil) - - // retrieve and verify - c.Assert(dir.FetchAttributes(), chk.IsNil) - c.Assert(dir.Metadata, chk.DeepEquals, md) -} diff --git a/storage/file.go b/storage/file.go deleted file mode 100644 index 575f3f7266dc..000000000000 --- a/storage/file.go +++ /dev/null @@ -1,360 +0,0 @@ -package storage - -import ( - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strconv" -) - -const fourMB = uint64(4194304) -const oneTB = uint64(1099511627776) - -// File represents a file on a share. -type File struct { - fsc *FileServiceClient - Metadata map[string]string - Name string `xml:"Name"` - parent *Directory - Properties FileProperties `xml:"Properties"` - share *Share -} - -// FileProperties contains various properties of a file. -type FileProperties struct { - CacheControl string `header:"x-ms-cache-control"` - Disposition string `header:"x-ms-content-disposition"` - Encoding string `header:"x-ms-content-encoding"` - Etag string - Language string `header:"x-ms-content-language"` - LastModified string - Length uint64 `xml:"Content-Length"` - MD5 string `header:"x-ms-content-md5"` - Type string `header:"x-ms-content-type"` -} - -// FileCopyState contains various properties of a file copy operation. -type FileCopyState struct { - CompletionTime string - ID string - Progress string - Source string - Status string - StatusDesc string -} - -// FileStream contains file data returned from a call to GetFile. -type FileStream struct { - Body io.ReadCloser - ContentMD5 string -} - -// FileRanges contains a list of file range information for a file. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166984.aspx -type FileRanges struct { - ContentLength uint64 - LastModified string - ETag string - FileRanges []FileRange `xml:"Range"` -} - -// FileRange contains range information for a file. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166984.aspx -type FileRange struct { - Start uint64 `xml:"Start"` - End uint64 `xml:"End"` -} - -func (fr FileRange) String() string { - return fmt.Sprintf("bytes=%d-%d", fr.Start, fr.End) -} - -// builds the complete file path for this file object -func (f *File) buildPath() string { - return f.parent.buildPath() + "/" + f.Name -} - -// ClearRange releases the specified range of space in a file. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn194276.aspx -func (f *File) ClearRange(fileRange FileRange) error { - headers, err := f.modifyRange(nil, fileRange, nil) - if err != nil { - return err - } - - f.updateEtagAndLastModified(headers) - return nil -} - -// Create creates a new file or replaces an existing one. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn194271.aspx -func (f *File) Create(maxSize uint64) error { - if maxSize > oneTB { - return fmt.Errorf("max file size is 1TB") - } - - extraHeaders := map[string]string{ - "x-ms-content-length": strconv.FormatUint(maxSize, 10), - "x-ms-type": "file", - } - - headers, err := f.fsc.createResource(f.buildPath(), resourceFile, mergeMDIntoExtraHeaders(f.Metadata, extraHeaders)) - if err != nil { - return err - } - - f.Properties.Length = maxSize - f.updateEtagAndLastModified(headers) - return nil -} - -// Delete immediately removes this file from the storage account. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn689085.aspx -func (f *File) Delete() error { - return f.fsc.deleteResource(f.buildPath(), resourceFile) -} - -// DeleteIfExists removes this file if it exists. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn689085.aspx -func (f *File) DeleteIfExists() (bool, error) { - resp, err := f.fsc.deleteResourceNoClose(f.buildPath(), resourceFile) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusAccepted, nil - } - } - return false, err -} - -// DownloadRangeToStream operation downloads the specified range of this file with optional MD5 hash. -// -// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file -func (f *File) DownloadRangeToStream(fileRange FileRange, getContentMD5 bool) (fs FileStream, err error) { - if getContentMD5 && isRangeTooBig(fileRange) { - return fs, fmt.Errorf("must specify a range less than or equal to 4MB when getContentMD5 is true") - } - - extraHeaders := map[string]string{ - "Range": fileRange.String(), - } - if getContentMD5 == true { - extraHeaders["x-ms-range-get-content-md5"] = "true" - } - - resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, http.MethodGet, extraHeaders) - if err != nil { - return fs, err - } - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK, http.StatusPartialContent}); err != nil { - resp.body.Close() - return fs, err - } - - fs.Body = resp.body - if getContentMD5 { - fs.ContentMD5 = resp.headers.Get("Content-MD5") - } - return fs, nil -} - -// Exists returns true if this file exists. -func (f *File) Exists() (bool, error) { - exists, headers, err := f.fsc.resourceExists(f.buildPath(), resourceFile) - if exists { - f.updateEtagAndLastModified(headers) - f.updateProperties(headers) - } - return exists, err -} - -// FetchAttributes updates metadata and properties for this file. -func (f *File) FetchAttributes() error { - headers, err := f.fsc.getResourceHeaders(f.buildPath(), compNone, resourceFile, http.MethodHead) - if err != nil { - return err - } - - f.updateEtagAndLastModified(headers) - f.updateProperties(headers) - f.Metadata = getMetadataFromHeaders(headers) - return nil -} - -// returns true if the range is larger than 4MB -func isRangeTooBig(fileRange FileRange) bool { - if fileRange.End-fileRange.Start > fourMB { - return true - } - - return false -} - -// ListRanges returns the list of valid ranges for this file. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166984.aspx -func (f *File) ListRanges(listRange *FileRange) (*FileRanges, error) { - params := url.Values{"comp": {"rangelist"}} - - // add optional range to list - var headers map[string]string - if listRange != nil { - headers = make(map[string]string) - headers["Range"] = listRange.String() - } - - resp, err := f.fsc.listContent(f.buildPath(), params, headers) - if err != nil { - return nil, err - } - - defer resp.body.Close() - var cl uint64 - cl, err = strconv.ParseUint(resp.headers.Get("x-ms-content-length"), 10, 64) - if err != nil { - return nil, err - } - - var out FileRanges - out.ContentLength = cl - out.ETag = resp.headers.Get("ETag") - out.LastModified = resp.headers.Get("Last-Modified") - - err = xmlUnmarshal(resp.body, &out) - return &out, err -} - -// modifies a range of bytes in this file -func (f *File) modifyRange(bytes io.Reader, fileRange FileRange, contentMD5 *string) (http.Header, error) { - if err := f.fsc.checkForStorageEmulator(); err != nil { - return nil, err - } - if fileRange.End < fileRange.Start { - return nil, errors.New("the value for rangeEnd must be greater than or equal to rangeStart") - } - if bytes != nil && isRangeTooBig(fileRange) { - return nil, errors.New("range cannot exceed 4MB in size") - } - - uri := f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), url.Values{"comp": {"range"}}) - - // default to clear - write := "clear" - cl := uint64(0) - - // if bytes is not nil then this is an update operation - if bytes != nil { - write = "update" - cl = (fileRange.End - fileRange.Start) + 1 - } - - extraHeaders := map[string]string{ - "Content-Length": strconv.FormatUint(cl, 10), - "Range": fileRange.String(), - "x-ms-write": write, - } - - if contentMD5 != nil { - extraHeaders["Content-MD5"] = *contentMD5 - } - - headers := mergeHeaders(f.fsc.client.getStandardHeaders(), extraHeaders) - resp, err := f.fsc.client.exec(http.MethodPut, uri, headers, bytes, f.fsc.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - return resp.headers, checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// SetMetadata replaces the metadata for this file. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by GetFileMetadata. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn689097.aspx -func (f *File) SetMetadata() error { - headers, err := f.fsc.setResourceHeaders(f.buildPath(), compMetadata, resourceFile, mergeMDIntoExtraHeaders(f.Metadata, nil)) - if err != nil { - return err - } - - f.updateEtagAndLastModified(headers) - return nil -} - -// SetProperties sets system properties on this file. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by SetFileProperties. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn166975.aspx -func (f *File) SetProperties() error { - headers, err := f.fsc.setResourceHeaders(f.buildPath(), compProperties, resourceFile, headersFromStruct(f.Properties)) - if err != nil { - return err - } - - f.updateEtagAndLastModified(headers) - return nil -} - -// updates Etag and last modified date -func (f *File) updateEtagAndLastModified(headers http.Header) { - f.Properties.Etag = headers.Get("Etag") - f.Properties.LastModified = headers.Get("Last-Modified") -} - -// updates file properties from the specified HTTP header -func (f *File) updateProperties(header http.Header) { - size, err := strconv.ParseUint(header.Get("Content-Length"), 10, 64) - if err == nil { - f.Properties.Length = size - } - - f.updateEtagAndLastModified(header) - f.Properties.CacheControl = header.Get("Cache-Control") - f.Properties.Disposition = header.Get("Content-Disposition") - f.Properties.Encoding = header.Get("Content-Encoding") - f.Properties.Language = header.Get("Content-Language") - f.Properties.MD5 = header.Get("Content-MD5") - f.Properties.Type = header.Get("Content-Type") -} - -// URL gets the canonical URL to this file. -// This method does not create a publicly accessible URL if the file -// is private and this method does not check if the file exists. -func (f *File) URL() string { - return f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), url.Values{}) -} - -// WriteRange writes a range of bytes to this file with an optional MD5 hash of the content. -// Note that the length of bytes must match (rangeEnd - rangeStart) + 1 with a maximum size of 4MB. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn194276.aspx -func (f *File) WriteRange(bytes io.Reader, fileRange FileRange, contentMD5 *string) error { - if bytes == nil { - return errors.New("bytes cannot be nil") - } - - headers, err := f.modifyRange(bytes, fileRange, contentMD5) - if err != nil { - return err - } - - f.updateEtagAndLastModified(headers) - return nil -} diff --git a/storage/file_test.go b/storage/file_test.go deleted file mode 100644 index 15c1ca9de48e..000000000000 --- a/storage/file_test.go +++ /dev/null @@ -1,274 +0,0 @@ -package storage - -import ( - "bytes" - "crypto/md5" - "encoding/base64" - "io" - - chk "gopkg.in/check.v1" -) - -type StorageFileSuite struct{} - -var _ = chk.Suite(&StorageFileSuite{}) - -func (s *StorageFileSuite) TestCreateFile(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - // create directory structure - dir1 := root.GetDirectoryReference("one") - c.Assert(dir1.Create(), chk.IsNil) - dir2 := dir1.GetDirectoryReference("two") - c.Assert(dir2.Create(), chk.IsNil) - - // verify file doesn't exist - file := dir2.GetFileReference("some.file") - exists, err := file.Exists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, false) - - // create file - c.Assert(file.Create(1024), chk.IsNil) - exists, err = file.Exists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, true) - - // delete file and verify - c.Assert(file.Delete(), chk.IsNil) - exists, err = file.Exists() - c.Assert(err, chk.IsNil) - c.Assert(exists, chk.Equals, false) -} - -func (s *StorageFileSuite) TestGetFile(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - // create file - const size = uint64(1024) - byteStream, _ := newByteStream(size) - file := root.GetFileReference("some.file") - c.Assert(file.Create(size), chk.IsNil) - - // fill file with some data - c.Assert(file.WriteRange(byteStream, FileRange{End: size - 1}, nil), chk.IsNil) - - // set some metadata - md := map[string]string{ - "something": "somethingvalue", - "another": "anothervalue", - } - file.Metadata = md - c.Assert(file.SetMetadata(), chk.IsNil) - - // retrieve full file content and verify - stream, err := file.DownloadRangeToStream(FileRange{Start: 0, End: size - 1}, false) - c.Assert(err, chk.IsNil) - defer stream.Body.Close() - var b1 [size]byte - count, _ := stream.Body.Read(b1[:]) - c.Assert(count, chk.Equals, int(size)) - var c1 [size]byte - bs, _ := newByteStream(size) - bs.Read(c1[:]) - c.Assert(b1, chk.DeepEquals, c1) - - // retrieve partial file content and verify - stream, err = file.DownloadRangeToStream(FileRange{Start: size / 2, End: size - 1}, false) - c.Assert(err, chk.IsNil) - defer stream.Body.Close() - var b2 [size / 2]byte - count, _ = stream.Body.Read(b2[:]) - c.Assert(count, chk.Equals, int(size)/2) - var c2 [size / 2]byte - bs, _ = newByteStream(size / 2) - bs.Read(c2[:]) - c.Assert(b2, chk.DeepEquals, c2) -} - -func (s *StorageFileSuite) TestFileRanges(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - // create file - fileSize := uint64(4096) - byteStream, _ := newByteStream(fileSize) - file := root.GetFileReference("test.dat") - c.Assert(file.Create(fileSize), chk.IsNil) - - // verify there are no valid ranges - ranges, err := file.ListRanges(nil) - c.Assert(err, chk.IsNil) - c.Assert(ranges.ContentLength, chk.Equals, fileSize) - c.Assert(ranges.FileRanges, chk.IsNil) - - // fill entire range and validate - c.Assert(file.WriteRange(byteStream, FileRange{End: fileSize - 1}, nil), chk.IsNil) - ranges, err = file.ListRanges(nil) - c.Assert(err, chk.IsNil) - c.Assert(len(ranges.FileRanges), chk.Equals, 1) - c.Assert((ranges.FileRanges[0].End-ranges.FileRanges[0].Start)+1, chk.Equals, fileSize) - - // clear entire range and validate - c.Assert(file.ClearRange(FileRange{End: fileSize - 1}), chk.IsNil) - ranges, err = file.ListRanges(nil) - c.Assert(err, chk.IsNil) - c.Assert(ranges.FileRanges, chk.IsNil) - - // put partial ranges on 512 byte aligned boundaries - putRanges := []FileRange{ - {End: 511}, - {Start: 1024, End: 1535}, - {Start: 2048, End: 2559}, - {Start: 3072, End: 3583}, - } - - for _, r := range putRanges { - byteStream, _ = newByteStream(512) - err = file.WriteRange(byteStream, r, nil) - c.Assert(err, chk.IsNil) - } - - // validate all ranges - ranges, err = file.ListRanges(nil) - c.Assert(err, chk.IsNil) - c.Assert(ranges.FileRanges, chk.DeepEquals, putRanges) - - // validate sub-ranges - ranges, err = file.ListRanges(&FileRange{Start: 1000, End: 3000}) - c.Assert(err, chk.IsNil) - c.Assert(ranges.FileRanges, chk.DeepEquals, putRanges[1:3]) - - // clear partial range and validate - c.Assert(file.ClearRange(putRanges[0]), chk.IsNil) - c.Assert(file.ClearRange(putRanges[2]), chk.IsNil) - ranges, err = file.ListRanges(nil) - c.Assert(err, chk.IsNil) - c.Assert(ranges.FileRanges, chk.HasLen, 2) - c.Assert(ranges.FileRanges[0], chk.DeepEquals, putRanges[1]) - c.Assert(ranges.FileRanges[1], chk.DeepEquals, putRanges[3]) -} - -func (s *StorageFileSuite) TestFileProperties(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - fileSize := uint64(512) - file := root.GetFileReference("test.dat") - c.Assert(file.Create(fileSize), chk.IsNil) - - // get initial set of properties - c.Assert(file.Properties.Length, chk.Equals, fileSize) - c.Assert(file.Properties.Etag, chk.NotNil) - - // set some file properties - cc := "cachecontrol" - ct := "mytype" - enc := "noencoding" - lang := "neutral" - disp := "friendly" - file.Properties.CacheControl = cc - file.Properties.Type = ct - file.Properties.Disposition = disp - file.Properties.Encoding = enc - file.Properties.Language = lang - c.Assert(file.SetProperties(), chk.IsNil) - - // retrieve and verify - c.Assert(file.FetchAttributes(), chk.IsNil) - c.Assert(file.Properties.CacheControl, chk.Equals, cc) - c.Assert(file.Properties.Type, chk.Equals, ct) - c.Assert(file.Properties.Disposition, chk.Equals, disp) - c.Assert(file.Properties.Encoding, chk.Equals, enc) - c.Assert(file.Properties.Language, chk.Equals, lang) -} - -func (s *StorageFileSuite) TestFileMetadata(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - fileSize := uint64(512) - file := root.GetFileReference("test.dat") - c.Assert(file.Create(fileSize), chk.IsNil) - - // get metadata, shouldn't be any - c.Assert(file.Metadata, chk.HasLen, 0) - - // set some custom metadata - md := map[string]string{ - "something": "somethingvalue", - "another": "anothervalue", - } - file.Metadata = md - c.Assert(file.SetMetadata(), chk.IsNil) - - // retrieve and verify - c.Assert(file.FetchAttributes(), chk.IsNil) - c.Assert(file.Metadata, chk.DeepEquals, md) -} - -func (s *StorageFileSuite) TestFileMD5(c *chk.C) { - // create share - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - root := share.GetRootDirectoryReference() - - // create file - const size = uint64(1024) - fileSize := uint64(size) - file := root.GetFileReference("test.dat") - c.Assert(file.Create(fileSize), chk.IsNil) - - // fill file with some data and MD5 hash - byteStream, contentMD5 := newByteStream(size) - c.Assert(file.WriteRange(byteStream, FileRange{End: size - 1}, &contentMD5), chk.IsNil) - - // download file and verify - stream, err := file.DownloadRangeToStream(FileRange{Start: 0, End: size - 1}, true) - c.Assert(err, chk.IsNil) - defer stream.Body.Close() - c.Assert(stream.ContentMD5, chk.Equals, contentMD5) -} - -// returns a byte stream along with a base-64 encoded MD5 hash of its contents -func newByteStream(count uint64) (io.Reader, string) { - b := make([]uint8, count) - for i := uint64(0); i < count; i++ { - b[i] = 0xff - } - - // create an MD5 hash of the array - hash := md5.Sum(b) - - return bytes.NewReader(b), base64.StdEncoding.EncodeToString(hash[:]) -} diff --git a/storage/fileserviceclient.go b/storage/fileserviceclient.go deleted file mode 100644 index 81e094f00d3a..000000000000 --- a/storage/fileserviceclient.go +++ /dev/null @@ -1,360 +0,0 @@ -package storage - -import ( - "encoding/xml" - "fmt" - "net/http" - "net/url" - "strings" -) - -// FileServiceClient contains operations for Microsoft Azure File Service. -type FileServiceClient struct { - client Client - auth authentication -} - -// ListSharesParameters defines the set of customizable parameters to make a -// List Shares call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn167009.aspx -type ListSharesParameters struct { - Prefix string - Marker string - Include string - MaxResults uint - Timeout uint -} - -// ShareListResponse contains the response fields from -// ListShares call. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn167009.aspx -type ShareListResponse struct { - XMLName xml.Name `xml:"EnumerationResults"` - Xmlns string `xml:"xmlns,attr"` - Prefix string `xml:"Prefix"` - Marker string `xml:"Marker"` - NextMarker string `xml:"NextMarker"` - MaxResults int64 `xml:"MaxResults"` - Shares []Share `xml:"Shares>Share"` -} - -type compType string - -const ( - compNone compType = "" - compList compType = "list" - compMetadata compType = "metadata" - compProperties compType = "properties" - compRangeList compType = "rangelist" -) - -func (ct compType) String() string { - return string(ct) -} - -type resourceType string - -const ( - resourceDirectory resourceType = "directory" - resourceFile resourceType = "" - resourceShare resourceType = "share" -) - -func (rt resourceType) String() string { - return string(rt) -} - -func (p ListSharesParameters) getParameters() url.Values { - out := url.Values{} - - if p.Prefix != "" { - out.Set("prefix", p.Prefix) - } - if p.Marker != "" { - out.Set("marker", p.Marker) - } - if p.Include != "" { - out.Set("include", p.Include) - } - if p.MaxResults != 0 { - out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults)) - } - if p.Timeout != 0 { - out.Set("timeout", fmt.Sprintf("%v", p.Timeout)) - } - - return out -} - -func (p ListDirsAndFilesParameters) getParameters() url.Values { - out := url.Values{} - - if p.Marker != "" { - out.Set("marker", p.Marker) - } - if p.MaxResults != 0 { - out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults)) - } - if p.Timeout != 0 { - out.Set("timeout", fmt.Sprintf("%v", p.Timeout)) - } - - return out -} - -// returns url.Values for the specified types -func getURLInitValues(comp compType, res resourceType) url.Values { - values := url.Values{} - if comp != compNone { - values.Set("comp", comp.String()) - } - if res != resourceFile { - values.Set("restype", res.String()) - } - return values -} - -// GetShareReference returns a Share object for the specified share name. -func (f FileServiceClient) GetShareReference(name string) Share { - return Share{ - fsc: &f, - Name: name, - Properties: ShareProperties{ - Quota: -1, - }, - } -} - -// ListShares returns the list of shares in a storage account along with -// pagination token and other response details. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx -func (f FileServiceClient) ListShares(params ListSharesParameters) (*ShareListResponse, error) { - q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}}) - - var out ShareListResponse - resp, err := f.listContent("", q, nil) - if err != nil { - return nil, err - } - defer resp.body.Close() - err = xmlUnmarshal(resp.body, &out) - - // assign our client to the newly created Share objects - for i := range out.Shares { - out.Shares[i].fsc = &f - } - return &out, err -} - -// retrieves directory or share content -func (f FileServiceClient) listContent(path string, params url.Values, extraHeaders map[string]string) (*storageResponse, error) { - if err := f.checkForStorageEmulator(); err != nil { - return nil, err - } - - uri := f.client.getEndpoint(fileServiceName, path, params) - extraHeaders = f.client.protectUserAgent(extraHeaders) - headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) - - resp, err := f.client.exec(http.MethodGet, uri, headers, nil, f.auth) - if err != nil { - return nil, err - } - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - resp.body.Close() - return nil, err - } - - return resp, nil -} - -// returns true if the specified resource exists -func (f FileServiceClient) resourceExists(path string, res resourceType) (bool, http.Header, error) { - if err := f.checkForStorageEmulator(); err != nil { - return false, nil, err - } - - uri := f.client.getEndpoint(fileServiceName, path, getURLInitValues(compNone, res)) - headers := f.client.getStandardHeaders() - - resp, err := f.client.exec(http.MethodHead, uri, headers, nil, f.auth) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusOK, resp.headers, nil - } - } - return false, nil, err -} - -// creates a resource depending on the specified resource type -func (f FileServiceClient) createResource(path string, res resourceType, extraHeaders map[string]string) (http.Header, error) { - resp, err := f.createResourceNoClose(path, res, extraHeaders) - if err != nil { - return nil, err - } - defer resp.body.Close() - return resp.headers, checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// creates a resource depending on the specified resource type, doesn't close the response body -func (f FileServiceClient) createResourceNoClose(path string, res resourceType, extraHeaders map[string]string) (*storageResponse, error) { - if err := f.checkForStorageEmulator(); err != nil { - return nil, err - } - - values := getURLInitValues(compNone, res) - uri := f.client.getEndpoint(fileServiceName, path, values) - extraHeaders = f.client.protectUserAgent(extraHeaders) - headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) - - return f.client.exec(http.MethodPut, uri, headers, nil, f.auth) -} - -// returns HTTP header data for the specified directory or share -func (f FileServiceClient) getResourceHeaders(path string, comp compType, res resourceType, verb string) (http.Header, error) { - resp, err := f.getResourceNoClose(path, comp, res, verb, nil) - if err != nil { - return nil, err - } - defer resp.body.Close() - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } - - return resp.headers, nil -} - -// gets the specified resource, doesn't close the response body -func (f FileServiceClient) getResourceNoClose(path string, comp compType, res resourceType, verb string, extraHeaders map[string]string) (*storageResponse, error) { - if err := f.checkForStorageEmulator(); err != nil { - return nil, err - } - - params := getURLInitValues(comp, res) - uri := f.client.getEndpoint(fileServiceName, path, params) - extraHeaders = f.client.protectUserAgent(extraHeaders) - headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) - - return f.client.exec(verb, uri, headers, nil, f.auth) -} - -// deletes the resource and returns the response -func (f FileServiceClient) deleteResource(path string, res resourceType) error { - resp, err := f.deleteResourceNoClose(path, res) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) -} - -// deletes the resource and returns the response, doesn't close the response body -func (f FileServiceClient) deleteResourceNoClose(path string, res resourceType) (*storageResponse, error) { - if err := f.checkForStorageEmulator(); err != nil { - return nil, err - } - - values := getURLInitValues(compNone, res) - uri := f.client.getEndpoint(fileServiceName, path, values) - return f.client.exec(http.MethodDelete, uri, f.client.getStandardHeaders(), nil, f.auth) -} - -// merges metadata into extraHeaders and returns extraHeaders -func mergeMDIntoExtraHeaders(metadata, extraHeaders map[string]string) map[string]string { - if metadata == nil && extraHeaders == nil { - return nil - } - if extraHeaders == nil { - extraHeaders = make(map[string]string) - } - for k, v := range metadata { - extraHeaders[userDefinedMetadataHeaderPrefix+k] = v - } - return extraHeaders -} - -// merges extraHeaders into headers and returns headers -func mergeHeaders(headers, extraHeaders map[string]string) map[string]string { - for k, v := range extraHeaders { - headers[k] = v - } - return headers -} - -// sets extra header data for the specified resource -func (f FileServiceClient) setResourceHeaders(path string, comp compType, res resourceType, extraHeaders map[string]string) (http.Header, error) { - if err := f.checkForStorageEmulator(); err != nil { - return nil, err - } - - params := getURLInitValues(comp, res) - uri := f.client.getEndpoint(fileServiceName, path, params) - extraHeaders = f.client.protectUserAgent(extraHeaders) - headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) - - resp, err := f.client.exec(http.MethodPut, uri, headers, nil, f.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - return resp.headers, checkRespCode(resp.statusCode, []int{http.StatusOK}) -} - -// gets metadata for the specified resource -func (f FileServiceClient) getMetadata(path string, res resourceType) (map[string]string, error) { - if err := f.checkForStorageEmulator(); err != nil { - return nil, err - } - - headers, err := f.getResourceHeaders(path, compMetadata, res, http.MethodGet) - if err != nil { - return nil, err - } - - return getMetadataFromHeaders(headers), nil -} - -// returns a map of custom metadata values from the specified HTTP header -func getMetadataFromHeaders(header http.Header) map[string]string { - metadata := make(map[string]string) - for k, v := range header { - // Can't trust CanonicalHeaderKey() to munge case - // reliably. "_" is allowed in identifiers: - // https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx - // https://msdn.microsoft.com/library/aa664670(VS.71).aspx - // http://tools.ietf.org/html/rfc7230#section-3.2 - // ...but "_" is considered invalid by - // CanonicalMIMEHeaderKey in - // https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542 - // so k can be "X-Ms-Meta-Foo" or "x-ms-meta-foo_bar". - k = strings.ToLower(k) - if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) { - continue - } - // metadata["foo"] = content of the last X-Ms-Meta-Foo header - k = k[len(userDefinedMetadataHeaderPrefix):] - metadata[k] = v[len(v)-1] - } - - if len(metadata) == 0 { - return nil - } - - return metadata -} - -//checkForStorageEmulator determines if the client is setup for use with -//Azure Storage Emulator, and returns a relevant error -func (f FileServiceClient) checkForStorageEmulator() error { - if f.client.accountName == StorageEmulatorAccountName { - return fmt.Errorf("Error: File service is not currently supported by Azure Storage Emulator") - } - return nil -} diff --git a/storage/queue.go b/storage/queue.go deleted file mode 100644 index 4dbb67733105..000000000000 --- a/storage/queue.go +++ /dev/null @@ -1,346 +0,0 @@ -package storage - -import ( - "encoding/xml" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" -) - -const ( - // casing is per Golang's http.Header canonicalizing the header names. - approximateMessagesCountHeader = "X-Ms-Approximate-Messages-Count" - userDefinedMetadataHeaderPrefix = "X-Ms-Meta-" -) - -// QueueServiceClient contains operations for Microsoft Azure Queue Storage -// Service. -type QueueServiceClient struct { - client Client - auth authentication -} - -func pathForQueue(queue string) string { return fmt.Sprintf("/%s", queue) } -func pathForQueueMessages(queue string) string { return fmt.Sprintf("/%s/messages", queue) } -func pathForMessage(queue, name string) string { return fmt.Sprintf("/%s/messages/%s", queue, name) } - -type putMessageRequest struct { - XMLName xml.Name `xml:"QueueMessage"` - MessageText string `xml:"MessageText"` -} - -// PutMessageParameters is the set of options can be specified for Put Messsage -// operation. A zero struct does not use any preferences for the request. -type PutMessageParameters struct { - VisibilityTimeout int - MessageTTL int -} - -func (p PutMessageParameters) getParameters() url.Values { - out := url.Values{} - if p.VisibilityTimeout != 0 { - out.Set("visibilitytimeout", strconv.Itoa(p.VisibilityTimeout)) - } - if p.MessageTTL != 0 { - out.Set("messagettl", strconv.Itoa(p.MessageTTL)) - } - return out -} - -// GetMessagesParameters is the set of options can be specified for Get -// Messsages operation. A zero struct does not use any preferences for the -// request. -type GetMessagesParameters struct { - NumOfMessages int - VisibilityTimeout int -} - -func (p GetMessagesParameters) getParameters() url.Values { - out := url.Values{} - if p.NumOfMessages != 0 { - out.Set("numofmessages", strconv.Itoa(p.NumOfMessages)) - } - if p.VisibilityTimeout != 0 { - out.Set("visibilitytimeout", strconv.Itoa(p.VisibilityTimeout)) - } - return out -} - -// PeekMessagesParameters is the set of options can be specified for Peek -// Messsage operation. A zero struct does not use any preferences for the -// request. -type PeekMessagesParameters struct { - NumOfMessages int -} - -func (p PeekMessagesParameters) getParameters() url.Values { - out := url.Values{"peekonly": {"true"}} // Required for peek operation - if p.NumOfMessages != 0 { - out.Set("numofmessages", strconv.Itoa(p.NumOfMessages)) - } - return out -} - -// UpdateMessageParameters is the set of options can be specified for Update Messsage -// operation. A zero struct does not use any preferences for the request. -type UpdateMessageParameters struct { - PopReceipt string - VisibilityTimeout int -} - -func (p UpdateMessageParameters) getParameters() url.Values { - out := url.Values{} - if p.PopReceipt != "" { - out.Set("popreceipt", p.PopReceipt) - } - if p.VisibilityTimeout != 0 { - out.Set("visibilitytimeout", strconv.Itoa(p.VisibilityTimeout)) - } - return out -} - -// GetMessagesResponse represents a response returned from Get Messages -// operation. -type GetMessagesResponse struct { - XMLName xml.Name `xml:"QueueMessagesList"` - QueueMessagesList []GetMessageResponse `xml:"QueueMessage"` -} - -// GetMessageResponse represents a QueueMessage object returned from Get -// Messages operation response. -type GetMessageResponse struct { - MessageID string `xml:"MessageId"` - InsertionTime string `xml:"InsertionTime"` - ExpirationTime string `xml:"ExpirationTime"` - PopReceipt string `xml:"PopReceipt"` - TimeNextVisible string `xml:"TimeNextVisible"` - DequeueCount int `xml:"DequeueCount"` - MessageText string `xml:"MessageText"` -} - -// PeekMessagesResponse represents a response returned from Get Messages -// operation. -type PeekMessagesResponse struct { - XMLName xml.Name `xml:"QueueMessagesList"` - QueueMessagesList []PeekMessageResponse `xml:"QueueMessage"` -} - -// PeekMessageResponse represents a QueueMessage object returned from Peek -// Messages operation response. -type PeekMessageResponse struct { - MessageID string `xml:"MessageId"` - InsertionTime string `xml:"InsertionTime"` - ExpirationTime string `xml:"ExpirationTime"` - DequeueCount int `xml:"DequeueCount"` - MessageText string `xml:"MessageText"` -} - -// QueueMetadataResponse represents user defined metadata and queue -// properties on a specific queue. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179384.aspx -type QueueMetadataResponse struct { - ApproximateMessageCount int - UserDefinedMetadata map[string]string -} - -// SetMetadata operation sets user-defined metadata on the specified queue. -// Metadata is associated with the queue as name-value pairs. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179348.aspx -func (c QueueServiceClient) SetMetadata(name string, metadata map[string]string) error { - uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{"comp": []string{"metadata"}}) - metadata = c.client.protectUserAgent(metadata) - headers := c.client.getStandardHeaders() - for k, v := range metadata { - headers[userDefinedMetadataHeaderPrefix+k] = v - } - - resp, err := c.client.exec(http.MethodPut, uri, headers, nil, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) -} - -// GetMetadata operation retrieves user-defined metadata and queue -// properties on the specified queue. Metadata is associated with -// the queue as name-values pairs. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179384.aspx -// -// Because the way Golang's http client (and http.Header in particular) -// canonicalize header names, the returned metadata names would always -// be all lower case. -func (c QueueServiceClient) GetMetadata(name string) (QueueMetadataResponse, error) { - qm := QueueMetadataResponse{} - qm.UserDefinedMetadata = make(map[string]string) - uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{"comp": []string{"metadata"}}) - headers := c.client.getStandardHeaders() - resp, err := c.client.exec(http.MethodGet, uri, headers, nil, c.auth) - if err != nil { - return qm, err - } - defer resp.body.Close() - - for k, v := range resp.headers { - if len(v) != 1 { - return qm, fmt.Errorf("Unexpected number of values (%d) in response header '%s'", len(v), k) - } - - value := v[0] - - if k == approximateMessagesCountHeader { - qm.ApproximateMessageCount, err = strconv.Atoi(value) - if err != nil { - return qm, fmt.Errorf("Unexpected value in response header '%s': '%s' ", k, value) - } - } else if strings.HasPrefix(k, userDefinedMetadataHeaderPrefix) { - name := strings.TrimPrefix(k, userDefinedMetadataHeaderPrefix) - qm.UserDefinedMetadata[strings.ToLower(name)] = value - } - } - - return qm, checkRespCode(resp.statusCode, []int{http.StatusOK}) -} - -// CreateQueue operation creates a queue under the given account. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179342.aspx -func (c QueueServiceClient) CreateQueue(name string) error { - uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{}) - headers := c.client.getStandardHeaders() - resp, err := c.client.exec(http.MethodPut, uri, headers, nil, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// DeleteQueue operation permanently deletes the specified queue. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179436.aspx -func (c QueueServiceClient) DeleteQueue(name string) error { - uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{}) - resp, err := c.client.exec(http.MethodDelete, uri, c.client.getStandardHeaders(), nil, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) -} - -// QueueExists returns true if a queue with given name exists. -func (c QueueServiceClient) QueueExists(name string) (bool, error) { - uri := c.client.getEndpoint(queueServiceName, pathForQueue(name), url.Values{"comp": {"metadata"}}) - resp, err := c.client.exec(http.MethodGet, uri, c.client.getStandardHeaders(), nil, c.auth) - if resp != nil && (resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound) { - return resp.statusCode == http.StatusOK, nil - } - - return false, err -} - -// PutMessage operation adds a new message to the back of the message queue. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179346.aspx -func (c QueueServiceClient) PutMessage(queue string, message string, params PutMessageParameters) error { - uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), params.getParameters()) - req := putMessageRequest{MessageText: message} - body, nn, err := xmlMarshal(req) - if err != nil { - return err - } - headers := c.client.getStandardHeaders() - headers["Content-Length"] = strconv.Itoa(nn) - resp, err := c.client.exec(http.MethodPost, uri, headers, body, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusCreated}) -} - -// ClearMessages operation deletes all messages from the specified queue. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179454.aspx -func (c QueueServiceClient) ClearMessages(queue string) error { - uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), url.Values{}) - resp, err := c.client.exec(http.MethodDelete, uri, c.client.getStandardHeaders(), nil, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) -} - -// GetMessages operation retrieves one or more messages from the front of the -// queue. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179474.aspx -func (c QueueServiceClient) GetMessages(queue string, params GetMessagesParameters) (GetMessagesResponse, error) { - var r GetMessagesResponse - uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), params.getParameters()) - resp, err := c.client.exec(http.MethodGet, uri, c.client.getStandardHeaders(), nil, c.auth) - if err != nil { - return r, err - } - defer resp.body.Close() - err = xmlUnmarshal(resp.body, &r) - return r, err -} - -// PeekMessages retrieves one or more messages from the front of the queue, but -// does not alter the visibility of the message. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179472.aspx -func (c QueueServiceClient) PeekMessages(queue string, params PeekMessagesParameters) (PeekMessagesResponse, error) { - var r PeekMessagesResponse - uri := c.client.getEndpoint(queueServiceName, pathForQueueMessages(queue), params.getParameters()) - resp, err := c.client.exec(http.MethodGet, uri, c.client.getStandardHeaders(), nil, c.auth) - if err != nil { - return r, err - } - defer resp.body.Close() - err = xmlUnmarshal(resp.body, &r) - return r, err -} - -// DeleteMessage operation deletes the specified message. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179347.aspx -func (c QueueServiceClient) DeleteMessage(queue, messageID, popReceipt string) error { - uri := c.client.getEndpoint(queueServiceName, pathForMessage(queue, messageID), url.Values{ - "popreceipt": {popReceipt}}) - resp, err := c.client.exec(http.MethodDelete, uri, c.client.getStandardHeaders(), nil, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) -} - -// UpdateMessage operation deletes the specified message. -// -// See https://msdn.microsoft.com/en-us/library/azure/hh452234.aspx -func (c QueueServiceClient) UpdateMessage(queue string, messageID string, message string, params UpdateMessageParameters) error { - uri := c.client.getEndpoint(queueServiceName, pathForMessage(queue, messageID), params.getParameters()) - req := putMessageRequest{MessageText: message} - body, nn, err := xmlMarshal(req) - if err != nil { - return err - } - headers := c.client.getStandardHeaders() - headers["Content-Length"] = fmt.Sprintf("%d", nn) - resp, err := c.client.exec(http.MethodPut, uri, headers, body, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) -} diff --git a/storage/queue_test.go b/storage/queue_test.go deleted file mode 100644 index 45a8901d6beb..000000000000 --- a/storage/queue_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package storage - -import ( - "time" - - chk "gopkg.in/check.v1" -) - -type StorageQueueSuite struct{} - -var _ = chk.Suite(&StorageQueueSuite{}) - -func getQueueClient(c *chk.C) QueueServiceClient { - return getBasicClient(c).GetQueueService() -} - -func (s *StorageQueueSuite) Test_pathForQueue(c *chk.C) { - c.Assert(pathForQueue("q"), chk.Equals, "/q") -} - -func (s *StorageQueueSuite) Test_pathForQueueMessages(c *chk.C) { - c.Assert(pathForQueueMessages("q"), chk.Equals, "/q/messages") -} - -func (s *StorageQueueSuite) Test_pathForMessage(c *chk.C) { - c.Assert(pathForMessage("q", "m"), chk.Equals, "/q/messages/m") -} - -func (s *StorageQueueSuite) TestCreateQueue_DeleteQueue(c *chk.C) { - cli := getQueueClient(c) - name := randString(20) - c.Assert(cli.CreateQueue(name), chk.IsNil) - c.Assert(cli.DeleteQueue(name), chk.IsNil) -} - -func (s *StorageQueueSuite) Test_GetMetadata_GetApproximateCount(c *chk.C) { - cli := getQueueClient(c) - name := randString(20) - c.Assert(cli.CreateQueue(name), chk.IsNil) - defer cli.DeleteQueue(name) - - qm, err := cli.GetMetadata(name) - c.Assert(err, chk.IsNil) - c.Assert(qm.ApproximateMessageCount, chk.Equals, 0) - - for ix := 0; ix < 3; ix++ { - err = cli.PutMessage(name, "foobar", PutMessageParameters{}) - c.Assert(err, chk.IsNil) - } - time.Sleep(1 * time.Second) - - qm, err = cli.GetMetadata(name) - c.Assert(err, chk.IsNil) - c.Assert(qm.ApproximateMessageCount, chk.Equals, 3) -} - -func (s *StorageQueueSuite) Test_SetMetadataGetMetadata_Roundtrips(c *chk.C) { - cli := getQueueClient(c) - name := randString(20) - c.Assert(cli.CreateQueue(name), chk.IsNil) - defer cli.DeleteQueue(name) - - metadata := make(map[string]string) - metadata["Foo1"] = "bar1" - metadata["fooBaz"] = "bar" - err := cli.SetMetadata(name, metadata) - c.Assert(err, chk.IsNil) - - qm, err := cli.GetMetadata(name) - c.Assert(err, chk.IsNil) - c.Assert(qm.UserDefinedMetadata["foo1"], chk.Equals, "bar1") - c.Assert(qm.UserDefinedMetadata["foobaz"], chk.Equals, "bar") -} - -func (s *StorageQueueSuite) TestQueueExists(c *chk.C) { - cli := getQueueClient(c) - ok, err := cli.QueueExists("nonexistent-queue") - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - name := randString(20) - c.Assert(cli.CreateQueue(name), chk.IsNil) - defer cli.DeleteQueue(name) - - ok, err = cli.QueueExists(name) - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) -} - -func (s *StorageQueueSuite) TestPutMessage_PeekMessage_UpdateMessage_DeleteMessage(c *chk.C) { - q := randString(20) - cli := getQueueClient(c) - c.Assert(cli.CreateQueue(q), chk.IsNil) - defer cli.DeleteQueue(q) - - msg := randString(64 * 1024) // exercise max length - c.Assert(cli.PutMessage(q, msg, PutMessageParameters{}), chk.IsNil) - r, err := cli.PeekMessages(q, PeekMessagesParameters{}) - c.Assert(err, chk.IsNil) - c.Assert(len(r.QueueMessagesList), chk.Equals, 1) - c.Assert(r.QueueMessagesList[0].MessageText, chk.Equals, msg) - - gr, gerr := cli.GetMessages(q, GetMessagesParameters{NumOfMessages: 1, VisibilityTimeout: 2}) - c.Assert(gerr, chk.IsNil) - - updatedMsg := "Test Message" - c.Assert(cli.UpdateMessage(q, r.QueueMessagesList[0].MessageID, updatedMsg, - UpdateMessageParameters{PopReceipt: gr.QueueMessagesList[0].PopReceipt, VisibilityTimeout: 2}), chk.IsNil) - r, err = cli.PeekMessages(q, PeekMessagesParameters{}) - c.Assert(err, chk.IsNil) - c.Assert(len(r.QueueMessagesList), chk.Equals, 0) -} - -func (s *StorageQueueSuite) TestGetMessages(c *chk.C) { - q := randString(20) - cli := getQueueClient(c) - c.Assert(cli.CreateQueue(q), chk.IsNil) - defer cli.DeleteQueue(q) - - n := 4 - for i := 0; i < n; i++ { - c.Assert(cli.PutMessage(q, randString(10), PutMessageParameters{}), chk.IsNil) - } - - r, err := cli.GetMessages(q, GetMessagesParameters{NumOfMessages: n}) - c.Assert(err, chk.IsNil) - c.Assert(len(r.QueueMessagesList), chk.Equals, n) -} - -func (s *StorageQueueSuite) TestDeleteMessages(c *chk.C) { - q := randString(20) - cli := getQueueClient(c) - c.Assert(cli.CreateQueue(q), chk.IsNil) - defer cli.DeleteQueue(q) - - c.Assert(cli.PutMessage(q, "message", PutMessageParameters{}), chk.IsNil) - r, err := cli.GetMessages(q, GetMessagesParameters{VisibilityTimeout: 1}) - c.Assert(err, chk.IsNil) - c.Assert(len(r.QueueMessagesList), chk.Equals, 1) - m := r.QueueMessagesList[0] - c.Assert(cli.DeleteMessage(q, m.MessageID, m.PopReceipt), chk.IsNil) -} diff --git a/storage/share.go b/storage/share.go deleted file mode 100644 index 8423aedcc549..000000000000 --- a/storage/share.go +++ /dev/null @@ -1,186 +0,0 @@ -package storage - -import ( - "fmt" - "net/http" - "net/url" - "strconv" -) - -// Share represents an Azure file share. -type Share struct { - fsc *FileServiceClient - Name string `xml:"Name"` - Properties ShareProperties `xml:"Properties"` - Metadata map[string]string -} - -// ShareProperties contains various properties of a share. -type ShareProperties struct { - LastModified string `xml:"Last-Modified"` - Etag string `xml:"Etag"` - Quota int `xml:"Quota"` -} - -// builds the complete path for this share object. -func (s *Share) buildPath() string { - return fmt.Sprintf("/%s", s.Name) -} - -// Create this share under the associated account. -// If a share with the same name already exists, the operation fails. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx -func (s *Share) Create() error { - headers, err := s.fsc.createResource(s.buildPath(), resourceShare, mergeMDIntoExtraHeaders(s.Metadata, nil)) - if err != nil { - return err - } - - s.updateEtagAndLastModified(headers) - return nil -} - -// CreateIfNotExists creates this share under the associated account if -// it does not exist. Returns true if the share is newly created or false if -// the share already exists. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx -func (s *Share) CreateIfNotExists() (bool, error) { - resp, err := s.fsc.createResourceNoClose(s.buildPath(), resourceShare, nil) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict { - if resp.statusCode == http.StatusCreated { - s.updateEtagAndLastModified(resp.headers) - return true, nil - } - return false, s.FetchAttributes() - } - } - - return false, err -} - -// Delete marks this share for deletion. The share along with any files -// and directories contained within it are later deleted during garbage -// collection. If the share does not exist the operation fails -// -// See https://msdn.microsoft.com/en-us/library/azure/dn689090.aspx -func (s *Share) Delete() error { - return s.fsc.deleteResource(s.buildPath(), resourceShare) -} - -// DeleteIfExists operation marks this share for deletion if it exists. -// -// See https://msdn.microsoft.com/en-us/library/azure/dn689090.aspx -func (s *Share) DeleteIfExists() (bool, error) { - resp, err := s.fsc.deleteResourceNoClose(s.buildPath(), resourceShare) - if resp != nil { - defer resp.body.Close() - if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { - return resp.statusCode == http.StatusAccepted, nil - } - } - return false, err -} - -// Exists returns true if this share already exists -// on the storage account, otherwise returns false. -func (s *Share) Exists() (bool, error) { - exists, headers, err := s.fsc.resourceExists(s.buildPath(), resourceShare) - if exists { - s.updateEtagAndLastModified(headers) - s.updateQuota(headers) - } - return exists, err -} - -// FetchAttributes retrieves metadata and properties for this share. -func (s *Share) FetchAttributes() error { - headers, err := s.fsc.getResourceHeaders(s.buildPath(), compNone, resourceShare, http.MethodHead) - if err != nil { - return err - } - - s.updateEtagAndLastModified(headers) - s.updateQuota(headers) - s.Metadata = getMetadataFromHeaders(headers) - - return nil -} - -// GetRootDirectoryReference returns a Directory object at the root of this share. -func (s *Share) GetRootDirectoryReference() *Directory { - return &Directory{ - fsc: s.fsc, - share: s, - } -} - -// ServiceClient returns the FileServiceClient associated with this share. -func (s *Share) ServiceClient() *FileServiceClient { - return s.fsc -} - -// SetMetadata replaces the metadata for this share. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by GetShareMetadata. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx -func (s *Share) SetMetadata() error { - headers, err := s.fsc.setResourceHeaders(s.buildPath(), compMetadata, resourceShare, mergeMDIntoExtraHeaders(s.Metadata, nil)) - if err != nil { - return err - } - - s.updateEtagAndLastModified(headers) - return nil -} - -// SetProperties sets system properties for this share. -// -// Some keys may be converted to Camel-Case before sending. All keys -// are returned in lower case by SetShareProperties. HTTP header names -// are case-insensitive so case munging should not matter to other -// applications either. -// -// See https://msdn.microsoft.com/en-us/library/azure/mt427368.aspx -func (s *Share) SetProperties() error { - if s.Properties.Quota < 1 || s.Properties.Quota > 5120 { - return fmt.Errorf("invalid value %v for quota, valid values are [1, 5120]", s.Properties.Quota) - } - - headers, err := s.fsc.setResourceHeaders(s.buildPath(), compProperties, resourceShare, map[string]string{ - "x-ms-share-quota": strconv.Itoa(s.Properties.Quota), - }) - if err != nil { - return err - } - - s.updateEtagAndLastModified(headers) - return nil -} - -// updates Etag and last modified date -func (s *Share) updateEtagAndLastModified(headers http.Header) { - s.Properties.Etag = headers.Get("Etag") - s.Properties.LastModified = headers.Get("Last-Modified") -} - -// updates quota value -func (s *Share) updateQuota(headers http.Header) { - quota, err := strconv.Atoi(headers.Get("x-ms-share-quota")) - if err == nil { - s.Properties.Quota = quota - } -} - -// URL gets the canonical URL to this share. This method does not create a publicly accessible -// URL if the share is private and this method does not check if the share exists. -func (s *Share) URL() string { - return s.fsc.client.getEndpoint(fileServiceName, s.buildPath(), url.Values{}) -} diff --git a/storage/share_test.go b/storage/share_test.go deleted file mode 100644 index 863c32d0bc82..000000000000 --- a/storage/share_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package storage - -import ( - "math/rand" - - chk "gopkg.in/check.v1" -) - -type StorageShareSuite struct{} - -var _ = chk.Suite(&StorageShareSuite{}) - -const testSharePrefix = "zzzzztest" - -func randShare() string { - return testSharePrefix + randString(32-len(testSharePrefix)) -} - -func getFileClient(c *chk.C) FileServiceClient { - return getBasicClient(c).GetFileService() -} - -func (s *StorageShareSuite) TestCreateShareDeleteShare(c *chk.C) { - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - c.Assert(share.Create(), chk.IsNil) - c.Assert(share.Delete(), chk.IsNil) -} - -func (s *StorageShareSuite) TestCreateShareIfNotExists(c *chk.C) { - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - // First create - ok, err := share.CreateIfNotExists() - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) - - // Second create, should not give errors - ok, err = share.CreateIfNotExists() - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - // cleanup - share.Delete() -} - -func (s *StorageShareSuite) TestDeleteShareIfNotExists(c *chk.C) { - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - // delete non-existing share - ok, err := share.DeleteIfExists() - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - c.Assert(share.Create(), chk.IsNil) - - // delete existing share - ok, err = share.DeleteIfExists() - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) -} - -func (s *StorageShareSuite) TestListShares(c *chk.C) { - cli := getFileClient(c) - c.Assert(deleteTestShares(cli), chk.IsNil) - - name := randShare() - share := cli.GetShareReference(name) - - c.Assert(share.Create(), chk.IsNil) - - resp, err := cli.ListShares(ListSharesParameters{ - MaxResults: 5, - Prefix: testSharePrefix}) - c.Assert(err, chk.IsNil) - - c.Check(len(resp.Shares), chk.Equals, 1) - c.Check(resp.Shares[0].Name, chk.Equals, name) - - // clean up via the retrieved share object - resp.Shares[0].Delete() -} - -func (s *StorageShareSuite) TestShareExists(c *chk.C) { - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - - ok, err := share.Exists() - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, false) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - - ok, err = share.Exists() - c.Assert(err, chk.IsNil) - c.Assert(ok, chk.Equals, true) -} - -func (s *StorageShareSuite) TestGetAndSetShareProperties(c *chk.C) { - cli := getFileClient(c) - share := cli.GetShareReference(randShare()) - quota := rand.Intn(5120) - - c.Assert(share.Create(), chk.IsNil) - defer share.Delete() - c.Assert(share.Properties.LastModified, chk.Not(chk.Equals), "") - - share.Properties.Quota = quota - err := share.SetProperties() - c.Assert(err, chk.IsNil) - - err = share.FetchAttributes() - c.Assert(err, chk.IsNil) - - c.Assert(share.Properties.Quota, chk.Equals, quota) -} - -func (s *StorageShareSuite) TestGetAndSetShareMetadata(c *chk.C) { - cli := getFileClient(c) - share1 := cli.GetShareReference(randShare()) - - c.Assert(share1.Create(), chk.IsNil) - defer share1.Delete() - - // by default there should be no metadata - c.Assert(share1.Metadata, chk.IsNil) - c.Assert(share1.FetchAttributes(), chk.IsNil) - c.Assert(share1.Metadata, chk.IsNil) - - share2 := cli.GetShareReference(randShare()) - c.Assert(share2.Create(), chk.IsNil) - defer share2.Delete() - - c.Assert(share2.Metadata, chk.IsNil) - - mPut := map[string]string{ - "foo": "bar", - "bar_baz": "waz qux", - } - - share2.Metadata = mPut - c.Assert(share2.SetMetadata(), chk.IsNil) - c.Check(share2.Metadata, chk.DeepEquals, mPut) - - c.Assert(share2.FetchAttributes(), chk.IsNil) - c.Check(share2.Metadata, chk.DeepEquals, mPut) - - // Case munging - - mPutUpper := map[string]string{ - "Foo": "different bar", - "bar_BAZ": "different waz qux", - } - mExpectLower := map[string]string{ - "foo": "different bar", - "bar_baz": "different waz qux", - } - - share2.Metadata = mPutUpper - c.Assert(share2.SetMetadata(), chk.IsNil) - - c.Check(share2.Metadata, chk.DeepEquals, mPutUpper) - c.Assert(share2.FetchAttributes(), chk.IsNil) - c.Check(share2.Metadata, chk.DeepEquals, mExpectLower) -} - -func deleteTestShares(cli FileServiceClient) error { - for { - resp, err := cli.ListShares(ListSharesParameters{Prefix: testSharePrefix}) - if err != nil { - return err - } - if len(resp.Shares) == 0 { - break - } - for _, c := range resp.Shares { - share := cli.GetShareReference(c.Name) - err = share.Delete() - if err != nil { - return err - } - } - } - return nil -} diff --git a/storage/storagepolicy.go b/storage/storagepolicy.go deleted file mode 100644 index bee1c31ad61f..000000000000 --- a/storage/storagepolicy.go +++ /dev/null @@ -1,47 +0,0 @@ -package storage - -import ( - "strings" - "time" -) - -// AccessPolicyDetailsXML has specifics about an access policy -// annotated with XML details. -type AccessPolicyDetailsXML struct { - StartTime time.Time `xml:"Start"` - ExpiryTime time.Time `xml:"Expiry"` - Permission string `xml:"Permission"` -} - -// SignedIdentifier is a wrapper for a specific policy -type SignedIdentifier struct { - ID string `xml:"Id"` - AccessPolicy AccessPolicyDetailsXML `xml:"AccessPolicy"` -} - -// SignedIdentifiers part of the response from GetPermissions call. -type SignedIdentifiers struct { - SignedIdentifiers []SignedIdentifier `xml:"SignedIdentifier"` -} - -// AccessPolicy is the response type from the GetPermissions call. -type AccessPolicy struct { - SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"` -} - -// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the -// AccessPolicy struct which will get converted to XML. -func convertAccessPolicyToXMLStructs(id string, startTime time.Time, expiryTime time.Time, permissions string) SignedIdentifier { - return SignedIdentifier{ - ID: id, - AccessPolicy: AccessPolicyDetailsXML{ - StartTime: startTime.UTC().Round(time.Second), - ExpiryTime: expiryTime.UTC().Round(time.Second), - Permission: permissions, - }, - } -} - -func updatePermissions(permissions, permission string) bool { - return strings.Contains(permissions, permission) -} diff --git a/storage/table.go b/storage/table.go deleted file mode 100644 index 5cbc13ff122d..000000000000 --- a/storage/table.go +++ /dev/null @@ -1,258 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "time" -) - -// TableServiceClient contains operations for Microsoft Azure Table Storage -// Service. -type TableServiceClient struct { - client Client - auth authentication -} - -// AzureTable is the typedef of the Azure Table name -type AzureTable string - -const ( - tablesURIPath = "/Tables" -) - -type createTableRequest struct { - TableName string `json:"TableName"` -} - -// TableAccessPolicy are used for SETTING table policies -type TableAccessPolicy struct { - ID string - StartTime time.Time - ExpiryTime time.Time - CanRead bool - CanAppend bool - CanUpdate bool - CanDelete bool -} - -func pathForTable(table AzureTable) string { return fmt.Sprintf("%s", table) } - -func (c *TableServiceClient) getStandardHeaders() map[string]string { - return map[string]string{ - "x-ms-version": "2015-02-21", - "x-ms-date": currentTimeRfc1123Formatted(), - "Accept": "application/json;odata=nometadata", - "Accept-Charset": "UTF-8", - "Content-Type": "application/json", - userAgentHeader: c.client.userAgent, - } -} - -// QueryTables returns the tables created in the -// *TableServiceClient storage account. -func (c *TableServiceClient) QueryTables() ([]AzureTable, error) { - uri := c.client.getEndpoint(tableServiceName, tablesURIPath, url.Values{}) - - headers := c.getStandardHeaders() - headers["Content-Length"] = "0" - - resp, err := c.client.execInternalJSON(http.MethodGet, uri, headers, nil, c.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } - - buf := new(bytes.Buffer) - if _, err := buf.ReadFrom(resp.body); err != nil { - return nil, err - } - - var respArray queryTablesResponse - if err := json.Unmarshal(buf.Bytes(), &respArray); err != nil { - return nil, err - } - - s := make([]AzureTable, len(respArray.TableName)) - for i, elem := range respArray.TableName { - s[i] = AzureTable(elem.TableName) - } - - return s, nil -} - -// CreateTable creates the table given the specific -// name. This function fails if the name is not compliant -// with the specification or the tables already exists. -func (c *TableServiceClient) CreateTable(table AzureTable) error { - uri := c.client.getEndpoint(tableServiceName, tablesURIPath, url.Values{}) - - headers := c.getStandardHeaders() - - req := createTableRequest{TableName: string(table)} - buf := new(bytes.Buffer) - - if err := json.NewEncoder(buf).Encode(req); err != nil { - return err - } - - headers["Content-Length"] = fmt.Sprintf("%d", buf.Len()) - - resp, err := c.client.execInternalJSON(http.MethodPost, uri, headers, buf, c.auth) - - if err != nil { - return err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil { - return err - } - - return nil -} - -// DeleteTable deletes the table given the specific -// name. This function fails if the table is not present. -// Be advised: DeleteTable deletes all the entries -// that may be present. -func (c *TableServiceClient) DeleteTable(table AzureTable) error { - uri := c.client.getEndpoint(tableServiceName, tablesURIPath, url.Values{}) - uri += fmt.Sprintf("('%s')", string(table)) - - headers := c.getStandardHeaders() - - headers["Content-Length"] = "0" - - resp, err := c.client.execInternalJSON(http.MethodDelete, uri, headers, nil, c.auth) - - if err != nil { - return err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { - return err - - } - return nil -} - -// SetTablePermissions sets up table ACL permissions as per REST details https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Table-ACL -func (c *TableServiceClient) SetTablePermissions(table AzureTable, policies []TableAccessPolicy, timeout uint) (err error) { - params := url.Values{"comp": {"acl"}} - - if timeout > 0 { - params.Add("timeout", fmt.Sprint(timeout)) - } - - uri := c.client.getEndpoint(tableServiceName, string(table), params) - headers := c.client.getStandardHeaders() - - body, length, err := generateTableACLPayload(policies) - if err != nil { - return err - } - headers["Content-Length"] = fmt.Sprintf("%v", length) - - resp, err := c.client.execInternalJSON(http.MethodPut, uri, headers, body, c.auth) - if err != nil { - return err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { - return err - } - return nil -} - -func generateTableACLPayload(policies []TableAccessPolicy) (io.Reader, int, error) { - sil := SignedIdentifiers{ - SignedIdentifiers: []SignedIdentifier{}, - } - for _, tap := range policies { - permission := generateTablePermissions(&tap) - signedIdentifier := convertAccessPolicyToXMLStructs(tap.ID, tap.StartTime, tap.ExpiryTime, permission) - sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier) - } - return xmlMarshal(sil) -} - -// GetTablePermissions gets the table ACL permissions, as per REST details https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-table-acl -func (c *TableServiceClient) GetTablePermissions(table AzureTable, timeout int) (permissionResponse []TableAccessPolicy, err error) { - params := url.Values{"comp": {"acl"}} - - if timeout > 0 { - params.Add("timeout", strconv.Itoa(timeout)) - } - - uri := c.client.getEndpoint(tableServiceName, string(table), params) - headers := c.client.getStandardHeaders() - resp, err := c.client.execInternalJSON(http.MethodGet, uri, headers, nil, c.auth) - if err != nil { - return nil, err - } - defer resp.body.Close() - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } - - var ap AccessPolicy - err = xmlUnmarshal(resp.body, &ap.SignedIdentifiersList) - if err != nil { - return nil, err - } - out := updateTableAccessPolicy(ap) - return out, nil -} - -func updateTableAccessPolicy(ap AccessPolicy) []TableAccessPolicy { - out := []TableAccessPolicy{} - for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers { - tap := TableAccessPolicy{ - ID: policy.ID, - StartTime: policy.AccessPolicy.StartTime, - ExpiryTime: policy.AccessPolicy.ExpiryTime, - } - tap.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r") - tap.CanAppend = updatePermissions(policy.AccessPolicy.Permission, "a") - tap.CanUpdate = updatePermissions(policy.AccessPolicy.Permission, "u") - tap.CanDelete = updatePermissions(policy.AccessPolicy.Permission, "d") - - out = append(out, tap) - } - return out -} - -func generateTablePermissions(tap *TableAccessPolicy) (permissions string) { - // generate the permissions string (raud). - // still want the end user API to have bool flags. - permissions = "" - - if tap.CanRead { - permissions += "r" - } - - if tap.CanAppend { - permissions += "a" - } - - if tap.CanUpdate { - permissions += "u" - } - - if tap.CanDelete { - permissions += "d" - } - return permissions -} diff --git a/storage/table_entities.go b/storage/table_entities.go deleted file mode 100644 index 36413a0cffd1..000000000000 --- a/storage/table_entities.go +++ /dev/null @@ -1,354 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "reflect" -) - -// Annotating as secure for gas scanning -/* #nosec */ -const ( - partitionKeyNode = "PartitionKey" - rowKeyNode = "RowKey" - tag = "table" - tagIgnore = "-" - continuationTokenPartitionKeyHeader = "X-Ms-Continuation-Nextpartitionkey" - continuationTokenRowHeader = "X-Ms-Continuation-Nextrowkey" - maxTopParameter = 1000 -) - -type queryTablesResponse struct { - TableName []struct { - TableName string `json:"TableName"` - } `json:"value"` -} - -const ( - tableOperationTypeInsert = iota - tableOperationTypeUpdate = iota - tableOperationTypeMerge = iota - tableOperationTypeInsertOrReplace = iota - tableOperationTypeInsertOrMerge = iota -) - -type tableOperation int - -// TableEntity interface specifies -// the functions needed to support -// marshaling and unmarshaling into -// Azure Tables. The struct must only contain -// simple types because Azure Tables do not -// support hierarchy. -type TableEntity interface { - PartitionKey() string - RowKey() string - SetPartitionKey(string) error - SetRowKey(string) error -} - -// ContinuationToken is an opaque (ie not useful to inspect) -// struct that Get... methods can return if there are more -// entries to be returned than the ones already -// returned. Just pass it to the same function to continue -// receiving the remaining entries. -type ContinuationToken struct { - NextPartitionKey string - NextRowKey string -} - -type getTableEntriesResponse struct { - Elements []map[string]interface{} `json:"value"` -} - -// QueryTableEntities queries the specified table and returns the unmarshaled -// entities of type retType. -// top parameter limits the returned entries up to top. Maximum top -// allowed by Azure API is 1000. In case there are more than top entries to be -// returned the function will return a non nil *ContinuationToken. You can call the -// same function again passing the received ContinuationToken as previousContToken -// parameter in order to get the following entries. The query parameter -// is the odata query. To retrieve all the entries pass the empty string. -// The function returns a pointer to a TableEntity slice, the *ContinuationToken -// if there are more entries to be returned and an error in case something went -// wrong. -// -// Example: -// entities, cToken, err = tSvc.QueryTableEntities("table", cToken, reflect.TypeOf(entity), 20, "") -func (c *TableServiceClient) QueryTableEntities(tableName AzureTable, previousContToken *ContinuationToken, retType reflect.Type, top int, query string) ([]TableEntity, *ContinuationToken, error) { - if top > maxTopParameter { - return nil, nil, fmt.Errorf("top accepts at maximum %d elements. Requested %d instead", maxTopParameter, top) - } - - uri := c.client.getEndpoint(tableServiceName, pathForTable(tableName), url.Values{}) - uri += fmt.Sprintf("?$top=%d", top) - if query != "" { - uri += fmt.Sprintf("&$filter=%s", url.QueryEscape(query)) - } - - if previousContToken != nil { - uri += fmt.Sprintf("&NextPartitionKey=%s&NextRowKey=%s", previousContToken.NextPartitionKey, previousContToken.NextRowKey) - } - - headers := c.getStandardHeaders() - - headers["Content-Length"] = "0" - - resp, err := c.client.execInternalJSON(http.MethodGet, uri, headers, nil, c.auth) - - if err != nil { - return nil, nil, err - } - - contToken := extractContinuationTokenFromHeaders(resp.headers) - - defer resp.body.Close() - - if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, contToken, err - } - - retEntries, err := deserializeEntity(retType, resp.body) - if err != nil { - return nil, contToken, err - } - - return retEntries, contToken, nil -} - -// InsertEntity inserts an entity in the specified table. -// The function fails if there is an entity with the same -// PartitionKey and RowKey in the table. -func (c *TableServiceClient) InsertEntity(table AzureTable, entity TableEntity) error { - sc, err := c.execTable(table, entity, false, http.MethodPost) - if err != nil { - return err - } - - return checkRespCode(sc, []int{http.StatusCreated}) -} - -func (c *TableServiceClient) execTable(table AzureTable, entity TableEntity, specifyKeysInURL bool, method string) (int, error) { - uri := c.client.getEndpoint(tableServiceName, pathForTable(table), url.Values{}) - if specifyKeysInURL { - uri += fmt.Sprintf("(PartitionKey='%s',RowKey='%s')", url.QueryEscape(entity.PartitionKey()), url.QueryEscape(entity.RowKey())) - } - - headers := c.getStandardHeaders() - - var buf bytes.Buffer - - if err := injectPartitionAndRowKeys(entity, &buf); err != nil { - return 0, err - } - - headers["Content-Length"] = fmt.Sprintf("%d", buf.Len()) - - resp, err := c.client.execInternalJSON(method, uri, headers, &buf, c.auth) - - if err != nil { - return 0, err - } - - defer resp.body.Close() - - return resp.statusCode, nil -} - -// UpdateEntity updates the contents of an entity with the -// one passed as parameter. The function fails if there is no entity -// with the same PartitionKey and RowKey in the table. -func (c *TableServiceClient) UpdateEntity(table AzureTable, entity TableEntity) error { - sc, err := c.execTable(table, entity, true, http.MethodPut) - if err != nil { - return err - } - - return checkRespCode(sc, []int{http.StatusNoContent}) -} - -// MergeEntity merges the contents of an entity with the -// one passed as parameter. -// The function fails if there is no entity -// with the same PartitionKey and RowKey in the table. -func (c *TableServiceClient) MergeEntity(table AzureTable, entity TableEntity) error { - sc, err := c.execTable(table, entity, true, "MERGE") - if err != nil { - return err - } - - return checkRespCode(sc, []int{http.StatusNoContent}) -} - -// DeleteEntityWithoutCheck deletes the entity matching by -// PartitionKey and RowKey. There is no check on IfMatch -// parameter so the entity is always deleted. -// The function fails if there is no entity -// with the same PartitionKey and RowKey in the table. -func (c *TableServiceClient) DeleteEntityWithoutCheck(table AzureTable, entity TableEntity) error { - return c.DeleteEntity(table, entity, "*") -} - -// DeleteEntity deletes the entity matching by -// PartitionKey, RowKey and ifMatch field. -// The function fails if there is no entity -// with the same PartitionKey and RowKey in the table or -// the ifMatch is different. -func (c *TableServiceClient) DeleteEntity(table AzureTable, entity TableEntity, ifMatch string) error { - uri := c.client.getEndpoint(tableServiceName, pathForTable(table), url.Values{}) - uri += fmt.Sprintf("(PartitionKey='%s',RowKey='%s')", url.QueryEscape(entity.PartitionKey()), url.QueryEscape(entity.RowKey())) - - headers := c.getStandardHeaders() - - headers["Content-Length"] = "0" - headers["If-Match"] = ifMatch - - resp, err := c.client.execInternalJSON(http.MethodDelete, uri, headers, nil, c.auth) - - if err != nil { - return err - } - defer resp.body.Close() - - if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { - return err - } - - return nil -} - -// InsertOrReplaceEntity inserts an entity in the specified table -// or replaced the existing one. -func (c *TableServiceClient) InsertOrReplaceEntity(table AzureTable, entity TableEntity) error { - sc, err := c.execTable(table, entity, true, http.MethodPut) - if err != nil { - return err - } - - return checkRespCode(sc, []int{http.StatusNoContent}) -} - -// InsertOrMergeEntity inserts an entity in the specified table -// or merges the existing one. -func (c *TableServiceClient) InsertOrMergeEntity(table AzureTable, entity TableEntity) error { - sc, err := c.execTable(table, entity, true, "MERGE") - if err != nil { - return err - } - - return checkRespCode(sc, []int{http.StatusNoContent}) -} - -func injectPartitionAndRowKeys(entity TableEntity, buf *bytes.Buffer) error { - if err := json.NewEncoder(buf).Encode(entity); err != nil { - return err - } - - dec := make(map[string]interface{}) - if err := json.NewDecoder(buf).Decode(&dec); err != nil { - return err - } - - // Inject PartitionKey and RowKey - dec[partitionKeyNode] = entity.PartitionKey() - dec[rowKeyNode] = entity.RowKey() - - // Remove tagged fields - // The tag is defined in the const section - // This is useful to avoid storing the PartitionKey and RowKey twice. - numFields := reflect.ValueOf(entity).Elem().NumField() - for i := 0; i < numFields; i++ { - f := reflect.ValueOf(entity).Elem().Type().Field(i) - - if f.Tag.Get(tag) == tagIgnore { - // we must look for its JSON name in the dictionary - // as the user can rename it using a tag - jsonName := f.Name - if f.Tag.Get("json") != "" { - jsonName = f.Tag.Get("json") - } - delete(dec, jsonName) - } - } - - buf.Reset() - - if err := json.NewEncoder(buf).Encode(&dec); err != nil { - return err - } - - return nil -} - -func deserializeEntity(retType reflect.Type, reader io.Reader) ([]TableEntity, error) { - buf := new(bytes.Buffer) - - var ret getTableEntriesResponse - if err := json.NewDecoder(reader).Decode(&ret); err != nil { - return nil, err - } - - tEntries := make([]TableEntity, len(ret.Elements)) - - for i, entry := range ret.Elements { - - buf.Reset() - if err := json.NewEncoder(buf).Encode(entry); err != nil { - return nil, err - } - - dec := make(map[string]interface{}) - if err := json.NewDecoder(buf).Decode(&dec); err != nil { - return nil, err - } - - var pKey, rKey string - // strip pk and rk - for key, val := range dec { - switch key { - case partitionKeyNode: - pKey = val.(string) - case rowKeyNode: - rKey = val.(string) - } - } - - delete(dec, partitionKeyNode) - delete(dec, rowKeyNode) - - buf.Reset() - if err := json.NewEncoder(buf).Encode(dec); err != nil { - return nil, err - } - - // Create a empty retType instance - tEntries[i] = reflect.New(retType.Elem()).Interface().(TableEntity) - // Popolate it with the values - if err := json.NewDecoder(buf).Decode(&tEntries[i]); err != nil { - return nil, err - } - - // Reset PartitionKey and RowKey - if err := tEntries[i].SetPartitionKey(pKey); err != nil { - return nil, err - } - if err := tEntries[i].SetRowKey(rKey); err != nil { - return nil, err - } - } - - return tEntries, nil -} - -func extractContinuationTokenFromHeaders(h http.Header) *ContinuationToken { - ct := ContinuationToken{h.Get(continuationTokenPartitionKeyHeader), h.Get(continuationTokenRowHeader)} - - if ct.NextPartitionKey != "" && ct.NextRowKey != "" { - return &ct - } - return nil -} diff --git a/storage/table_test.go b/storage/table_test.go deleted file mode 100644 index 4f7d2e452909..000000000000 --- a/storage/table_test.go +++ /dev/null @@ -1,338 +0,0 @@ -package storage - -import ( - "crypto/rand" - "fmt" - "reflect" - "time" - - chk "gopkg.in/check.v1" -) - -type TableClient struct{} - -func getTableClient(c *chk.C) TableServiceClient { - return getBasicClient(c).GetTableService() -} - -type CustomEntity struct { - Name string `json:"name"` - Surname string `json:"surname"` - Number int - PKey string `json:"pk" table:"-"` - RKey string `json:"rk" table:"-"` -} - -type CustomEntityExtended struct { - *CustomEntity - ExtraField string -} - -func (c *CustomEntity) PartitionKey() string { - return c.PKey -} - -func (c *CustomEntity) RowKey() string { - return c.RKey -} - -func (c *CustomEntity) SetPartitionKey(s string) error { - c.PKey = s - return nil -} - -func (c *CustomEntity) SetRowKey(s string) error { - c.RKey = s - return nil -} - -func (s *StorageBlobSuite) Test_CreateAndDeleteTable(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - - err = cli.DeleteTable(tn) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) Test_InsertEntities(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - ce := &CustomEntity{Name: "Luke", Surname: "Skywalker", Number: 1543, PKey: "pkey"} - - for i := 0; i < 12; i++ { - ce.SetRowKey(fmt.Sprintf("%d", i)) - - err = cli.InsertEntity(tn, ce) - c.Assert(err, chk.IsNil) - } -} - -func (s *StorageBlobSuite) Test_InsertEntitiesRandomTableFails(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - ce := &CustomEntity{Name: "Luke", Surname: "Skywalker", Number: 1543, PKey: "pkey", RKey: "5"} - err := cli.InsertEntity(tn, ce) - c.Assert(err, chk.NotNil) -} - -func (s *StorageBlobSuite) Test_InsertOrReplaceEntities(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - ce := &CustomEntity{Name: "Darth", Surname: "Skywalker", Number: 60, PKey: "pkey", RKey: "5"} - - err = cli.InsertOrReplaceEntity(tn, ce) - c.Assert(err, chk.IsNil) - - cextra := &CustomEntityExtended{&CustomEntity{PKey: "pkey", RKey: "5"}, "extra"} - err = cli.InsertOrReplaceEntity(tn, cextra) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) Test_InsertOrMergeEntities(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - ce := &CustomEntity{Name: "Darth", Surname: "Skywalker", Number: 60, PKey: "pkey", RKey: "5"} - - err = cli.InsertOrMergeEntity(tn, ce) - c.Assert(err, chk.IsNil) - - cextra := &CustomEntityExtended{&CustomEntity{PKey: "pkey", RKey: "5"}, "extra"} - err = cli.InsertOrReplaceEntity(tn, cextra) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) Test_InsertAndGetEntities(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - ce := &CustomEntity{Name: "Darth", Surname: "Skywalker", Number: 60, PKey: "pkey", RKey: "100"} - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - - ce.SetRowKey("200") - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - - entries, _, err := cli.QueryTableEntities(tn, nil, reflect.TypeOf(ce), 10, "") - c.Assert(err, chk.IsNil) - - c.Assert(len(entries), chk.Equals, 2) - - c.Assert(ce.RowKey(), chk.Equals, entries[1].RowKey()) - - c.Assert(entries[1].(*CustomEntity), chk.DeepEquals, ce) -} - -func (s *StorageBlobSuite) Test_InsertAndQueryEntities(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - ce := &CustomEntity{Name: "Darth", Surname: "Skywalker", Number: 60, PKey: "pkey", RKey: "100"} - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - - ce.SetRowKey("200") - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - - entries, _, err := cli.QueryTableEntities(tn, nil, reflect.TypeOf(ce), 10, "RowKey eq '200'") - c.Assert(err, chk.IsNil) - - c.Assert(len(entries), chk.Equals, 1) - - c.Assert(ce.RowKey(), chk.Equals, entries[0].RowKey()) -} - -func (s *StorageBlobSuite) Test_InsertAndDeleteEntities(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - ce := &CustomEntity{Name: "Test", Surname: "Test2", Number: 0, PKey: "pkey", RKey: "r01"} - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - - ce.Number = 1 - ce.SetRowKey("r02") - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - - entries, _, err := cli.QueryTableEntities(tn, nil, reflect.TypeOf(ce), 10, "Number eq 1") - c.Assert(err, chk.IsNil) - - c.Assert(len(entries), chk.Equals, 1) - - c.Assert(entries[0].(*CustomEntity), chk.DeepEquals, ce) - - c.Assert(cli.DeleteEntityWithoutCheck(tn, entries[0]), chk.IsNil) - - entries, _, err = cli.QueryTableEntities(tn, nil, reflect.TypeOf(ce), 10, "") - c.Assert(err, chk.IsNil) - - // only 1 entry must be present - c.Assert(len(entries), chk.Equals, 1) -} - -func (s *StorageBlobSuite) Test_ContinuationToken(c *chk.C) { - cli := getTableClient(c) - - tn := AzureTable(randTable()) - - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - var ce *CustomEntity - var ceList [5]*CustomEntity - - for i := 0; i < 5; i++ { - ce = &CustomEntity{Name: "Test", Surname: "Test2", Number: i, PKey: "pkey", RKey: fmt.Sprintf("r%d", i)} - ceList[i] = ce - c.Assert(cli.InsertOrReplaceEntity(tn, ce), chk.IsNil) - } - - // retrieve using top = 2. Should return 2 entries, 2 entries and finally - // 1 entry - entries, contToken, err := cli.QueryTableEntities(tn, nil, reflect.TypeOf(ce), 2, "") - c.Assert(err, chk.IsNil) - c.Assert(len(entries), chk.Equals, 2) - c.Assert(entries[0].(*CustomEntity), chk.DeepEquals, ceList[0]) - c.Assert(entries[1].(*CustomEntity), chk.DeepEquals, ceList[1]) - c.Assert(contToken, chk.NotNil) - - entries, contToken, err = cli.QueryTableEntities(tn, contToken, reflect.TypeOf(ce), 2, "") - c.Assert(err, chk.IsNil) - c.Assert(len(entries), chk.Equals, 2) - c.Assert(entries[0].(*CustomEntity), chk.DeepEquals, ceList[2]) - c.Assert(entries[1].(*CustomEntity), chk.DeepEquals, ceList[3]) - c.Assert(contToken, chk.NotNil) - - entries, contToken, err = cli.QueryTableEntities(tn, contToken, reflect.TypeOf(ce), 2, "") - c.Assert(err, chk.IsNil) - c.Assert(len(entries), chk.Equals, 1) - c.Assert(entries[0].(*CustomEntity), chk.DeepEquals, ceList[4]) - c.Assert(contToken, chk.IsNil) -} - -func randTable() string { - const alphanum = "abcdefghijklmnopqrstuvwxyz" - var bytes = make([]byte, 32) - rand.Read(bytes) - for i, b := range bytes { - bytes[i] = alphanum[b%byte(len(alphanum))] - } - return string(bytes) -} - -func appendTablePermission(policies []TableAccessPolicy, ID string, - canRead bool, canAppend bool, canUpdate bool, canDelete bool, - startTime time.Time, expiryTime time.Time) []TableAccessPolicy { - - tap := TableAccessPolicy{ - ID: ID, - StartTime: startTime, - ExpiryTime: expiryTime, - CanRead: canRead, - CanAppend: canAppend, - CanUpdate: canUpdate, - CanDelete: canDelete, - } - policies = append(policies, tap) - return policies -} - -func (s *StorageBlobSuite) TestSetTablePermissionsSuccessfully(c *chk.C) { - cli := getTableClient(c) - tn := AzureTable(randTable()) - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - policies := []TableAccessPolicy{} - policies = appendTablePermission(policies, "GolangRocksOnAzure", true, true, true, true, now, now.Add(10*time.Hour)) - - err = cli.SetTablePermissions(tn, policies, 0) - c.Assert(err, chk.IsNil) -} - -func (s *StorageBlobSuite) TestSetTablePermissionsUnsuccessfully(c *chk.C) { - cli := getTableClient(c) - tn := AzureTable("nonexistingtable") - - policies := []TableAccessPolicy{} - policies = appendTablePermission(policies, "GolangRocksOnAzure", true, true, true, true, now, now.Add(10*time.Hour)) - - err := cli.SetTablePermissions(tn, policies, 0) - c.Assert(err, chk.NotNil) -} - -func (s *StorageBlobSuite) TestSetThenGetTablePermissionsSuccessfully(c *chk.C) { - cli := getTableClient(c) - tn := AzureTable(randTable()) - err := cli.CreateTable(tn) - c.Assert(err, chk.IsNil) - defer cli.DeleteTable(tn) - - policies := []TableAccessPolicy{} - policies = appendTablePermission(policies, "GolangRocksOnAzure", true, true, true, true, now, now.Add(10*time.Hour)) - policies = appendTablePermission(policies, "AutoRestIsSuperCool", true, true, false, true, now.Add(20*time.Hour), now.Add(30*time.Hour)) - err = cli.SetTablePermissions(tn, policies, 0) - c.Assert(err, chk.IsNil) - - returnedPolicies, err := cli.GetTablePermissions(tn, 0) - c.Assert(err, chk.IsNil) - - // now check policy set. - c.Assert(returnedPolicies, chk.HasLen, 2) - - for i := range policies { - c.Assert(returnedPolicies[i].ID, chk.Equals, policies[i].ID) - - // test timestamps down the second - // rounding start/expiry time original perms since the returned perms would have been rounded. - // so need rounded vs rounded. - c.Assert(returnedPolicies[i].StartTime.UTC().Round(time.Second).Format(time.RFC1123), - chk.Equals, policies[i].StartTime.UTC().Round(time.Second).Format(time.RFC1123)) - - c.Assert(returnedPolicies[i].ExpiryTime.UTC().Round(time.Second).Format(time.RFC1123), - chk.Equals, policies[i].ExpiryTime.UTC().Round(time.Second).Format(time.RFC1123)) - - c.Assert(returnedPolicies[i].CanRead, chk.Equals, policies[i].CanRead) - c.Assert(returnedPolicies[i].CanAppend, chk.Equals, policies[i].CanAppend) - c.Assert(returnedPolicies[i].CanUpdate, chk.Equals, policies[i].CanUpdate) - c.Assert(returnedPolicies[i].CanDelete, chk.Equals, policies[i].CanDelete) - - } -} diff --git a/storage/util.go b/storage/util.go deleted file mode 100644 index 57ca1b6d937e..000000000000 --- a/storage/util.go +++ /dev/null @@ -1,85 +0,0 @@ -package storage - -import ( - "bytes" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/xml" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "reflect" - "time" -) - -func (c Client) computeHmac256(message string) string { - h := hmac.New(sha256.New, c.accountKey) - h.Write([]byte(message)) - return base64.StdEncoding.EncodeToString(h.Sum(nil)) -} - -func currentTimeRfc1123Formatted() string { - return timeRfc1123Formatted(time.Now().UTC()) -} - -func timeRfc1123Formatted(t time.Time) string { - return t.Format(http.TimeFormat) -} - -func mergeParams(v1, v2 url.Values) url.Values { - out := url.Values{} - for k, v := range v1 { - out[k] = v - } - for k, v := range v2 { - vals, ok := out[k] - if ok { - vals = append(vals, v...) - out[k] = vals - } else { - out[k] = v - } - } - return out -} - -func prepareBlockListRequest(blocks []Block) string { - s := `` - for _, v := range blocks { - s += fmt.Sprintf("<%s>%s", v.Status, v.ID, v.Status) - } - s += `` - return s -} - -func xmlUnmarshal(body io.Reader, v interface{}) error { - data, err := ioutil.ReadAll(body) - if err != nil { - return err - } - return xml.Unmarshal(data, v) -} - -func xmlMarshal(v interface{}) (io.Reader, int, error) { - b, err := xml.Marshal(v) - if err != nil { - return nil, 0, err - } - return bytes.NewReader(b), len(b), nil -} - -func headersFromStruct(v interface{}) map[string]string { - headers := make(map[string]string) - value := reflect.ValueOf(v) - for i := 0; i < value.NumField(); i++ { - key := value.Type().Field(i).Tag.Get("header") - val := value.Field(i).String() - if key != "" && val != "" { - headers[key] = val - } - } - return headers -} diff --git a/storage/util_test.go b/storage/util_test.go deleted file mode 100644 index a1817d162428..000000000000 --- a/storage/util_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package storage - -import ( - "encoding/xml" - "io/ioutil" - "net/url" - "strings" - "time" - - chk "gopkg.in/check.v1" -) - -func (s *StorageClientSuite) Test_timeRfc1123Formatted(c *chk.C) { - now := time.Now().UTC() - expectedLayout := "Mon, 02 Jan 2006 15:04:05 GMT" - c.Assert(timeRfc1123Formatted(now), chk.Equals, now.Format(expectedLayout)) -} - -func (s *StorageClientSuite) Test_mergeParams(c *chk.C) { - v1 := url.Values{ - "k1": {"v1"}, - "k2": {"v2"}} - v2 := url.Values{ - "k1": {"v11"}, - "k3": {"v3"}} - out := mergeParams(v1, v2) - c.Assert(out.Get("k1"), chk.Equals, "v1") - c.Assert(out.Get("k2"), chk.Equals, "v2") - c.Assert(out.Get("k3"), chk.Equals, "v3") - c.Assert(out["k1"], chk.DeepEquals, []string{"v1", "v11"}) -} - -func (s *StorageClientSuite) Test_prepareBlockListRequest(c *chk.C) { - empty := []Block{} - expected := `` - c.Assert(prepareBlockListRequest(empty), chk.DeepEquals, expected) - - blocks := []Block{{"foo", BlockStatusLatest}, {"bar", BlockStatusUncommitted}} - expected = `foobar` - c.Assert(prepareBlockListRequest(blocks), chk.DeepEquals, expected) -} - -func (s *StorageClientSuite) Test_xmlUnmarshal(c *chk.C) { - xml := ` - - myblob - ` - var blob Blob - body := ioutil.NopCloser(strings.NewReader(xml)) - c.Assert(xmlUnmarshal(body, &blob), chk.IsNil) - c.Assert(blob.Name, chk.Equals, "myblob") -} - -func (s *StorageClientSuite) Test_xmlMarshal(c *chk.C) { - type t struct { - XMLName xml.Name `xml:"S"` - Name string `xml:"Name"` - } - - b := t{Name: "myblob"} - expected := `myblob` - r, i, err := xmlMarshal(b) - c.Assert(err, chk.IsNil) - o, err := ioutil.ReadAll(r) - c.Assert(err, chk.IsNil) - out := string(o) - c.Assert(out, chk.Equals, expected) - c.Assert(i, chk.Equals, len(expected)) -} - -func (s *StorageClientSuite) Test_headersFromStruct(c *chk.C) { - type t struct { - header1 string `header:"HEADER1"` - header2 string `header:"HEADER2"` - } - - h := t{header1: "value1", header2: "value2"} - expected := map[string]string{ - "HEADER1": "value1", - "HEADER2": "value2", - } - - out := headersFromStruct(h) - - c.Assert(out, chk.DeepEquals, expected) -} diff --git a/storage/version.go b/storage/version.go deleted file mode 100644 index 575eb1429c9e..000000000000 --- a/storage/version.go +++ /dev/null @@ -1,5 +0,0 @@ -package storage - -var ( - sdkVersion = "8.1.0-beta" -)