From 38e0b7b2469742dc27d1cee4225b56b006e33a9f Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Fri, 21 Dec 2018 16:47:45 +0100 Subject: [PATCH 01/20] change-me --- Gopkg.lock | 107 +++++++++++++++++++++++++++++++++---- Makefile | 2 +- go.mod | 34 ++++++++++++ go.sum | 61 +++++++++++++++++++++ pkg/engine/image_engine.go | 2 +- pkg/lock/memory.go | 4 +- pkg/object/file_object.go | 19 +++++-- pkg/object/preset.go | 2 +- pkg/processor/processor.go | 19 ++++--- pkg/storage/storage.go | 32 +++++------ 10 files changed, 236 insertions(+), 46 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/Gopkg.lock b/Gopkg.lock index 52fe1f8..a481e38 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,12 +3,15 @@ [[projects]] branch = "master" + digest = "1:2b6c78428111bb428246399c8483524351490a1ab5fccead942f41ce3b45bb16" name = "github.com/aldor007/go-aws-auth" packages = ["."] + pruneopts = "" revision = "00898dfb9272da8d7e51d30a603e3b8cc8033df4" [[projects]] branch = "develop" + digest = "1:74364c3433398a1aa76d11355d62a50946337271b088371a5cb04beffe2faa15" name = "github.com/aldor007/stow" packages = [ ".", @@ -16,11 +19,13 @@ "local", "local-meta", "noop", - "s3" + "s3", ] + pruneopts = "" revision = "67ba3fffe7b8e37ac47eed73b337ca1e913090b5" [[projects]] + digest = "1:9f6fa20324f7bffcc51cf98a1da62533f4437dcc4e0b14358f2aa795e72b4e80" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -47,147 +52,189 @@ "private/protocol/restxml", "private/protocol/xml/xmlutil", "service/s3", - "service/sts" + "service/sts", ] + pruneopts = "" revision = "a532359524fb25c60b6c938ec321b571adc857de" version = "v1.12.28" [[projects]] branch = "master" + digest = "1:0c5485088ce274fac2e931c1b979f2619345097b39d91af3239977114adf0320" name = "github.com/beorn7/perks" packages = ["quantile"] + pruneopts = "" revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" [[projects]] + digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "" revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" [[projects]] branch = "master" + digest = "1:36df4c13897f4a3fb444dbd14fc7f1a314456f2f875b286708b9d5282f949a80" name = "github.com/djherbis/stream" packages = ["."] + pruneopts = "" revision = "5cf27428a0140c094dfac111446cce6706c22b68" [[projects]] + digest = "1:8f22308229a05b68c76ad443a55bd97b679d1fe49a70e4909285cbc93a5a446d" name = "github.com/go-chi/chi" packages = [ ".", - "middleware" + "middleware", ] + pruneopts = "" revision = "f7c66f685bcab06bcce78ac212c5f3553c063d19" version = "v3.3.0" [[projects]] + digest = "1:f3c498cfe3758b828f1fd3272ad5d6ddf0ecf1a9dfbb7e4573c153b858338717" name = "github.com/go-ini/ini" packages = ["."] + pruneopts = "" revision = "7e7da451323b6766da368f8a1e8ec9a88a16b4a0" version = "v1.31.1" [[projects]] branch = "master" + digest = "1:3b760d3b93f994df8eb1d9ebfad17d3e9e37edcb7f7efaa15b427c0d7a64f4e4" name = "github.com/golang/protobuf" packages = ["proto"] + pruneopts = "" revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845" [[projects]] + digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e" name = "github.com/jmespath/go-jmespath" packages = ["."] + pruneopts = "" revision = "0b12d6b5" [[projects]] branch = "master" + digest = "1:2db9ca2a56645072341a390802f4cbbea15eaf1e2f914958da2089e70260190a" name = "github.com/karlseguin/ccache" packages = ["."] + pruneopts = "" revision = "b425c9ca005a2050ebe723f6a0cddcb907354ab7" [[projects]] + digest = "1:4c23ced97a470b17d9ffd788310502a077b9c1f60221a85563e49696276b4147" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] + pruneopts = "" revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" version = "v1.0.0" [[projects]] + digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" name = "github.com/pkg/errors" packages = ["."] + pruneopts = "" revision = "645ef00459ed84a119197bfb8d8205042c6df63d" version = "v0.8.0" [[projects]] + digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" name = "github.com/pmezard/go-difflib" packages = ["difflib"] + pruneopts = "" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] + digest = "1:4142d94383572e74b42352273652c62afec5b23f325222ed09198f46009022d1" name = "github.com/prometheus/client_golang" packages = [ "prometheus", - "prometheus/promhttp" + "prometheus/promhttp", ] + pruneopts = "" revision = "c5b7fccd204277076155f10851dad72b76a49317" version = "v0.8.0" [[projects]] branch = "master" + digest = "1:60aca47f4eeeb972f1b9da7e7db51dee15ff6c59f7b401c1588b8e6771ba15ef" name = "github.com/prometheus/client_model" packages = ["go"] + pruneopts = "" revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" [[projects]] branch = "master" + digest = "1:acf9415ef3a5f298495b0e1aa4d0e18f571a3c845872944e6c52777496819b21" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", - "model" + "model", ] + pruneopts = "" revision = "89604d197083d4781071d3c65855d24ecfb0a563" [[projects]] branch = "master" + digest = "1:f0857d075687b4ddebb10c8403d5fec9f093f7208b34ed5b6f3101ee2e77cec5" name = "github.com/prometheus/procfs" packages = [ ".", - "xfs" + "xfs", ] + pruneopts = "" revision = "b15cd069a83443be3154b719d0cc9fe8117f09fb" [[projects]] branch = "master" + digest = "1:633b5b39909d9d794f5543baee4971890e1866f5c082cdf4694954c1ff489a18" name = "github.com/spaolacci/murmur3" packages = ["."] + pruneopts = "" revision = "9f5d223c60793748f04a9d5b4b4eacddfc1f755d" [[projects]] + digest = "1:3926a4ec9a4ff1a072458451aa2d9b98acd059a45b38f7335d31e06c3d6a0159" name = "github.com/stretchr/testify" packages = ["assert"] + pruneopts = "" revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0" version = "v1.1.4" [[projects]] + digest = "1:529626e3e6f2fa1b4086f231458f3a59325229c43f1f0b9c1bd404e57ba2a426" name = "github.com/vmihailenco/msgpack" packages = [ ".", - "codes" + "codes", ] + pruneopts = "" revision = "df5e5d20905d833657b92cafe6f6341ca1378083" version = "v3.2.0" [[projects]] + digest = "1:6e8d05f4da3ff913adf807cb34623811f764b4924369f65c3ab9db744e8fe6ad" name = "go.uber.org/atomic" packages = ["."] + pruneopts = "" revision = "8474b86a5a6f79c443ce4b2992817ff32cf208b8" version = "v1.3.1" [[projects]] + digest = "1:22c7effcb4da0eacb2bb1940ee173fac010e9ef3c691f5de4b524d538bd980f5" name = "go.uber.org/multierr" packages = ["."] + pruneopts = "" revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a" version = "v1.1.0" [[projects]] + digest = "1:cfeddca8479edac1039a52dac1ff6e7a76252ebaecb86680383b0522423b7565" name = "go.uber.org/zap" packages = [ ".", @@ -195,18 +242,22 @@ "internal/bufferpool", "internal/color", "internal/exit", - "zapcore" + "zapcore", ] + pruneopts = "" revision = "35aad584952c3e7020db7b839f6b102de6271f89" version = "v1.7.1" [[projects]] branch = "master" + digest = "1:130b1bec86c62e121967ee0c69d9c263dc2d3ffe6c7c9a82aca4071c4d068861" name = "golang.org/x/net" packages = ["context"] + pruneopts = "" revision = "9dfe39835686865bff950a07b394c12a98ddc811" [[projects]] + digest = "1:934fb8966f303ede63aa405e2c8d7f0a427a05ea8df335dfdc1833dd4d40756f" name = "google.golang.org/appengine" packages = [ ".", @@ -217,41 +268,75 @@ "internal/datastore", "internal/log", "internal/modules", - "internal/remote_api" + "internal/remote_api", ] + pruneopts = "" revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" version = "v1.0.0" [[projects]] + digest = "1:c76862fa9fce00d62a1a24150eb609af6aac4d42ac5ef5aeab38b87d62ea2b49" name = "gopkg.in/h2non/bimg.v1" packages = ["."] + pruneopts = "" revision = "02e621739c77c791d8c153f240b7a1f75b07816f" version = "v1.0.18" [[projects]] + digest = "1:97139857c282164af86895a0b04661ca548a0acf9c8911b5ff04b62886c98b20" name = "gopkg.in/h2non/gock.v1" packages = ["."] + pruneopts = "" revision = "4692cba4394f79372bfe85996501ad43db9d82e0" version = "v1.0.9" [[projects]] branch = "master" + digest = "1:74098215108d9166810b6405326885090059267f64da1c83dffa4b0fa73e7f5b" name = "gopkg.in/kothar/brotli-go.v0" packages = [ "enc", - "shared" + "shared", ] + pruneopts = "" revision = "771231d473d6c4896e97c2b6e3855f1afc0cdada" [[projects]] branch = "v2" + digest = "1:f769ed60e075e4221612c2f4162fccc9d3795ef358fa463425e3b3d7a5debb27" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "" revision = "287cf08546ab5e7e37d55a84f7ed3fd1db036de5" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a3830c5da14fcf0e9ccd38e36e415cc13c009e0225318f7109c0897bbd29c82c" + input-imports = [ + "github.com/aldor007/go-aws-auth", + "github.com/aldor007/stow", + "github.com/aldor007/stow/http", + "github.com/aldor007/stow/local", + "github.com/aldor007/stow/local-meta", + "github.com/aldor007/stow/noop", + "github.com/aldor007/stow/s3", + "github.com/djherbis/stream", + "github.com/go-chi/chi", + "github.com/go-chi/chi/middleware", + "github.com/karlseguin/ccache", + "github.com/pkg/errors", + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/prometheus/client_model/go", + "github.com/spaolacci/murmur3", + "github.com/stretchr/testify/assert", + "go.uber.org/zap", + "go.uber.org/zap/zapcore", + "golang.org/x/net/context", + "gopkg.in/h2non/bimg.v1", + "gopkg.in/h2non/gock.v1", + "gopkg.in/kothar/brotli-go.v0/enc", + "gopkg.in/yaml.v2", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index 9472caf..6dcad97 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ install: dep ensure unit: - @(go list ./... | grep -v "vendor/" | xargs -n1 go test -race -v -cover) + @(go list ./... | grep -v "vendor/" | xargs -n1 go test -race -cover) unit-bench: ./scripts/unit-travis.sh diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..10af1c9 --- /dev/null +++ b/go.mod @@ -0,0 +1,34 @@ +module github.com/aldor007/mort + +require ( + github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 + github.com/aldor007/stow v0.0.0-20171229203915-67ba3fffe7b8 + github.com/aws/aws-sdk-go v1.12.28 + github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a + github.com/davecgh/go-spew v1.1.0 + github.com/djherbis/stream v0.0.0-20170408001054-5cf27428a014 + github.com/go-chi/chi v3.3.0+incompatible + github.com/go-ini/ini v1.31.1 + github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf + github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 + github.com/karlseguin/ccache v0.0.0-20170904064732-b425c9ca005a + github.com/matttproud/golang_protobuf_extensions v1.0.0 + github.com/pkg/errors v0.8.0 + github.com/pmezard/go-difflib v1.0.0 + github.com/prometheus/client_golang v0.8.0 + github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 + github.com/prometheus/common v0.0.0-20180110214958-89604d197083 + github.com/prometheus/procfs v0.0.0-20171226183907-b15cd069a834 + github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 + github.com/stretchr/testify v1.1.4 + github.com/vmihailenco/msgpack v3.2.0+incompatible + go.uber.org/atomic v1.3.1 + go.uber.org/multierr v1.1.0 + go.uber.org/zap v1.7.1 + golang.org/x/net v0.0.0-20171115151908-9dfe39835686 + google.golang.org/appengine v1.0.0 + gopkg.in/h2non/bimg.v1 v1.0.18 + gopkg.in/h2non/gock.v1 v1.0.9 + gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6 + gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4780753 --- /dev/null +++ b/go.sum @@ -0,0 +1,61 @@ +github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 h1:TuxcUGZxsHKCW50twhDbGdRtuMXmMDCmOssevV0ZoOA= +github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272/go.mod h1:gm7JFOl7Sp8r+lJqHXOkL0HyUFR05QXueqxVufqfkTg= +github.com/aldor007/mort v0.12.0 h1:2B6AE+1dN+6HORZ9t7u2ysJ4Kfz3lHIJoiICvf3hQyA= +github.com/aldor007/mort v0.12.0/go.mod h1:LFg/bthesXeJvAdfULjLOhu4KcuTClSigO2KP5HM+CM= +github.com/aldor007/stow v0.0.0-20171229203915-67ba3fffe7b8 h1:pD2XaHkkGTs3BjuWx+jEgtMcjO3NhMbp62Gb96WNkP0= +github.com/aldor007/stow v0.0.0-20171229203915-67ba3fffe7b8/go.mod h1:TUBrYvdpmfMQKeKUIZeveJE/Y5VufnvvtDE/kxJ9X3U= +github.com/aws/aws-sdk-go v1.12.28 h1:nXc2Sj3QwHle2Kf52Fr6HpyDNzNd5oYlPnqbB8hUBOg= +github.com/aws/aws-sdk-go v1.12.28/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/stream v0.0.0-20170408001054-5cf27428a014 h1:h5mCyHPtrWnvW1EabH6GRSs3RCWv6M3ZAEp3CwL9x14= +github.com/djherbis/stream v0.0.0-20170408001054-5cf27428a014/go.mod h1:ZNVKPVRCmrwhCwQHZUpVHHrq2rtGLrG1t3T/TThYLP8= +github.com/go-chi/chi v3.3.0+incompatible h1:19pl0NEHtjUmuCdXZpZ4RP3dJWdf05Fg8DDTFLnq++8= +github.com/go-chi/chi v3.3.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-ini/ini v1.31.1 h1:Ir+/CttgobzfH80vVjhKUdZVaS2oVD7/33M37NwE5HM= +github.com/go-ini/ini v1.31.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf h1:pFr/u+m8QUBMW/itAczltF3guNRAL7XDs5tD3f6nSD0= +github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/karlseguin/ccache v0.0.0-20170904064732-b425c9ca005a h1:HBV4xAoAO7cxLe8pRDakTUWx1WBNv6rUDuU1S9S0a7E= +github.com/karlseguin/ccache v0.0.0-20170904064732-b425c9ca005a/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= +github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc= +github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083 h1:BVsJT8+ZbyuL3hypz/HmEiM8h2P6hBQGig4el9/MdjA= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20171226183907-b15cd069a834 h1:jQVavQNtg1+b1/lxzhVA1JS3NYnCTo4b5Mo6z1N6apo= +github.com/prometheus/procfs v0.0.0-20171226183907-b15cd069a834/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 h1:lDiM+yMjW7Ork8mhl0YN0qO1Z02qGoe1vwzGc1gP/8U= +github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/testify v1.1.4 h1:ToftOQTytwshuOSj6bDSolVUa3GINfJP/fg3OkkOzQQ= +github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/vmihailenco/msgpack v3.2.0+incompatible h1:ZnAl9N6gKGsMZ7Js3co4p+qK1Nu2+8iuD8RyWTsAajs= +github.com/vmihailenco/msgpack v3.2.0+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +go.uber.org/atomic v1.3.1 h1:U8WaWEmp56LGz7PReduqHRVF6zzs9GbMC2NEZ42dxSQ= +go.uber.org/atomic v1.3.1/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.7.1 h1:wKPciimwkIgV4Aag/wpSDzvtO5JrfwdHKHO7blTHx7Q= +go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/net v0.0.0-20171115151908-9dfe39835686 h1:fxZ+mPcFhowcPZdlXrTF3GFhWVr/3wZyXQ8xW8WYGLU= +golang.org/x/net v0.0.0-20171115151908-9dfe39835686/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= +gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/gock.v1 v1.0.9 h1:qZnr7hBXCBgWcQ0MO9PGsdSK5Rpw2Oz6yXZPcKVL5Vs= +gopkg.in/h2non/gock.v1 v1.0.9/go.mod h1:KHI4Z1sxDW6P4N3DfTWSEza07YpkQP7KJBfglRMEjKY= +gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6 h1:M8GdJL0oESXVmjOOT3upJyFkKs5o1jJERiKYOZjVes0= +gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6/go.mod h1:nVee4zUY+UoXjOfM57w44w2XjsoIqIKd4A9vktFSQ6I= +gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= +gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/pkg/engine/image_engine.go b/pkg/engine/image_engine.go index b0a7e61..b29a8b5 100644 --- a/pkg/engine/image_engine.go +++ b/pkg/engine/image_engine.go @@ -83,7 +83,7 @@ func (c *ImageEngine) Process(obj *object.FileObject, trans []transforms.Transfo res.Set("x-amz-meta-public-height", strconv.Itoa(meta.Size.Height)) } else { - monitoring.Log().Warn("ImageEngine/process unable to get metadata", zap.String("obj.key", obj.Key), zap.Error(err)) + monitoring.Log().Warn("ImageEngine/process unable to get metadata", zap.Error(err), obj.LogData()...) } return res, nil diff --git a/pkg/lock/memory.go b/pkg/lock/memory.go index 5ef63c0..e9db849 100644 --- a/pkg/lock/memory.go +++ b/pkg/lock/memory.go @@ -1,10 +1,10 @@ package lock import ( + "github.com/aldor007/mort/pkg/monitoring" "github.com/aldor007/mort/pkg/response" - "sync" "go.uber.org/zap" - "github.com/aldor007/mort/pkg/monitoring" + "sync" ) // MemoryLock is in memory lock for single mort instance diff --git a/pkg/object/file_object.go b/pkg/object/file_object.go index d162c51..cb506c5 100644 --- a/pkg/object/file_object.go +++ b/pkg/object/file_object.go @@ -7,6 +7,7 @@ import ( //"github.com/aldor007/mort/pkg/uri" "context" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "net/url" ) @@ -39,8 +40,7 @@ func NewFileObjectFromPath(path string, mortConfig *config.Config) (*FileObject, err := Parse(obj.Uri, mortConfig, &obj) - monitoring.Log().Info("FileObject", zap.String("path", path), zap.String("key", obj.Key), zap.String("bucket", obj.Bucket), zap.String("storage", obj.Storage.Kind), - zap.Bool("hasTransforms", obj.HasTransform()), zap.Bool("hasParent", obj.HasParent())) + monitoring.Log().Info("FileObject", obj.LogData()...) return &obj, err } @@ -56,8 +56,7 @@ func NewFileObject(uri *url.URL, mortConfig *config.Config) (*FileObject, error) err := Parse(uri, mortConfig, &obj) - monitoring.Log().Info("FileObject", zap.String("path", uri.Path), zap.String("key", obj.Key), zap.String("bucket", obj.Bucket), zap.String("storage", obj.Storage.Kind), - zap.Bool("hasTransforms", obj.HasTransform()), zap.Bool("hasParent", obj.HasParent())) + monitoring.Log().Info("FileObject", obj.LogData()...) return &obj, err } @@ -76,3 +75,15 @@ func (o *FileObject) UpdateKey(str string) { o.key = o.key + str o.Key = o.Key + str } + +func (obj *FileObject) LogData() []zapcore.Field { + result := []zapcore.Field{zap.String("obj.path", obj.Uri.Path), zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Storage", obj.Storage.Kind), + zap.Bool("obj.HasTransforms", obj.HasTransform()), zap.Bool("obj.HasParent", obj.HasParent())} + + if obj.HasParent() { + result = append(result, zap.String("parent.Key", obj.Parent.Key), zap.String("parent.Path", obj.Parent.Uri.Path)) + } + + return result + +} diff --git a/pkg/object/preset.go b/pkg/object/preset.go index 59435fb..d6ced83 100644 --- a/pkg/object/preset.go +++ b/pkg/object/preset.go @@ -42,7 +42,7 @@ func decodePreset(_ *url.URL, bucketConfig config.Bucket, obj *FileObject) (stri parent := subMatchMap["parent"] if _, ok := trans.Presets[presetName]; !ok { - monitoring.Log().Warn("FileObject decodePreset unknown preset", zap.String("obj.Key", obj.Key), zap.String("parent", parent), zap.String("presetName", presetName), + monitoring.Log().Warn("FileObject decodePreset unknown preset", zap.String("obj.path", obj.Uri.Path), zap.String("obj.Key", obj.Key), zap.String("parent", parent), zap.String("presetName", presetName), zap.String("regexp", trans.Path)) return "", errors.New("unknown preset " + presetName) } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 03adefb..6713681 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -84,7 +84,7 @@ func (r *RequestProcessor) Process(req *http.Request, obj *object.FileObject) *r case <-ctx.Done(): msg.cancel <- struct{}{} close(msg.responseChan) - monitoring.Log().Warn("Process timeout", zap.String("obj.Key", obj.Key), zap.String("error", "Context.timeout")) + monitoring.Log().Warn("Process timeout", obj.LogData()..., zap.String("error", "Context.timeout")) return r.replyWithError(obj, 499, errContextCancel) case res := <-msg.responseChan: r.plugins.PostProcess(obj, req, res) @@ -125,7 +125,7 @@ func (r *RequestProcessor) replyWithError(obj *object.FileObject, sc int, err er lockData, locked := r.collapse.Lock(key) if locked { defer r.collapse.Release(key) - monitoring.Log().Info("Lock acquired for error response", zap.String("obj.Key", obj.Key)) + monitoring.Log().Info("Lock acquired for error response", obj.LogData()...) parent := response.NewBuf(200, r.serverConfig.Placeholder.Buf) transformsTab := []transforms.Transforms{obj.Transforms} @@ -176,14 +176,14 @@ func (r *RequestProcessor) collapseGET(req *http.Request, obj *object.FileObject ctx := obj.Ctx lockResult, locked := r.collapse.Lock(obj.Key) if locked { - monitoring.Log().Info("Lock acquired", zap.String("obj.Key", obj.Key)) + monitoring.Log().Info("Lock acquired", obj.LogData()...) res := r.handleGET(req, obj) r.collapse.NotifyAndRelease(obj.Key, res) return res } monitoring.Report().Inc("collapsed_count") - monitoring.Log().Info("Lock not acquired", zap.String("obj.Key", obj.Key)) + monitoring.Log().Info("Lock not acquired", obj.LogData()...) timer := time.NewTimer(r.lockTimeout) for { @@ -349,7 +349,7 @@ func (r *RequestProcessor) handleNotFound(obj, parentObj *object.FileObject, tra if parentRes.HasError() { return r.replyWithError(obj, parentRes.StatusCode, parentRes.Error()) } else if parentRes.StatusCode == 404 { - monitoring.Log().Warn("Missing parent for object", zap.String("obj.Key", obj.Key), zap.String("parent.Key", parentObj.Key)) + monitoring.Log().Warn("Missing parent for object", obj.LogData()...) return parentRes } @@ -363,8 +363,7 @@ func (r *RequestProcessor) handleNotFound(obj, parentObj *object.FileObject, tra return r.processImage(obj, parentRes, transformsTab) } else if obj.HasTransform() { parentRes.Close() - monitoring.Log().Warn("Not performing transforms", zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Key", obj.Key), - zap.String("parent.Key", parentObj.Key), zap.Int("parent.sc", parentRes.StatusCode), zap.String("parent.ContentType", parentRes.Headers.Get(response.HeaderContentType)), zap.Error(parentRes.Error())) + monitoring.Log().Warn("Not performing transforms", obj.LogData()..., zap.Int("parent.sc", parentRes.StatusCode), zap.String("parent.ContentType", parentRes.Headers.Get(response.HeaderContentType)), zap.Error(parentRes.Error())) } } @@ -409,7 +408,7 @@ func (r *RequestProcessor) processImage(obj *object.FileObject, parent *response ctx := obj.Ctx taked := r.throttler.Take(ctx) if !taked { - monitoring.Log().Warn("Processor/processImage", zap.String("obj.Key", obj.Key), zap.String("error", "throttled")) + monitoring.Log().Warn("Processor/processImage", obj.LogData()..., zap.String("error", "throttled")) monitoring.Report().Inc("throttled_count") return r.replyWithError(obj, 503, errThrottled) } @@ -419,7 +418,7 @@ func (r *RequestProcessor) processImage(obj *object.FileObject, parent *response mergedTrans := transforms.Merge(transformsTab) mergedLen := len(mergedTrans) - monitoring.Log().Info("Performing transforms", zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Key", obj.Key), zap.Int("transformsLen", transformsLen), zap.Int("mergedLen", mergedLen)) + monitoring.Log().Info("Performing transforms", obj.LogData()..., zap.Int("transformsLen", transformsLen), zap.Int("mergedLen", mergedLen)) eng := engine.NewImageEngine(parent) res, err := eng.Process(obj, mergedTrans) if err != nil { @@ -436,7 +435,7 @@ func (r *RequestProcessor) processImage(obj *object.FileObject, parent *response resS.Close() }(*obj, resCpy) } else { - monitoring.Log().Warn("Processor/processImage", zap.String("obj.Key", obj.Key), zap.Error(err)) + monitoring.Log().Warn("Processor/processImage", obj.LogData()..., zap.Error(err)) } return res diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 01a1444..b46df32 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -40,7 +40,7 @@ func Get(obj *object.FileObject) *response.Response { key := getKey(obj) client, err := getClient(obj) if err != nil { - monitoring.Log().Info("Storage/Get get client", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Error(err)) + monitoring.Log().Info("Storage/Get get client", obj.LogData()..., zap.Error(err)) return response.NewError(503, err) } @@ -51,14 +51,14 @@ func Get(obj *object.FileObject) *response.Response { return response.NewString(404, notFound) } - monitoring.Log().Info("Storage/Get item response", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Error(err)) + monitoring.Log().Info("Storage/Get item response", obj.LogData()..., zap.Error(err)) return response.NewError(500, err) } if isDir(item) == false { reader, err := item.Open() if err != nil { - monitoring.Logs().Warnw("Storage/Get open item", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Logs().Warnw("Storage/Get open item", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } return prepareResponse(obj, reader, item) @@ -77,18 +77,18 @@ func Head(obj *object.FileObject) *response.Response { key := getKey(obj) client, err := getClient(obj) if err != nil { - monitoring.Logs().Infow("Storage/Head get client", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Error(err)) + monitoring.Logs().Infow("Storage/Head get client", obj.LogData()..., zap.Error(err)) return response.NewError(503, err) } item, err := client.Item(key) if err != nil { if err == stow.ErrNotFound { - monitoring.Logs().Infow("Storage/Head item response", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 404)) + monitoring.Logs().Infow("Storage/Head item response", obj.LogData()..., zap.Int("sc", 404)) return response.NewString(404, notFound) } - monitoring.Logs().Infow("Storage/Head item response", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Error(err)) + monitoring.Logs().Infow("Storage/Head item response", obj.LogData()..., zap.Error(err)) return response.NewError(500, err) } @@ -102,14 +102,14 @@ func Set(obj *object.FileObject, metaHeaders http.Header, contentLen int64, body defer t.Done() client, err := getClient(obj) if err != nil { - monitoring.Logs().Warnw("Storage/Set create client", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 503), zap.Error(err)) + monitoring.Logs().Warnw("Storage/Set create client", obj.LogData()..., zap.Int("sc", 503), zap.Error(err)) return response.NewError(503, err) } _, err = client.Put(getKey(obj), body, contentLen, prepareMetadata(obj, metaHeaders)) if err != nil { - monitoring.Logs().Warnw("Storage/Set cannot set", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Logs().Warnw("Storage/Set cannot set", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } @@ -125,7 +125,7 @@ func Delete(obj *object.FileObject) *response.Response { defer t.Done() client, err := getClient(obj) if err != nil { - monitoring.Logs().Warnw("Storage/Delete create client", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 503), zap.Error(err)) + monitoring.Logs().Warnw("Storage/Delete create client", obj.LogData()..., zap.Int("sc", 503), zap.Error(err)) return response.NewError(503, err) } @@ -134,7 +134,7 @@ func Delete(obj *object.FileObject) *response.Response { err = client.RemoveItem(getKey(obj)) if err != nil { - monitoring.Logs().Warnw("Storage/Delete cannot delete", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Logs().Warnw("Storage/Delete cannot delete", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } } else if resHead.StatusCode == 404 { @@ -150,7 +150,7 @@ func Delete(obj *object.FileObject) *response.Response { func List(obj *object.FileObject, maxKeys int, _ string, prefix string, marker string) *response.Response { client, err := getClient(obj) if err != nil { - monitoring.Logs().Warnw("Storage/List", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 503), zap.Error(err)) + monitoring.Logs().Warnw("Storage/List", obj.LogData()..., zap.Int("sc", 503), zap.Error(err)) return response.NewError(503, err) } @@ -158,7 +158,7 @@ func List(obj *object.FileObject, maxKeys int, _ string, prefix string, marker s _, err = client.Item(prefix) if err != nil { if err == stow.ErrNotFound { - monitoring.Logs().Infow("Storage/List item not fountresponse", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 404)) + monitoring.Logs().Infow("Storage/List item not fountresponse", obj.LogData()..., zap.Int("sc", 404)) return response.NewString(404, obj.Key) } } @@ -166,7 +166,7 @@ func List(obj *object.FileObject, maxKeys int, _ string, prefix string, marker s items, resultMarker, err := client.Items(prefix, marker, maxKeys) if err != nil { - monitoring.Logs().Warnw("Storage/List", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Logs().Warnw("Storage/List", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } @@ -335,7 +335,7 @@ func prepareResponse(obj *object.FileObject, stream io.ReadCloser, item stow.Ite metadata, err := item.Metadata() if err != nil { - monitoring.Log().Warn("Storage/prepareResponse read metadata error", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/prepareResponse read metadata error", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } @@ -343,13 +343,13 @@ func prepareResponse(obj *object.FileObject, stream io.ReadCloser, item stow.Ite etag, err := item.ETag() if err != nil { - monitoring.Log().Warn("Storage/prepareResponse read etag error", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/prepareResponse read etag error", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } lastMod, err := item.LastMod() if err != nil { - monitoring.Log().Warn("Storage/prepareResponse read lastmod error", zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/prepareResponse read lastmod error", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) return response.NewError(500, err) } From 7468a4b1cbbbe6dabb0d6c9aa16b92859254cc4b Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 9 Jan 2019 19:53:53 +0100 Subject: [PATCH 02/20] feat(dpes): update deps feat(storage): new storage b2 bugfix(response): fix last mod in response feat(logs): add more logs from objecct --- Gopkg.lock | 254 ++++++++++++++++++------------ Gopkg.toml | 153 +++++------------- go.mod | 58 ++++--- go.sum | 61 ------- package-lock.json | 212 ++++++++++++------------- pkg/config/config.go | 2 +- pkg/config/types.go | 4 +- pkg/engine/image_engine.go | 6 +- pkg/monitoring/prometheus_test.go | 2 +- pkg/object/file_object.go | 4 +- pkg/processor/processor.go | 14 +- pkg/response/response.go | 6 +- pkg/storage/storage.go | 70 ++++---- tests-int/Images.Spec.js | 18 +-- 14 files changed, 397 insertions(+), 467 deletions(-) delete mode 100644 go.sum diff --git a/Gopkg.lock b/Gopkg.lock index a481e38..c639c9e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,29 +3,30 @@ [[projects]] branch = "master" - digest = "1:2b6c78428111bb428246399c8483524351490a1ab5fccead942f41ce3b45bb16" + digest = "1:f9dbc21e82fee60d7f113e2a083afcf76f7d914066664f6022bbc707f669d320" name = "github.com/aldor007/go-aws-auth" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "00898dfb9272da8d7e51d30a603e3b8cc8033df4" [[projects]] branch = "develop" - digest = "1:74364c3433398a1aa76d11355d62a50946337271b088371a5cb04beffe2faa15" + digest = "1:3da15e2ab879a6d15a175eda02ed868e19ea36fe1c7f1649a2916ab9e94a86c1" name = "github.com/aldor007/stow" packages = [ ".", + "b2", "http", "local", "local-meta", "noop", "s3", ] - pruneopts = "" - revision = "67ba3fffe7b8e37ac47eed73b337ca1e913090b5" + pruneopts = "UT" + revision = "dee0ba7576487f919def95881b3e5bb6fe03a106" [[projects]] - digest = "1:9f6fa20324f7bffcc51cf98a1da62533f4437dcc4e0b14358f2aa795e72b4e80" + digest = "1:e2a8eda4c51943dfe50caa35e6f4318cc66c659ffcbe633e0aaa0e2ec03aa5bb" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -37,204 +38,238 @@ "aws/credentials", "aws/credentials/ec2rolecreds", "aws/credentials/endpointcreds", + "aws/credentials/processcreds", "aws/credentials/stscreds", + "aws/csm", "aws/defaults", "aws/ec2metadata", "aws/endpoints", "aws/request", "aws/session", "aws/signer/v4", + "internal/ini", + "internal/s3err", + "internal/sdkio", + "internal/sdkrand", + "internal/sdkuri", "internal/shareddefaults", "private/protocol", + "private/protocol/eventstream", + "private/protocol/eventstream/eventstreamapi", "private/protocol/query", "private/protocol/query/queryutil", "private/protocol/rest", "private/protocol/restxml", "private/protocol/xml/xmlutil", "service/s3", + "service/s3/s3iface", + "service/s3/s3manager", "service/sts", ] - pruneopts = "" - revision = "a532359524fb25c60b6c938ec321b571adc857de" - version = "v1.12.28" + pruneopts = "UT" + revision = "62936e15518acb527a1a9cb4a39d96d94d0fd9a2" + version = "v1.16.15" [[projects]] branch = "master" - digest = "1:0c5485088ce274fac2e931c1b979f2619345097b39d91af3239977114adf0320" + digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" name = "github.com/beorn7/perks" packages = ["quantile"] - pruneopts = "" - revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" + pruneopts = "UT" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] - digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" packages = ["spew"] - pruneopts = "" - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" + pruneopts = "UT" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" [[projects]] - branch = "master" - digest = "1:36df4c13897f4a3fb444dbd14fc7f1a314456f2f875b286708b9d5282f949a80" + digest = "1:97304aaf0b57ebf727c422fea540c69d75d1534ab0e424eebf4bf10e87e58e65" name = "github.com/djherbis/stream" packages = ["."] - pruneopts = "" - revision = "5cf27428a0140c094dfac111446cce6706c22b68" + pruneopts = "UT" + revision = "26b9a1fb834dbe1e5782c5c87f119338d0de7af3" + version = "v1.2.0" [[projects]] - digest = "1:8f22308229a05b68c76ad443a55bd97b679d1fe49a70e4909285cbc93a5a446d" + digest = "1:cb53653e4e410ef91f3a83568464ef966e1732ad9a00b2719571f90eae9a15a7" name = "github.com/go-chi/chi" packages = [ ".", "middleware", ] - pruneopts = "" - revision = "f7c66f685bcab06bcce78ac212c5f3553c063d19" - version = "v3.3.0" + pruneopts = "UT" + revision = "08d9051ef6546d57c5dca8eae13e6df362e2d568" + version = "v3.3.4" [[projects]] - digest = "1:f3c498cfe3758b828f1fd3272ad5d6ddf0ecf1a9dfbb7e4573c153b858338717" - name = "github.com/go-ini/ini" + branch = "master" + digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" + name = "github.com/golang/glog" packages = ["."] - pruneopts = "" - revision = "7e7da451323b6766da368f8a1e8ec9a88a16b4a0" - version = "v1.31.1" + pruneopts = "UT" + revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] - branch = "master" - digest = "1:3b760d3b93f994df8eb1d9ebfad17d3e9e37edcb7f7efaa15b427c0d7a64f4e4" + digest = "1:97df918963298c287643883209a2c3f642e6593379f97ab400c2a2e219ab647d" name = "github.com/golang/protobuf" packages = ["proto"] - pruneopts = "" - revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845" + pruneopts = "UT" + revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" + version = "v1.2.0" [[projects]] - digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e" + branch = "master" + digest = "1:1073b6851142e8f2b75558fcd995d3ff30fd39191de081c4180cebc520a74706" + name = "github.com/google/readahead" + packages = ["."] + pruneopts = "UT" + revision = "eaceba16903255cb149d1efc316f6cc83d765268" + +[[projects]] + digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" name = "github.com/jmespath/go-jmespath" packages = ["."] - pruneopts = "" - revision = "0b12d6b5" + pruneopts = "UT" + revision = "c2b33e84" [[projects]] - branch = "master" - digest = "1:2db9ca2a56645072341a390802f4cbbea15eaf1e2f914958da2089e70260190a" + digest = "1:0a4e87d658551ece5cc450a2f4a7c60f1f7656f128189e302c743ebf471797a6" name = "github.com/karlseguin/ccache" packages = ["."] - pruneopts = "" - revision = "b425c9ca005a2050ebe723f6a0cddcb907354ab7" + pruneopts = "UT" + revision = "a2d62155777b39595c825ed3824279e642a5db3c" + version = "v2.0.2" [[projects]] - digest = "1:4c23ced97a470b17d9ffd788310502a077b9c1f60221a85563e49696276b4147" + digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] - pruneopts = "" - revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" - version = "v1.0.0" + pruneopts = "UT" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" [[projects]] - digest = "1:7365acd48986e205ccb8652cc746f09c8b7876030d53710ea6ef7d0bd0dcd7ca" + digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b" name = "github.com/pkg/errors" packages = ["."] - pruneopts = "" - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" + pruneopts = "UT" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" [[projects]] - digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] - pruneopts = "" + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] - digest = "1:4142d94383572e74b42352273652c62afec5b23f325222ed09198f46009022d1" + branch = "master" + digest = "1:63987b971c0f3240a0f1eed70b80b360e6a57d7a8c85ed7daa8bf770319ab308" + name = "github.com/pquerna/ffjson" + packages = [ + "ffjson", + "fflib/v1", + "fflib/v1/internal", + ] + pruneopts = "UT" + revision = "e517b90714f7c0eabe6d2e570a5886ae077d6db6" + +[[projects]] + digest = "1:93a746f1060a8acbcf69344862b2ceced80f854170e1caae089b2834c5fbf7f4" name = "github.com/prometheus/client_golang" packages = [ "prometheus", + "prometheus/internal", "prometheus/promhttp", ] - pruneopts = "" - revision = "c5b7fccd204277076155f10851dad72b76a49317" - version = "v0.8.0" + pruneopts = "UT" + revision = "505eaef017263e299324067d40ca2c48f6a2cf50" + version = "v0.9.2" [[projects]] branch = "master" - digest = "1:60aca47f4eeeb972f1b9da7e7db51dee15ff6c59f7b401c1588b8e6771ba15ef" + digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" name = "github.com/prometheus/client_model" packages = ["go"] - pruneopts = "" - revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" + pruneopts = "UT" + revision = "f287a105a20ec685d797f65cd0ce8fbeaef42da1" [[projects]] branch = "master" - digest = "1:acf9415ef3a5f298495b0e1aa4d0e18f571a3c845872944e6c52777496819b21" + digest = "1:ce62b400185bf6b16ef6088011b719e449f5c15c4adb6821589679f752c2788e" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", "model", ] - pruneopts = "" - revision = "89604d197083d4781071d3c65855d24ecfb0a563" + pruneopts = "UT" + revision = "2998b132700a7d019ff618c06a234b47c1f3f681" [[projects]] branch = "master" - digest = "1:f0857d075687b4ddebb10c8403d5fec9f093f7208b34ed5b6f3101ee2e77cec5" + digest = "1:08eb8b60450efe841e37512d66ce366a87d187505d7c67b99307a6c1803483a2" name = "github.com/prometheus/procfs" packages = [ ".", + "internal/util", + "nfs", "xfs", ] - pruneopts = "" - revision = "b15cd069a83443be3154b719d0cc9fe8117f09fb" + pruneopts = "UT" + revision = "b1a0a9a36d7453ba0f62578b99712f3a6c5f82d1" [[projects]] - branch = "master" - digest = "1:633b5b39909d9d794f5543baee4971890e1866f5c082cdf4694954c1ff489a18" + digest = "1:919bb3aa6d9d0b67648c219fa4925312bc3c2872da19e818fa769e9c97a2b643" name = "github.com/spaolacci/murmur3" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "9f5d223c60793748f04a9d5b4b4eacddfc1f755d" + version = "v1.1" [[projects]] - digest = "1:3926a4ec9a4ff1a072458451aa2d9b98acd059a45b38f7335d31e06c3d6a0159" + digest = "1:972c2427413d41a1e06ca4897e8528e5a1622894050e2f527b38ddf0f343f759" name = "github.com/stretchr/testify" packages = ["assert"] - pruneopts = "" - revision = "69483b4bd14f5845b5a1e55bca19e954e827f1d0" - version = "v1.1.4" + pruneopts = "UT" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" [[projects]] - digest = "1:529626e3e6f2fa1b4086f231458f3a59325229c43f1f0b9c1bd404e57ba2a426" + digest = "1:b77a6808d2479a400579eb8d3bf18bd5c884fa8f038f3a1ce77a47c15bb17e2c" name = "github.com/vmihailenco/msgpack" packages = [ ".", "codes", ] - pruneopts = "" - revision = "df5e5d20905d833657b92cafe6f6341ca1378083" - version = "v3.2.0" + pruneopts = "UT" + revision = "c4fc0fcdd500af473157759b6d73f935e727ec77" + version = "v4.0.1" [[projects]] - digest = "1:6e8d05f4da3ff913adf807cb34623811f764b4924369f65c3ab9db744e8fe6ad" + digest = "1:777e729b475d3895c7229552aa10076f0d177daf37c0a72258006d046d329960" name = "go.uber.org/atomic" packages = ["."] - pruneopts = "" - revision = "8474b86a5a6f79c443ce4b2992817ff32cf208b8" - version = "v1.3.1" + pruneopts = "UT" + revision = "4e336646b2ef9fc6e47be8e21594178f98e5ebcf" + version = "v1.2.0" [[projects]] - digest = "1:22c7effcb4da0eacb2bb1940ee173fac010e9ef3c691f5de4b524d538bd980f5" + digest = "1:60bf2a5e347af463c42ed31a493d817f8a72f102543060ed992754e689805d1a" name = "go.uber.org/multierr" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a" version = "v1.1.0" [[projects]] - digest = "1:cfeddca8479edac1039a52dac1ff6e7a76252ebaecb86680383b0522423b7565" + digest = "1:c52caf7bd44f92e54627a31b85baf06a68333a196b3d8d241480a774733dcf8b" name = "go.uber.org/zap" packages = [ ".", @@ -244,20 +279,20 @@ "internal/exit", "zapcore", ] - pruneopts = "" - revision = "35aad584952c3e7020db7b839f6b102de6271f89" - version = "v1.7.1" + pruneopts = "UT" + revision = "ff33455a0e382e8a81d14dd7c922020b6b5e7982" + version = "v1.9.1" [[projects]] branch = "master" - digest = "1:130b1bec86c62e121967ee0c69d9c263dc2d3ffe6c7c9a82aca4071c4d068861" + digest = "1:76ee51c3f468493aff39dbacc401e8831fbb765104cbf613b89bef01cf4bad70" name = "golang.org/x/net" packages = ["context"] - pruneopts = "" - revision = "9dfe39835686865bff950a07b394c12a98ddc811" + pruneopts = "UT" + revision = "1e06a53dbb7e2ed46e91183f219db23c6943c532" [[projects]] - digest = "1:934fb8966f303ede63aa405e2c8d7f0a427a05ea8df335dfdc1833dd4d40756f" + digest = "1:56dfffc6e6ceeaebb6db73e02a846a92a850c53130078682993bd028a2ef717d" name = "google.golang.org/appengine" packages = [ ".", @@ -270,44 +305,52 @@ "internal/modules", "internal/remote_api", ] - pruneopts = "" - revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" - version = "v1.0.0" + pruneopts = "UT" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" [[projects]] - digest = "1:c76862fa9fce00d62a1a24150eb609af6aac4d42ac5ef5aeab38b87d62ea2b49" + digest = "1:bc6ccb93f9d234e65a2a859c12bcbe3bf93d2c6c66841a887eae169406999cda" name = "gopkg.in/h2non/bimg.v1" packages = ["."] - pruneopts = "" - revision = "02e621739c77c791d8c153f240b7a1f75b07816f" - version = "v1.0.18" + pruneopts = "UT" + revision = "4b9abdb945893c295f0e58b6803102f23e061ad0" + version = "v1.0.19" [[projects]] - digest = "1:97139857c282164af86895a0b04661ca548a0acf9c8911b5ff04b62886c98b20" + digest = "1:f1e9221c1e17a438f8758d8da7be7ced1fc6132c4555d589d80f697d2cba7c8a" name = "gopkg.in/h2non/gock.v1" packages = ["."] - pruneopts = "" - revision = "4692cba4394f79372bfe85996501ad43db9d82e0" - version = "v1.0.9" + pruneopts = "UT" + revision = "a6029d17e101ac1594adafea9e63fd8c988a8701" + version = "v1.0.12" [[projects]] branch = "master" - digest = "1:74098215108d9166810b6405326885090059267f64da1c83dffa4b0fa73e7f5b" + digest = "1:6de28cf34c450cfcfc25d28230311872599ccfcb9c2aba3a8d177a9225582130" name = "gopkg.in/kothar/brotli-go.v0" packages = [ "enc", "shared", ] - pruneopts = "" + pruneopts = "UT" revision = "771231d473d6c4896e97c2b6e3855f1afc0cdada" [[projects]] - branch = "v2" - digest = "1:f769ed60e075e4221612c2f4162fccc9d3795ef358fa463425e3b3d7a5debb27" + branch = "master" + digest = "1:863b86a1a9cb2aca2ae27e576c345f7eec79ef99bdd74a4d8fdd966af299f2f2" + name = "gopkg.in/kothar/go-backblaze.v0" + packages = ["."] + pruneopts = "UT" + revision = "9ac0cf0dab1a77a5c25e0e38e6b8e493385f1add" + +[[projects]] + digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" packages = ["."] - pruneopts = "" - revision = "287cf08546ab5e7e37d55a84f7ed3fd1db036de5" + pruneopts = "UT" + revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" + version = "v2.2.2" [solve-meta] analyzer-name = "dep" @@ -315,6 +358,7 @@ input-imports = [ "github.com/aldor007/go-aws-auth", "github.com/aldor007/stow", + "github.com/aldor007/stow/b2", "github.com/aldor007/stow/http", "github.com/aldor007/stow/local", "github.com/aldor007/stow/local-meta", diff --git a/Gopkg.toml b/Gopkg.toml index 698ea17..5ffb737 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,7 +1,6 @@ - # Gopkg.toml example # -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html # for detailed Gopkg.toml documentation. # # required = ["github.com/user/thing/cmd/thing"] @@ -17,159 +16,79 @@ # source = "github.com/myfork/project2" # # [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - -[[constraint]] - name = "cloud.google.com/go" - version = "0.13.0" - -[[constraint]] - name = "github.com/Azure/azure-sdk-for-go" - version = "10.3.1-beta" +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true -[[constraint]] - name = "github.com/aws/aws-sdk-go" - version = "1.10.51" [[constraint]] branch = "master" - name = "github.com/cheekybits/is" - -[[constraint]] - name = "github.com/davecgh/go-spew" - version = "1.1.0" - -[[constraint]] - name = "github.com/dgrijalva/jwt-go" - version = "3.0.0" - -[[constraint]] - name = "github.com/google/go-cmp" - version = "0.1.0" + name = "github.com/aldor007/go-aws-auth" [[constraint]] - branch = "master" - name = "github.com/graymeta/stow" + branch = "develop" + name = "github.com/aldor007/stow" [[constraint]] - name = "github.com/mattn/go-colorable" - version = "0.0.9" + name = "github.com/djherbis/stream" + version = "1.2.0" [[constraint]] - name = "github.com/mattn/go-isatty" - version = "0.0.2" + name = "github.com/go-chi/chi" + version = "3.3.4" [[constraint]] - branch = "master" - name = "github.com/ncw/swift" + name = "github.com/karlseguin/ccache" + version = "2.0.2" [[constraint]] name = "github.com/pkg/errors" - version = "0.8.0" - -[[constraint]] - name = "github.com/pmezard/go-difflib" - version = "1.0.0" - -[[constraint]] - branch = "master" - name = "github.com/stretchr/objx" + version = "0.8.1" [[constraint]] - name = "github.com/stretchr/testify" - version = "1.1.4" + name = "github.com/prometheus/client_golang" + version = "0.9.2" [[constraint]] branch = "master" - name = "github.com/valyala/bytebufferpool" + name = "github.com/prometheus/client_model" [[constraint]] - branch = "master" - name = "github.com/valyala/fasttemplate" + name = "github.com/spaolacci/murmur3" + version = "1.1.0" [[constraint]] - branch = "master" - name = "go4.org" + name = "github.com/stretchr/testify" + version = "1.3.0" [[constraint]] - branch = "master" - name = "golang.org/x/crypto" + name = "go.uber.org/zap" + version = "1.9.1" [[constraint]] branch = "master" name = "golang.org/x/net" -[[constraint]] - branch = "master" - name = "golang.org/x/oauth2" - -[[constraint]] - branch = "master" - name = "golang.org/x/sync" - -[[constraint]] - branch = "master" - name = "golang.org/x/sys" - -[[constraint]] - branch = "master" - name = "golang.org/x/text" - -[[constraint]] - branch = "master" - name = "google.golang.org/api" - -[[constraint]] - name = "google.golang.org/appengine" - version = "1.0.0" - -[[constraint]] - branch = "master" - name = "google.golang.org/genproto" - -[[constraint]] - name = "google.golang.org/grpc" - version = "1.6.0" - -[[constraint]] - branch = "v1" - name = "gopkg.in/check.v1" - [[constraint]] name = "gopkg.in/h2non/bimg.v1" - version = "1.0.17" + version = "1.0.19" [[constraint]] - name = "gopkg.in/h2non/filetype.v1" - version = "1.0.3" + name = "gopkg.in/h2non/gock.v1" + version = "1.0.12" [[constraint]] - branch = "v2" - name = "gopkg.in/yaml.v2" - -[[constraint]] - name = "github.com/djherbis/stream" - branch = "master" - -[[constraint]] - name = "github.com/go-chi/chi" - version = "v3.3.0" - -[[constraint]] - name = "github.com/spaolacci/murmur3" branch = "master" + name = "gopkg.in/kothar/brotli-go.v0" [[constraint]] - name = "github.com/aldor007/go-aws-auth" - branch = "master" - -[[constraint]] - name = "github.com/karlseguin/ccache" - branch = "master" - -[[constraint]] - name = "github.com/prometheus/client_golang" - version = "v0.8.0" + name = "gopkg.in/yaml.v2" + version = "2.2.2" +[prune] + go-tests = true + unused-packages = true diff --git a/go.mod b/go.mod index 10af1c9..24252f7 100644 --- a/go.mod +++ b/go.mod @@ -1,34 +1,40 @@ -module github.com/aldor007/mort +module github.com/Aldor007/mort require ( github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 - github.com/aldor007/stow v0.0.0-20171229203915-67ba3fffe7b8 - github.com/aws/aws-sdk-go v1.12.28 - github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a - github.com/davecgh/go-spew v1.1.0 - github.com/djherbis/stream v0.0.0-20170408001054-5cf27428a014 - github.com/go-chi/chi v3.3.0+incompatible - github.com/go-ini/ini v1.31.1 - github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf - github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 - github.com/karlseguin/ccache v0.0.0-20170904064732-b425c9ca005a - github.com/matttproud/golang_protobuf_extensions v1.0.0 - github.com/pkg/errors v0.8.0 + github.com/aldor007/stow v0.0.0-20190109171535-d59ae5fea5cf + github.com/aws/aws-sdk-go v1.16.14 + github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 + github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 + github.com/davecgh/go-spew v1.1.1 + github.com/djherbis/stream v1.2.0 + github.com/go-chi/chi v3.3.3+incompatible + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/golang/protobuf v1.2.0 + github.com/google/readahead v0.0.0-20161222183148-eaceba169032 + github.com/graymeta/stow v0.0.0-20181228161447-b469cfb112f8 + github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af + github.com/karlseguin/ccache v2.0.2+incompatible + github.com/kurin/blazer v0.5.1 + github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/pkg/errors v0.8.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v0.8.0 - github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 - github.com/prometheus/common v0.0.0-20180110214958-89604d197083 - github.com/prometheus/procfs v0.0.0-20171226183907-b15cd069a834 + github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 + github.com/prometheus/client_golang v0.9.2 + github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 + github.com/prometheus/common v0.0.0-20190107103113-2998b132700a + github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 - github.com/stretchr/testify v1.1.4 - github.com/vmihailenco/msgpack v3.2.0+incompatible - go.uber.org/atomic v1.3.1 + github.com/stretchr/testify v1.3.0 + github.com/vmihailenco/msgpack v4.0.1+incompatible + go.uber.org/atomic v1.2.0 go.uber.org/multierr v1.1.0 - go.uber.org/zap v1.7.1 - golang.org/x/net v0.0.0-20171115151908-9dfe39835686 - google.golang.org/appengine v1.0.0 - gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/h2non/gock.v1 v1.0.9 + go.uber.org/zap v1.9.1 + golang.org/x/net v0.0.0-20190107155100-1a61f4433d85 + google.golang.org/appengine v1.4.0 + gopkg.in/h2non/bimg.v1 v1.0.19 + gopkg.in/h2non/gock.v1 v1.0.12 gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6 - gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab + gopkg.in/kothar/go-backblaze.v0 v0.0.0-20180916190456-9ac0cf0dab1a + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum deleted file mode 100644 index 4780753..0000000 --- a/go.sum +++ /dev/null @@ -1,61 +0,0 @@ -github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 h1:TuxcUGZxsHKCW50twhDbGdRtuMXmMDCmOssevV0ZoOA= -github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272/go.mod h1:gm7JFOl7Sp8r+lJqHXOkL0HyUFR05QXueqxVufqfkTg= -github.com/aldor007/mort v0.12.0 h1:2B6AE+1dN+6HORZ9t7u2ysJ4Kfz3lHIJoiICvf3hQyA= -github.com/aldor007/mort v0.12.0/go.mod h1:LFg/bthesXeJvAdfULjLOhu4KcuTClSigO2KP5HM+CM= -github.com/aldor007/stow v0.0.0-20171229203915-67ba3fffe7b8 h1:pD2XaHkkGTs3BjuWx+jEgtMcjO3NhMbp62Gb96WNkP0= -github.com/aldor007/stow v0.0.0-20171229203915-67ba3fffe7b8/go.mod h1:TUBrYvdpmfMQKeKUIZeveJE/Y5VufnvvtDE/kxJ9X3U= -github.com/aws/aws-sdk-go v1.12.28 h1:nXc2Sj3QwHle2Kf52Fr6HpyDNzNd5oYlPnqbB8hUBOg= -github.com/aws/aws-sdk-go v1.12.28/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/djherbis/stream v0.0.0-20170408001054-5cf27428a014 h1:h5mCyHPtrWnvW1EabH6GRSs3RCWv6M3ZAEp3CwL9x14= -github.com/djherbis/stream v0.0.0-20170408001054-5cf27428a014/go.mod h1:ZNVKPVRCmrwhCwQHZUpVHHrq2rtGLrG1t3T/TThYLP8= -github.com/go-chi/chi v3.3.0+incompatible h1:19pl0NEHtjUmuCdXZpZ4RP3dJWdf05Fg8DDTFLnq++8= -github.com/go-chi/chi v3.3.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-ini/ini v1.31.1 h1:Ir+/CttgobzfH80vVjhKUdZVaS2oVD7/33M37NwE5HM= -github.com/go-ini/ini v1.31.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf h1:pFr/u+m8QUBMW/itAczltF3guNRAL7XDs5tD3f6nSD0= -github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/karlseguin/ccache v0.0.0-20170904064732-b425c9ca005a h1:HBV4xAoAO7cxLe8pRDakTUWx1WBNv6rUDuU1S9S0a7E= -github.com/karlseguin/ccache v0.0.0-20170904064732-b425c9ca005a/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= -github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc= -github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083 h1:BVsJT8+ZbyuL3hypz/HmEiM8h2P6hBQGig4el9/MdjA= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/procfs v0.0.0-20171226183907-b15cd069a834 h1:jQVavQNtg1+b1/lxzhVA1JS3NYnCTo4b5Mo6z1N6apo= -github.com/prometheus/procfs v0.0.0-20171226183907-b15cd069a834/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 h1:lDiM+yMjW7Ork8mhl0YN0qO1Z02qGoe1vwzGc1gP/8U= -github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/stretchr/testify v1.1.4 h1:ToftOQTytwshuOSj6bDSolVUa3GINfJP/fg3OkkOzQQ= -github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/vmihailenco/msgpack v3.2.0+incompatible h1:ZnAl9N6gKGsMZ7Js3co4p+qK1Nu2+8iuD8RyWTsAajs= -github.com/vmihailenco/msgpack v3.2.0+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -go.uber.org/atomic v1.3.1 h1:U8WaWEmp56LGz7PReduqHRVF6zzs9GbMC2NEZ42dxSQ= -go.uber.org/atomic v1.3.1/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.7.1 h1:wKPciimwkIgV4Aag/wpSDzvtO5JrfwdHKHO7blTHx7Q= -go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/net v0.0.0-20171115151908-9dfe39835686 h1:fxZ+mPcFhowcPZdlXrTF3GFhWVr/3wZyXQ8xW8WYGLU= -golang.org/x/net v0.0.0-20171115151908-9dfe39835686/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= -gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= -gopkg.in/h2non/gock.v1 v1.0.9 h1:qZnr7hBXCBgWcQ0MO9PGsdSK5Rpw2Oz6yXZPcKVL5Vs= -gopkg.in/h2non/gock.v1 v1.0.9/go.mod h1:KHI4Z1sxDW6P4N3DfTWSEza07YpkQP7KJBfglRMEjKY= -gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6 h1:M8GdJL0oESXVmjOOT3upJyFkKs5o1jJERiKYOZjVes0= -gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6/go.mod h1:nVee4zUY+UoXjOfM57w44w2XjsoIqIKd4A9vktFSQ6I= -gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= -gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/package-lock.json b/package-lock.json index 7f043fb..01df4d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,10 @@ "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", "dev": true, "requires": { - "co": "4.6.0", - "fast-deep-equal": "1.1.0", - "fast-json-stable-stringify": "2.0.0", - "json-schema-traverse": "0.3.1" + "co": "^4.6.0", + "fast-deep-equal": "^1.0.0", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.3.0" } }, "asn1": { @@ -45,7 +45,7 @@ "requires": { "buffer": "4.9.1", "crypto-browserify": "1.0.9", - "events": "1.1.1", + "events": "^1.1.1", "jmespath": "0.15.0", "querystring": "0.2.0", "sax": "1.2.1", @@ -84,7 +84,7 @@ "dev": true, "optional": true, "requires": { - "tweetnacl": "0.14.5" + "tweetnacl": "^0.14.3" } }, "boom": { @@ -93,7 +93,7 @@ "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", "dev": true, "requires": { - "hoek": "4.2.1" + "hoek": "4.x.x" } }, "brace-expansion": { @@ -101,7 +101,7 @@ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", "requires": { - "balanced-match": "1.0.0", + "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, @@ -115,9 +115,9 @@ "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "requires": { - "base64-js": "1.2.1", - "ieee754": "1.1.8", - "isarray": "1.0.0" + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" } }, "caseless": { @@ -131,12 +131,12 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.2.tgz", "integrity": "sha1-D2RYS6ZC8PKs4oBiefTwbKI61zw=", "requires": { - "assertion-error": "1.0.2", - "check-error": "1.0.2", - "deep-eql": "3.0.1", - "get-func-name": "2.0.0", - "pathval": "1.1.0", - "type-detect": "4.0.5" + "assertion-error": "^1.0.1", + "check-error": "^1.0.1", + "deep-eql": "^3.0.0", + "get-func-name": "^2.0.0", + "pathval": "^1.0.0", + "type-detect": "^4.0.0" } }, "check-error": { @@ -155,7 +155,7 @@ "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", "requires": { - "delayed-stream": "1.0.0" + "delayed-stream": "~1.0.0" } }, "commander": { @@ -189,7 +189,7 @@ "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", "dev": true, "requires": { - "boom": "5.2.0" + "boom": "5.x.x" }, "dependencies": { "boom": { @@ -198,7 +198,7 @@ "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", "dev": true, "requires": { - "hoek": "4.2.1" + "hoek": "4.x.x" } } } @@ -214,7 +214,7 @@ "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", "dev": true, "requires": { - "assert-plus": "1.0.0" + "assert-plus": "^1.0.0" } }, "debug": { @@ -230,7 +230,7 @@ "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", "requires": { - "type-detect": "4.0.5" + "type-detect": "^4.0.0" } }, "delayed-stream": { @@ -250,7 +250,7 @@ "dev": true, "optional": true, "requires": { - "jsbn": "0.1.1" + "jsbn": "~0.1.0" } }, "escape-string-regexp": { @@ -297,9 +297,9 @@ "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=", "requires": { - "asynckit": "0.4.0", - "combined-stream": "1.0.5", - "mime-types": "2.1.17" + "asynckit": "^0.4.0", + "combined-stream": "^1.0.5", + "mime-types": "^2.1.12" } }, "formidable": { @@ -323,7 +323,7 @@ "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", "dev": true, "requires": { - "assert-plus": "1.0.0" + "assert-plus": "^1.0.0" } }, "glob": { @@ -331,12 +331,12 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", "requires": { - "fs.realpath": "1.0.0", - "inflight": "1.0.6", - "inherits": "2.0.3", - "minimatch": "3.0.4", - "once": "1.4.0", - "path-is-absolute": "1.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" } }, "growl": { @@ -356,8 +356,8 @@ "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", "dev": true, "requires": { - "ajv": "5.5.2", - "har-schema": "2.0.0" + "ajv": "^5.1.0", + "har-schema": "^2.0.0" } }, "has-flag": { @@ -371,10 +371,10 @@ "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", "dev": true, "requires": { - "boom": "4.3.1", - "cryptiles": "3.1.2", - "hoek": "4.2.1", - "sntp": "2.1.0" + "boom": "4.x.x", + "cryptiles": "3.x.x", + "hoek": "4.x.x", + "sntp": "2.x.x" } }, "he": { @@ -394,9 +394,9 @@ "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", "dev": true, "requires": { - "assert-plus": "1.0.0", - "jsprim": "1.4.1", - "sshpk": "1.13.1" + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" } }, "ieee754": { @@ -409,8 +409,8 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "requires": { - "once": "1.4.0", - "wrappy": "1.0.2" + "once": "^1.3.0", + "wrappy": "1" } }, "inherits": { @@ -502,7 +502,7 @@ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz", "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=", "requires": { - "mime-db": "1.30.0" + "mime-db": "~1.30.0" } }, "minimatch": { @@ -510,7 +510,7 @@ "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "requires": { - "brace-expansion": "1.1.8" + "brace-expansion": "^1.1.7" } }, "minimist": { @@ -565,7 +565,7 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "requires": { - "wrappy": "1.0.2" + "wrappy": "1" } }, "path-is-absolute": { @@ -609,13 +609,13 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "1.0.7", - "safe-buffer": "5.1.1", - "string_decoder": "1.0.3", - "util-deprecate": "1.0.2" + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.0.3", + "util-deprecate": "~1.0.1" } }, "request": { @@ -624,28 +624,28 @@ "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", "dev": true, "requires": { - "aws-sign2": "0.7.0", - "aws4": "1.6.0", - "caseless": "0.12.0", - "combined-stream": "1.0.5", - "extend": "3.0.1", - "forever-agent": "0.6.1", - "form-data": "2.3.1", - "har-validator": "5.0.3", - "hawk": "6.0.2", - "http-signature": "1.2.0", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.17", - "oauth-sign": "0.8.2", - "performance-now": "2.1.0", - "qs": "6.5.1", - "safe-buffer": "5.1.1", - "stringstream": "0.0.5", - "tough-cookie": "2.3.4", - "tunnel-agent": "0.6.0", - "uuid": "3.1.0" + "aws-sign2": "~0.7.0", + "aws4": "^1.6.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.5", + "extend": "~3.0.1", + "forever-agent": "~0.6.1", + "form-data": "~2.3.1", + "har-validator": "~5.0.3", + "hawk": "~6.0.2", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.17", + "oauth-sign": "~0.8.2", + "performance-now": "^2.1.0", + "qs": "~6.5.1", + "safe-buffer": "^5.1.1", + "stringstream": "~0.0.5", + "tough-cookie": "~2.3.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.1.0" } }, "safe-buffer": { @@ -664,7 +664,7 @@ "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", "dev": true, "requires": { - "hoek": "4.2.1" + "hoek": "4.x.x" } }, "sshpk": { @@ -673,14 +673,14 @@ "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", "dev": true, "requires": { - "asn1": "0.2.3", - "assert-plus": "1.0.0", - "bcrypt-pbkdf": "1.0.1", - "dashdash": "1.14.1", - "ecc-jsbn": "0.1.1", - "getpass": "0.1.7", - "jsbn": "0.1.1", - "tweetnacl": "0.14.5" + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "tweetnacl": "~0.14.0" } }, "string_decoder": { @@ -688,7 +688,7 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", "requires": { - "safe-buffer": "5.1.1" + "safe-buffer": "~5.1.0" } }, "stringstream": { @@ -702,16 +702,16 @@ "resolved": "https://registry.npmjs.org/superagent/-/superagent-3.8.1.tgz", "integrity": "sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==", "requires": { - "component-emitter": "1.2.1", - "cookiejar": "2.1.1", - "debug": "3.1.0", - "extend": "3.0.1", - "form-data": "2.3.1", - "formidable": "1.1.1", - "methods": "1.1.2", - "mime": "1.4.1", - "qs": "6.5.1", - "readable-stream": "2.3.3" + "component-emitter": "^1.2.0", + "cookiejar": "^2.1.0", + "debug": "^3.1.0", + "extend": "^3.0.0", + "form-data": "^2.3.1", + "formidable": "^1.1.1", + "methods": "^1.1.1", + "mime": "^1.4.1", + "qs": "^6.5.1", + "readable-stream": "^2.0.5" } }, "superagent-binary-parser": { @@ -725,8 +725,8 @@ "resolved": "https://registry.npmjs.org/supertest/-/supertest-3.0.0.tgz", "integrity": "sha1-jUu2j9GDDuBwM7HFpamkAhyWUpY=", "requires": { - "methods": "1.1.2", - "superagent": "3.8.1" + "methods": "~1.1.2", + "superagent": "^3.0.0" } }, "supports-color": { @@ -734,7 +734,7 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.4.0.tgz", "integrity": "sha512-rKC3+DyXWgK0ZLKwmRsrkyHVZAjNkfzeehuFWdGGcqGDTZFH73+RH6S/RDAAxl9GusSjZSUWYLmT9N5pzXFOXQ==", "requires": { - "has-flag": "2.0.0" + "has-flag": "^2.0.0" } }, "tough-cookie": { @@ -743,7 +743,7 @@ "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", "dev": true, "requires": { - "punycode": "1.4.1" + "punycode": "^1.4.1" }, "dependencies": { "punycode": { @@ -760,7 +760,7 @@ "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", "dev": true, "requires": { - "safe-buffer": "5.1.1" + "safe-buffer": "^5.0.1" } }, "tweetnacl": { @@ -800,9 +800,9 @@ "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", "dev": true, "requires": { - "assert-plus": "1.0.0", + "assert-plus": "^1.0.0", "core-util-is": "1.0.2", - "extsprintf": "1.3.0" + "extsprintf": "^1.2.0" } }, "wrappy": { @@ -815,8 +815,8 @@ "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz", "integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=", "requires": { - "sax": "1.2.1", - "xmlbuilder": "4.2.1" + "sax": ">=0.6.0", + "xmlbuilder": "^4.1.0" } }, "xmlbuilder": { @@ -824,7 +824,7 @@ "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz", "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=", "requires": { - "lodash": "4.17.4" + "lodash": "^4.0.0" } } } diff --git a/pkg/config/config.go b/pkg/config/config.go index 9bcbba1..3c7b7ff 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -28,7 +28,7 @@ var instance *Config var once sync.Once // storageKinds is list of available storage kinds -var storageKinds = []string{"local", "local-meta", "s3", "http", "noop"} +var storageKinds = []string{"local", "local-meta", "s3", "http", "b2", "noop"} // transformKind is list of available kinds of transforms var transformKinds = []string{"query", "presets", "presets-query"} diff --git a/pkg/config/types.go b/pkg/config/types.go index 8ea2828..f992d7e 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -52,7 +52,7 @@ type Transform struct { // Storage contains information about kind of used storage type Storage struct { RootPath string `yaml:"rootPath,omitempty"` // root path for local-* storage - Kind string `yaml:"kind"` // type of storage from list ("local", "local-meta", "s3", "http", "noop") + Kind string `yaml:"kind"` // type of storage from list ("local", "local-meta", "s3", "http", "b2","noop") Url string `yaml:"url,omitempty"` // Url for http storage Headers map[string]string `yaml:"headers,omitempty"` // request headers for http storage AccessKey string `yaml:"accessKey,omitempty"` // access key for s3 storage @@ -61,6 +61,8 @@ type Storage struct { Endpoint string `yaml:"endpoint,omitempty"` // endpoint for s3 storage PathPrefix string `yaml:"pathPrefix,omitempty"` // prefix in path for all storage Bucket string `yaml:"bucket"` + Account string `yaml:"account"` // account name for b2 + Key string `yaml:"key"` // key for b2 Hash string // unique hash for given storage } diff --git a/pkg/engine/image_engine.go b/pkg/engine/image_engine.go index b29a8b5..9cfcddc 100644 --- a/pkg/engine/image_engine.go +++ b/pkg/engine/image_engine.go @@ -7,7 +7,6 @@ import ( "time" "github.com/spaolacci/murmur3" - "go.uber.org/zap" "gopkg.in/h2non/bimg.v1" "bytes" @@ -15,6 +14,7 @@ import ( "github.com/aldor007/mort/pkg/object" "github.com/aldor007/mort/pkg/response" "github.com/aldor007/mort/pkg/transforms" + "go.uber.org/zap" "sync" ) @@ -75,7 +75,7 @@ func (c *ImageEngine) Process(obj *object.FileObject, trans []transforms.Transfo res := response.NewBuf(200, buf) res.SetContentType("image/" + bimg.DetermineImageTypeName(buf)) //res.Set("cache-control", "max-age=6000, public") - res.Set("Last-Modified", time.Now().Format(http.TimeFormat)) + res.Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) res.Set("ETag", createWeakEtag(strconv.FormatUint(hash.Sum64(), 16))) meta, err := bimg.Metadata(buf) if err == nil { @@ -83,7 +83,7 @@ func (c *ImageEngine) Process(obj *object.FileObject, trans []transforms.Transfo res.Set("x-amz-meta-public-height", strconv.Itoa(meta.Size.Height)) } else { - monitoring.Log().Warn("ImageEngine/process unable to get metadata", zap.Error(err), obj.LogData()...) + monitoring.Log().Warn("ImageEngine/process unable to get metadata", obj.LogData(zap.Error(err))...) } return res, nil diff --git a/pkg/monitoring/prometheus_test.go b/pkg/monitoring/prometheus_test.go index 9035eaa..2d7ac44 100644 --- a/pkg/monitoring/prometheus_test.go +++ b/pkg/monitoring/prometheus_test.go @@ -144,7 +144,7 @@ func TestPrometheusReporter_TimerVec(t *testing.T) { tr.Done() result := dto.Metric{} - p.histogramsVec["test-2-hist-vec"].With(prometheus.Labels{"label": "elo"}).Write(&result) + p.histogramsVec["test-2-hist-vec"].With(prometheus.Labels{"label": "elo"}).(prometheus.Metric).Write(&result) assert.InEpsilon(t, *result.Histogram.SampleSum, 100, 5) } diff --git a/pkg/object/file_object.go b/pkg/object/file_object.go index cb506c5..0b78bf8 100644 --- a/pkg/object/file_object.go +++ b/pkg/object/file_object.go @@ -76,7 +76,7 @@ func (o *FileObject) UpdateKey(str string) { o.Key = o.Key + str } -func (obj *FileObject) LogData() []zapcore.Field { +func (obj *FileObject) LogData(fields ...zapcore.Field) []zapcore.Field { result := []zapcore.Field{zap.String("obj.path", obj.Uri.Path), zap.String("obj.Key", obj.Key), zap.String("obj.Bucket", obj.Bucket), zap.String("obj.Storage", obj.Storage.Kind), zap.Bool("obj.HasTransforms", obj.HasTransform()), zap.Bool("obj.HasParent", obj.HasParent())} @@ -84,6 +84,6 @@ func (obj *FileObject) LogData() []zapcore.Field { result = append(result, zap.String("parent.Key", obj.Parent.Key), zap.String("parent.Path", obj.Parent.Uri.Path)) } - return result + return append(result, fields...) } diff --git a/pkg/processor/processor.go b/pkg/processor/processor.go index 6713681..b74e6a6 100644 --- a/pkg/processor/processor.go +++ b/pkg/processor/processor.go @@ -84,7 +84,7 @@ func (r *RequestProcessor) Process(req *http.Request, obj *object.FileObject) *r case <-ctx.Done(): msg.cancel <- struct{}{} close(msg.responseChan) - monitoring.Log().Warn("Process timeout", obj.LogData()..., zap.String("error", "Context.timeout")) + monitoring.Log().Warn("Process timeout", obj.LogData(zap.String("error", "Context.timeout"))...) return r.replyWithError(obj, 499, errContextCancel) case res := <-msg.responseChan: r.plugins.PostProcess(obj, req, res) @@ -169,6 +169,7 @@ func (r *RequestProcessor) process(req *http.Request, obj *object.FileObject) *r } func handlePUT(req *http.Request, obj *object.FileObject) *response.Response { + defer req.Body.Close() return storage.Set(obj, req.Header, req.ContentLength, req.Body) } @@ -183,7 +184,7 @@ func (r *RequestProcessor) collapseGET(req *http.Request, obj *object.FileObject } monitoring.Report().Inc("collapsed_count") - monitoring.Log().Info("Lock not acquired", obj.LogData()...) + monitoring.Log().Info("Lock not acquired", obj.LogData()...) timer := time.NewTimer(r.lockTimeout) for { @@ -363,7 +364,8 @@ func (r *RequestProcessor) handleNotFound(obj, parentObj *object.FileObject, tra return r.processImage(obj, parentRes, transformsTab) } else if obj.HasTransform() { parentRes.Close() - monitoring.Log().Warn("Not performing transforms", obj.LogData()..., zap.Int("parent.sc", parentRes.StatusCode), zap.String("parent.ContentType", parentRes.Headers.Get(response.HeaderContentType)), zap.Error(parentRes.Error())) + monitoring.Log().Warn("Not performing transforms", obj.LogData(zap.Int("parent.sc", parentRes.StatusCode), + zap.String("parent.ContentType", parentRes.Headers.Get(response.HeaderContentType)), zap.Error(parentRes.Error()))...) } } @@ -408,7 +410,7 @@ func (r *RequestProcessor) processImage(obj *object.FileObject, parent *response ctx := obj.Ctx taked := r.throttler.Take(ctx) if !taked { - monitoring.Log().Warn("Processor/processImage", obj.LogData()..., zap.String("error", "throttled")) + monitoring.Log().Warn("Processor/processImage", obj.LogData(zap.String("error", "throttled"))...) monitoring.Report().Inc("throttled_count") return r.replyWithError(obj, 503, errThrottled) } @@ -418,7 +420,7 @@ func (r *RequestProcessor) processImage(obj *object.FileObject, parent *response mergedTrans := transforms.Merge(transformsTab) mergedLen := len(mergedTrans) - monitoring.Log().Info("Performing transforms", obj.LogData()..., zap.Int("transformsLen", transformsLen), zap.Int("mergedLen", mergedLen)) + monitoring.Log().Info("Performing transforms", obj.LogData(zap.Int("transformsLen", transformsLen), zap.Int("mergedLen", mergedLen))...) eng := engine.NewImageEngine(parent) res, err := eng.Process(obj, mergedTrans) if err != nil { @@ -435,7 +437,7 @@ func (r *RequestProcessor) processImage(obj *object.FileObject, parent *response resS.Close() }(*obj, resCpy) } else { - monitoring.Log().Warn("Processor/processImage", obj.LogData()..., zap.Error(err)) + monitoring.Log().Warn("Processor/processImage", obj.LogData(zap.Error(err))...) } return res diff --git a/pkg/response/response.go b/pkg/response/response.go index c0a7453..fcb48d0 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -319,7 +319,11 @@ func (r *Response) CopyWithStream() (*Response, error) { r.bodyReader = r.reader - r.resStream = stream.NewMemStream() + var err error + r.resStream, err = stream.New("res") + if err != nil { + return nil, err + } c.resStream = r.resStream c.hasParent = true r.reader = ioutil.NopCloser(io.TeeReader(r.bodyReader, r.resStream)) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index b46df32..7fb8d4f 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -7,18 +7,18 @@ import ( fileStorage "github.com/aldor007/stow/local" metaStorage "github.com/aldor007/stow/local-meta" // import blank to register noop adapter in stow.Register + "encoding/xml" + "github.com/aldor007/mort/pkg/monitoring" + "github.com/aldor007/mort/pkg/object" + "github.com/aldor007/mort/pkg/response" + b2Storage "github.com/aldor007/stow/b2" _ "github.com/aldor007/stow/noop" s3Storage "github.com/aldor007/stow/s3" + "go.uber.org/zap" "io" "mime" "net/http" "path" - - "encoding/xml" - "github.com/aldor007/mort/pkg/monitoring" - "github.com/aldor007/mort/pkg/object" - "github.com/aldor007/mort/pkg/response" - "go.uber.org/zap" "strings" "sync" "time" @@ -40,7 +40,7 @@ func Get(obj *object.FileObject) *response.Response { key := getKey(obj) client, err := getClient(obj) if err != nil { - monitoring.Log().Info("Storage/Get get client", obj.LogData()..., zap.Error(err)) + monitoring.Log().Info("Storage/Get get client", obj.LogData(zap.Error(err))...) return response.NewError(503, err) } @@ -51,14 +51,14 @@ func Get(obj *object.FileObject) *response.Response { return response.NewString(404, notFound) } - monitoring.Log().Info("Storage/Get item response", obj.LogData()..., zap.Error(err)) + monitoring.Log().Info("Storage/Get item response", obj.LogData(zap.Error(err))...) return response.NewError(500, err) } if isDir(item) == false { reader, err := item.Open() if err != nil { - monitoring.Logs().Warnw("Storage/Get open item", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/Get open item", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } return prepareResponse(obj, reader, item) @@ -77,18 +77,18 @@ func Head(obj *object.FileObject) *response.Response { key := getKey(obj) client, err := getClient(obj) if err != nil { - monitoring.Logs().Infow("Storage/Head get client", obj.LogData()..., zap.Error(err)) + monitoring.Log().Info("Storage/Head get client", obj.LogData(zap.Error(err))...) return response.NewError(503, err) } item, err := client.Item(key) if err != nil { if err == stow.ErrNotFound { - monitoring.Logs().Infow("Storage/Head item response", obj.LogData()..., zap.Int("sc", 404)) + monitoring.Log().Info("Storage/Head item response", obj.LogData(zap.Int("sc", 404))...) return response.NewString(404, notFound) } - monitoring.Logs().Infow("Storage/Head item response", obj.LogData()..., zap.Error(err)) + monitoring.Log().Info("Storage/Head item response", obj.LogData(zap.Error(err))...) return response.NewError(500, err) } @@ -102,14 +102,14 @@ func Set(obj *object.FileObject, metaHeaders http.Header, contentLen int64, body defer t.Done() client, err := getClient(obj) if err != nil { - monitoring.Logs().Warnw("Storage/Set create client", obj.LogData()..., zap.Int("sc", 503), zap.Error(err)) + monitoring.Log().Warn("Storage/Set create client", obj.LogData(zap.Int("sc", 503), zap.Error(err))...) return response.NewError(503, err) } _, err = client.Put(getKey(obj), body, contentLen, prepareMetadata(obj, metaHeaders)) if err != nil { - monitoring.Logs().Warnw("Storage/Set cannot set", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/Set cannot set", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } @@ -125,7 +125,7 @@ func Delete(obj *object.FileObject) *response.Response { defer t.Done() client, err := getClient(obj) if err != nil { - monitoring.Logs().Warnw("Storage/Delete create client", obj.LogData()..., zap.Int("sc", 503), zap.Error(err)) + monitoring.Log().Warn("Storage/Delete create client", obj.LogData(zap.Int("sc", 503), zap.Error(err))...) return response.NewError(503, err) } @@ -134,7 +134,7 @@ func Delete(obj *object.FileObject) *response.Response { err = client.RemoveItem(getKey(obj)) if err != nil { - monitoring.Logs().Warnw("Storage/Delete cannot delete", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/Delete cannot delete", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } } else if resHead.StatusCode == 404 { @@ -150,7 +150,7 @@ func Delete(obj *object.FileObject) *response.Response { func List(obj *object.FileObject, maxKeys int, _ string, prefix string, marker string) *response.Response { client, err := getClient(obj) if err != nil { - monitoring.Logs().Warnw("Storage/List", obj.LogData()..., zap.Int("sc", 503), zap.Error(err)) + monitoring.Log().Warn("Storage/List", obj.LogData(zap.Int("sc", 503), zap.Error(err))...) return response.NewError(503, err) } @@ -158,7 +158,7 @@ func List(obj *object.FileObject, maxKeys int, _ string, prefix string, marker s _, err = client.Item(prefix) if err != nil { if err == stow.ErrNotFound { - monitoring.Logs().Infow("Storage/List item not fountresponse", obj.LogData()..., zap.Int("sc", 404)) + monitoring.Log().Info("Storage/List item not fountresponse", obj.LogData(zap.Int("sc", 404))...) return response.NewString(404, obj.Key) } } @@ -166,7 +166,7 @@ func List(obj *object.FileObject, maxKeys int, _ string, prefix string, marker s items, resultMarker, err := client.Items(prefix, marker, maxKeys) if err != nil { - monitoring.Logs().Warnw("Storage/List", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/List", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } @@ -287,6 +287,11 @@ func getClient(obj *object.FileObject) (stow.Container, error) { config = stow.ConfigMap{ metaStorage.ConfigKeyPath: storageCfg.RootPath, } + case "b2": + config = stow.ConfigMap{ + b2Storage.ConfigAccountID: storageCfg.Account, + b2Storage.ConfigApplicationKey: storageCfg.Key, + } } @@ -306,7 +311,7 @@ func getClient(obj *object.FileObject) (stow.Container, error) { container, err := client.Container(bucketName) if err != nil { - monitoring.Log().Info("Storage/getClient error", zap.String("kind", storageCfg.Kind), zap.String("bucket", obj.Bucket), zap.Error(err)) + monitoring.Log().Info("Storage/getClient container get error", zap.String("kind", storageCfg.Kind), zap.String("bucket", bucketName), zap.Error(err)) if err == stow.ErrNotFound && strings.HasPrefix(storageCfg.Kind, "local") { container, err = client.CreateContainer(obj.Bucket) if err != nil { @@ -326,7 +331,13 @@ func getClient(obj *object.FileObject) (stow.Container, error) { } func getKey(obj *object.FileObject) string { - return path.Join(obj.Storage.PathPrefix, obj.Key) + switch obj.Storage.Kind { + case "b2": + return strings.TrimPrefix(path.Join(obj.Storage.PathPrefix, obj.Key), "/") + default: + return path.Join(obj.Storage.PathPrefix, obj.Key) + + } } func prepareResponse(obj *object.FileObject, stream io.ReadCloser, item stow.Item) *response.Response { @@ -335,7 +346,7 @@ func prepareResponse(obj *object.FileObject, stream io.ReadCloser, item stow.Ite metadata, err := item.Metadata() if err != nil { - monitoring.Log().Warn("Storage/prepareResponse read metadata error", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/prepareResponse read metadata error", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } @@ -343,13 +354,13 @@ func prepareResponse(obj *object.FileObject, stream io.ReadCloser, item stow.Ite etag, err := item.ETag() if err != nil { - monitoring.Log().Warn("Storage/prepareResponse read etag error", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/prepareResponse read etag error", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } lastMod, err := item.LastMod() if err != nil { - monitoring.Log().Warn("Storage/prepareResponse read lastmod error", obj.LogData()..., zap.Int("sc", 500), zap.Error(err)) + monitoring.Log().Warn("Storage/prepareResponse read lastmod error", obj.LogData(zap.Int("sc", 500), zap.Error(err))...) return response.NewError(500, err) } @@ -361,10 +372,12 @@ func prepareResponse(obj *object.FileObject, stream io.ReadCloser, item stow.Ite if etag != "" { res.Set("ETag", etag) } - res.Set("Last-Modified", lastMod.Format(http.TimeFormat)) + res.Set("Last-Modified", lastMod.UTC().Format(http.TimeFormat)) if contentType, ok := metadata["Content-Type"]; ok { res.SetContentType(contentType.(string)) + } else if contentType, ok := metadata["content-type"]; ok { + res.SetContentType(contentType.(string)) } else { ct := mime.TypeByExtension(path.Ext(obj.Uri.Path)) if ct != "" { @@ -391,7 +404,7 @@ func prepareMetadata(obj *object.FileObject, metaHeaders http.Header) map[string default: keyLower := strings.ToLower(k) if strings.HasPrefix(keyLower, "x-amz-meta") || keyLower == "content-type" || keyLower == "etag" { - metadata[k] = v[0] + metadata[keyLower] = v[0] } } } @@ -401,13 +414,14 @@ func prepareMetadata(obj *object.FileObject, metaHeaders http.Header) map[string func parseMetadata(obj *object.FileObject, metadata map[string]interface{}, res *response.Response) { for k, v := range metadata { + k = strings.ToLower(k) switch k { - case "Cache-Control": + case "cache-control": res.Set(k, v.(string)) } - if strings.HasPrefix(k, "X-") { + if strings.HasPrefix(k, "x-") { res.Set(k, v.(string)) } } diff --git a/tests-int/Images.Spec.js b/tests-int/Images.Spec.js index 640752b..8d1bff8 100644 --- a/tests-int/Images.Spec.js +++ b/tests-int/Images.Spec.js @@ -20,7 +20,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.within(9300, 9600); + expect(body.length).to.be.within(9300, 9700); expect(res.headers['x-amz-meta-public-width']).to.eql('150'); expect(res.headers['x-amz-meta-public-height']).to.eql('200'); @@ -54,7 +54,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.eql(30959); + expect(body.length).to.be.within(30950, 33000); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -89,7 +89,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.eql(10228); + expect(body.length).to.be.within(10222, 11000); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -123,7 +123,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.eql(4502); + expect(body.length).to.be.within(4500, 4600); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -158,7 +158,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.eql(2488); + expect(body.length).to.be.within(2400, 2700); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -244,7 +244,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.within(1651, 1654); + expect(body.length).to.be.within(1600, 1654); expect(res.headers['x-amz-meta-public-width']).to.eql('100'); expect(res.headers['x-amz-meta-public-height']).to.eql('100'); @@ -279,7 +279,7 @@ describe('Image processing', function () { const body = res.body; const width = '400'; const height = '100'; - expect(body.length).to.be.eql(1164); + expect(body.length).to.be.within(1160, 1300); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -313,7 +313,7 @@ describe('Image processing', function () { const body = res.body; const width = '400'; const height = '320'; - expect(body.length).to.be.eql(18189); + expect(body.length).to.be.within(18100, 18300); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -347,7 +347,7 @@ describe('Image processing', function () { const body = res.body; const width = '400'; const height = '100'; - expect(body.length).to.be.eql(4505); + expect(body.length).to.be.within(4000, 4600); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); From 5a262646d62814d6629728f06257f63780f5688f Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 9 Jan 2019 22:03:06 +0100 Subject: [PATCH 03/20] add libbrotli-dev for travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 40eebfc..24e0e8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ addons: - libpango1.0-dev - libpoppler-glib-dev - libwebp-dev + - libbrotli-dev before_install: - wget https://github.com/jcupitt/libvips/archive/v$LIBVIPS.zip From 1fc352a4713c631a310dcb30c9048062585d4e99 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 9 Jan 2019 22:17:44 +0100 Subject: [PATCH 04/20] chore(travis): update dist --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 24e0e8d..4dc80c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go sudo: required -dist: trusty +dist: bionic go: - '1.10' From ddf0c31ec7b104c4475a6b7900ea09177325cd95 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 10:49:46 +0100 Subject: [PATCH 05/20] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4dc80c7..31d4a2a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ addons: - libpoppler-glib-dev - libwebp-dev - libbrotli-dev + - gtk-doc-tools before_install: - wget https://github.com/jcupitt/libvips/archive/v$LIBVIPS.zip From 7097fb891182fe5d56a28ea0abebfda5af03600e Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 11:00:25 +0100 Subject: [PATCH 06/20] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 31d4a2a..fc76c06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ addons: - libpoppler-glib-dev - libwebp-dev - libbrotli-dev - - gtk-doc-tools + - gtk-doc before_install: - wget https://github.com/jcupitt/libvips/archive/v$LIBVIPS.zip From 0aaf1173da99e5231c0084d02c821565b3381e28 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:05:47 +0100 Subject: [PATCH 07/20] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fc76c06..a67e297 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ go: - '1.10' env: - - LIBVIPS=8.6.4 + - LIBVIPS=8.7.3 cache: apt From 1950378a4f292ac5dd7dfd0b92a450c4f8d64edd Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:10:06 +0100 Subject: [PATCH 08/20] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a67e297..fb9eaff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ addons: - gtk-doc before_install: - - wget https://github.com/jcupitt/libvips/archive/v$LIBVIPS.zip + - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS - test -f autogen.sh && ./autogen.sh || ./bootstrap.sh From 5c63a351b0839a0ecc4572d91898d17ba948831e Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:16:22 +0100 Subject: [PATCH 09/20] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fb9eaff..ae1ba3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,7 @@ addons: - libbrotli-dev - gtk-doc -before_install: +install: - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS From 1d3720f16513998d6ac7883766f1a828e94d0741 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:18:11 +0100 Subject: [PATCH 10/20] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ae1ba3d..d05648d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ addons: - gtk-doc install: + - sudo apt-get install gtk-doc-tools -y - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS From 660a37f14c87d452ef592d22535a5afb8bd542c6 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:22:42 +0100 Subject: [PATCH 11/20] Update .travis.yml --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d05648d..eefc3a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ go: - '1.10' env: - - LIBVIPS=8.7.3 + - LIBVIPS=8.7.2 cache: apt @@ -30,8 +30,7 @@ addons: - libbrotli-dev - gtk-doc -install: - - sudo apt-get install gtk-doc-tools -y +script: - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS @@ -53,7 +52,6 @@ install: - sudo make install - sudo ldconfig -script: - cd - - sudo curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 && sudo chmod +x /usr/local/bin/dep - dep ensure -vendor-only From 150de8a5de18e711e028fe932042dc5922aa5ee5 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:28:38 +0100 Subject: [PATCH 12/20] Update .travis.yml --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index eefc3a8..c57475d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,7 +30,12 @@ addons: - libbrotli-dev - gtk-doc -script: +addons: + apt: + update: true + +before_install: + - sudo apt-get install libbrotli-dev gtk-doc-tools -y - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS @@ -51,7 +56,8 @@ script: - make - sudo make install - sudo ldconfig - + +script: - cd - - sudo curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 && sudo chmod +x /usr/local/bin/dep - dep ensure -vendor-only From 2a40bbd3f7be62ca9136b7091bfd82e7fbe4d029 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:33:37 +0100 Subject: [PATCH 13/20] Update .travis.yml --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index c57475d..5e21700 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ cache: apt addons: apt: + update: true packages: - gobject-introspection - gtk-doc-tools @@ -29,11 +30,7 @@ addons: - libwebp-dev - libbrotli-dev - gtk-doc - -addons: - apt: - update: true - + before_install: - sudo apt-get install libbrotli-dev gtk-doc-tools -y - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip From 905d84e6d2c7c3c2e90fa914e59f1b96033de37c Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:37:50 +0100 Subject: [PATCH 14/20] Update .travis.yml --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e21700..db689c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,6 @@ go: env: - LIBVIPS=8.7.2 -cache: apt - addons: apt: update: true @@ -30,9 +28,10 @@ addons: - libwebp-dev - libbrotli-dev - gtk-doc + - pkg-config before_install: - - sudo apt-get install libbrotli-dev gtk-doc-tools -y + - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config -y - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS From 656865042e1830251e8022574a1d47eb38726a50 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:41:23 +0100 Subject: [PATCH 15/20] Update .travis.yml --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index db689c5..9d3e90e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,9 +29,11 @@ addons: - libbrotli-dev - gtk-doc - pkg-config + - swig + - automake before_install: - - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config -y + - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config swig automake -y - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS From 018142d1dcb603dac9828a82cbd74193315f2c8c Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:49:21 +0100 Subject: [PATCH 16/20] Update .travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d3e90e..b0ea003 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,9 +31,10 @@ addons: - pkg-config - swig - automake + - bc before_install: - - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config swig automake -y + - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config swig automake bc -y - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS From 73177724e534621ec28c68f08358e88846fe1223 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:53:01 +0100 Subject: [PATCH 17/20] Update .travis.yml --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0ea003..947a40c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,9 +32,13 @@ addons: - swig - automake - bc - + before_install: - - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config swig automake bc -y + - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config swig automake bc gobject-introspection -y + - sudo apt-get install -y automake build-essential gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng-dev \ + - sudo apt-get install -y libwebp-dev libtiff5-dev libgif-dev libexif-dev libxml2-dev libpoppler-glib-dev + - sudo apt-get install -y swig libmagickwand-dev libpango1.0-dev libmatio-dev libopenslide-dev libcfitsio-dev + - sudo apt-get install -y libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev swig - wget https://github.com/libvips/libvips/archive/v$LIBVIPS.zip - unzip v$LIBVIPS - cd libvips-$LIBVIPS From a6fea81e0e114847c8ddff2ee5804d394df9e800 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 22:53:55 +0100 Subject: [PATCH 18/20] feat(docker): update base image and deps --- Dockerfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index afe0ecf..af5af39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM ubuntu:16.04 as builder +FROM ubuntu:18.04 as builder -ENV LIBVIPS_VERSION 8.6.2 -ENV DEP_VERSION v0.4.1 +ENV LIBVIPS_VERSION 8.7.3 +ENV DEP_VERSION v0.5.0 ENV GOLANG_VERSION 1.11 # Installs libvips + required libraries @@ -9,10 +9,10 @@ RUN \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ automake build-essential curl \ - gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng12-dev \ + gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng-dev \ libwebp-dev libtiff5-dev libgif-dev libexif-dev libxml2-dev libpoppler-glib-dev \ swig libmagickwand-dev libpango1.0-dev libmatio-dev libopenslide-dev libcfitsio-dev \ - libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev && \ + libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev swig libbrotli-dev && \ cd /tmp && \ curl -OL https://github.com/libvips/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.gz && \ tar zvxf vips-${LIBVIPS_VERSION}.tar.gz && \ @@ -50,21 +50,23 @@ ADD . /go/src/github.com/aldor007/mort RUN cd /go/src/github.com/aldor007/mort && dep ensure -vendor-only RUN cd /go/src/github.com/aldor007/mort; go build -o /go/mort cmd/mort/mort.go; -FROM ubuntu:16.04 +FROM ubuntu:18.04 RUN \ # Install runtime dependencies apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - libglib2.0-0 libjpeg-turbo8 libpng12-0 libopenexr22 \ - libwebp5 libtiff5 libgif7 libexif12 libxml2 libpoppler-glib8 \ - libmagickwand-6.q16-2 libpango1.0-0 libmatio2 libopenslide0 \ - libgsf-1-114 fftw3 liborc-0.4 librsvg2-2 libcfitsio2 && \ + libglib2.0-0 libjpeg-turbo8 libpng16-16 libopenexr22 \ + libwebp6 libtiff5 libgif7 libexif12 libxml2 libpoppler-glib8 \ + libmagickwand-6.q16hdri-3 libmagickcore-6.q16-3-extra libmagickcore-6.q16hdri-3 \ + libpango1.0-0 libmatio4 libopenslide0 libwebpmux3 \ + libgsf-1-114 fftw3 liborc-0.4 librsvg2-2 libcfitsio-bin libbrotli1 && \ apt-get install -y ca-certificates && \ # Clean up apt-get autoremove -y && \ apt-get autoclean && \ apt-get clean && \ + ldconfig /usr/local/lib && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN mkdir -p /etc/mort/ From 66732d16230cd1cf22acbb0fee1e6c9f3be34cdf Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Thu, 10 Jan 2019 17:55:19 +0100 Subject: [PATCH 19/20] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 947a40c..b67bc9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ addons: before_install: - sudo apt-get install libbrotli-dev gtk-doc-tools pkg-config swig automake bc gobject-introspection -y - - sudo apt-get install -y automake build-essential gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng-dev \ + - sudo apt-get install -y automake build-essential gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng-dev - sudo apt-get install -y libwebp-dev libtiff5-dev libgif-dev libexif-dev libxml2-dev libpoppler-glib-dev - sudo apt-get install -y swig libmagickwand-dev libpango1.0-dev libmatio-dev libopenslide-dev libcfitsio-dev - sudo apt-get install -y libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev swig From 7d30efc843a96eeefe676c98e216ed7835c8a605 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Wed, 16 Jan 2019 21:03:12 +0100 Subject: [PATCH 20/20] feat(hash): add new hash algorithm for key --- Dockerfile | 3 +- Gopkg.lock | 14 +------ go.mod | 3 +- pkg/object/file_object_test.go | 36 ++++++++++++++++ pkg/object/preset.go | 8 ++-- pkg/object/query.go | 1 - pkg/object/testdata/bucket-transform-hash.yml | 33 +++++++++++++++ .../testdata/bucket-transform-hashParent.yml | 33 +++++++++++++++ pkg/object/uri.go | 42 +++++++++++++++---- pkg/processor/plugins/compress_test.go | 4 +- pkg/storage/storage.go | 5 ++- 11 files changed, 151 insertions(+), 31 deletions(-) create mode 100644 pkg/object/testdata/bucket-transform-hash.yml create mode 100644 pkg/object/testdata/bucket-transform-hashParent.yml diff --git a/Dockerfile b/Dockerfile index af5af39..808c9b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:18.04 as builder -ENV LIBVIPS_VERSION 8.7.3 +# ENV LIBVIPS_VERSION 8.7.3 +ENV LIBVIPS_VERSION 8.6.2 ENV DEP_VERSION v0.5.0 ENV GOLANG_VERSION 1.11 diff --git a/Gopkg.lock b/Gopkg.lock index 1560c22..720ca27 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,7 +11,7 @@ [[projects]] branch = "develop" - digest = "1:3da15e2ab879a6d15a175eda02ed868e19ea36fe1c7f1649a2916ab9e94a86c1" + digest = "1:6521bf74452fe2c579b7e99fa555613f33e9a21801eaf29f297e47ad509692e8" name = "github.com/aldor007/stow" packages = [ ".", @@ -333,17 +333,6 @@ revision = "a6029d17e101ac1594adafea9e63fd8c988a8701" version = "v1.0.12" -[[projects]] - branch = "master" - digest = "1:6de28cf34c450cfcfc25d28230311872599ccfcb9c2aba3a8d177a9225582130" - name = "gopkg.in/kothar/brotli-go.v0" - packages = [ - "enc", - "shared", - ] - pruneopts = "UT" - revision = "771231d473d6c4896e97c2b6e3855f1afc0cdada" - [[projects]] branch = "master" digest = "1:863b86a1a9cb2aca2ae27e576c345f7eec79ef99bdd74a4d8fdd966af299f2f2" @@ -388,7 +377,6 @@ "golang.org/x/net/context", "gopkg.in/h2non/bimg.v1", "gopkg.in/h2non/gock.v1", - "gopkg.in/kothar/brotli-go.v0/enc", "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" diff --git a/go.mod b/go.mod index 24252f7..5e93aa4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/Aldor007/mort +module github.com/aldor007/mort require ( github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 @@ -11,6 +11,7 @@ require ( github.com/go-chi/chi v3.3.3+incompatible github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/protobuf v1.2.0 + github.com/google/brotli v1.0.7 github.com/google/readahead v0.0.0-20161222183148-eaceba169032 github.com/graymeta/stow v0.0.0-20181228161447-b469cfb112f8 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af diff --git a/pkg/object/file_object_test.go b/pkg/object/file_object_test.go index f009a61..e7de09a 100644 --- a/pkg/object/file_object_test.go +++ b/pkg/object/file_object_test.go @@ -162,6 +162,42 @@ func TestNewFileObjecWithNestedParent(t *testing.T) { assert.Equal(t, "/parent.jpg", parent.Parent.Key, "parent of parent should have correct path") } +func TestNewFileObjecWithNestedParentHash(t *testing.T) { + mortConfig := config.GetInstance() + mortConfig.Load("testdata/bucket-transform-hash.yml") + obj, err := NewFileObject(pathToURL("/bucket/width/bucket/height/bucket/parent.jpg"), mortConfig) + + assert.Nil(t, err, "Unexpected to have error when parsing path") + + assert.NotNil(t, obj, "obj should be nil") + + assert.True(t, obj.HasParent(), "obj should have parent") + + parent := obj.Parent + + assert.True(t, parent.HasParent(), "parent should have parent") + + assert.Equal(t, "/6ca/hei/height-bucket-parent.jpg-6ca0dabe9909875a", obj.Key) +} + +func TestNewFileObjecWithNestedParentHashParent(t *testing.T) { + mortConfig := config.GetInstance() + mortConfig.Load("testdata/bucket-transform-hashParent.yml") + obj, err := NewFileObject(pathToURL("/bucket/width/bucket/height/bucket/parent.jpg"), mortConfig) + + assert.Nil(t, err, "Unexpected to have error when parsing path") + + assert.NotNil(t, obj, "obj should be nil") + + assert.True(t, obj.HasParent(), "obj should have parent") + + parent := obj.Parent + + assert.True(t, parent.HasParent(), "parent should have parent") + + assert.Equal(t, "/parent.jpg/545d03ded959a5c0", obj.Key) +} + func TestNewFileObjectQueryResize(t *testing.T) { mortConfig := &config.Config{} mortConfig.Load("testdata/bucket-transform-query-parent-storage.yml") diff --git a/pkg/object/preset.go b/pkg/object/preset.go index d6ced83..5ff2416 100644 --- a/pkg/object/preset.go +++ b/pkg/object/preset.go @@ -70,10 +70,10 @@ func decodePreset(_ *url.URL, bucketConfig config.Bucket, obj *FileObject) (stri parent = "/" + parent } - if bucketConfig.Transform.ResultKey == "hash" { - obj.Key = hashKey(obj.Transforms.Hash(), subMatchMap["parent"]) - obj.allowChangeKey = false - } + //if bucketConfig.Transform.ResultKey == "hash" { + // obj.Key = hashKey(obj.Transforms.Hash(), subMatchMap["parent"]) + // obj.allowChangeKey = false + //} return parent, err } diff --git a/pkg/object/query.go b/pkg/object/query.go index 067723d..a7fb85b 100644 --- a/pkg/object/query.go +++ b/pkg/object/query.go @@ -24,7 +24,6 @@ func decodeQuery(url *url.URL, bucketConfig config.Bucket, obj *FileObject) (str parent = "/" + path.Join(trans.ParentBucket, obj.Key) } - obj.Key = hashKey(obj.Transforms.Hash(), parent) return parent, err } diff --git a/pkg/object/testdata/bucket-transform-hash.yml b/pkg/object/testdata/bucket-transform-hash.yml new file mode 100644 index 0000000..53604b6 --- /dev/null +++ b/pkg/object/testdata/bucket-transform-hash.yml @@ -0,0 +1,33 @@ +buckets: + bucket: + transform: + path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" + kind: "presets" + resultKey: "hash" + presets: + blog_small: + quality: 75 + filters: + thumbnail: + width: 100 + height: 100 + mode: outbound + width: + quality: 75 + filters: + thumbnail: + width: 100 + mode: outbound + height: + quality: 75 + filters: + thumbnail: + height: 100 + mode: outbound + storages: + basic: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" + transform: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" diff --git a/pkg/object/testdata/bucket-transform-hashParent.yml b/pkg/object/testdata/bucket-transform-hashParent.yml new file mode 100644 index 0000000..1a4c7da --- /dev/null +++ b/pkg/object/testdata/bucket-transform-hashParent.yml @@ -0,0 +1,33 @@ +buckets: + bucket: + transform: + path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" + kind: "presets" + resultKey: "hashParent" + presets: + blog_small: + quality: 75 + filters: + thumbnail: + width: 100 + height: 100 + mode: outbound + width: + quality: 75 + filters: + thumbnail: + width: 100 + mode: outbound + height: + quality: 75 + filters: + thumbnail: + height: 100 + mode: outbound + storages: + basic: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" + transform: + kind: "local" + rootPath: "/Users/aldor/workspace/mkaciubacom/web" diff --git a/pkg/object/uri.go b/pkg/object/uri.go index a0c5796..d6a2d0e 100644 --- a/pkg/object/uri.go +++ b/pkg/object/uri.go @@ -2,9 +2,9 @@ package object import ( "bytes" + "encoding/hex" "errors" "github.com/aldor007/mort/pkg/config" - "hash" "net/url" "strconv" "strings" @@ -63,8 +63,11 @@ func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error { obj.CheckParent = bucketConfig.Transform.CheckParent if obj.Transforms.NotEmpty { obj.Storage = bucketConfig.Storages.Transform() - if obj.allowChangeKey == true && bucketConfig.Transform.ResultKey == "hash" { - obj.Key = hashKey(obj.Transforms.Hash(), parentObj.key) + switch bucketConfig.Transform.ResultKey { + case "hash": + obj.Key = hashKey(obj) + case "hashParent": + obj.Key = hashKeyParent(obj) } } else { obj.Storage = bucketConfig.Storages.Basic() @@ -80,10 +83,10 @@ func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error { return errors.New("unknown bucket") } -func hashKey(h hash.Hash64, suffix string) string { - hashB := []byte(strconv.FormatUint(uint64(h.Sum64()), 16)) +func hashKey(obj *FileObject) string { + hashB := []byte(strconv.FormatUint(uint64(obj.Transforms.Hash().Sum64()), 16)) buf := bufPool.Get().(*bytes.Buffer) - safePath := strings.Replace(suffix, "/", "-", -1) + safePath := strings.Replace(obj.Parent.key, "/", "-", -1) sliceRange := 3 if l := len(safePath); l < 3 { @@ -99,7 +102,32 @@ func hashKey(h hash.Hash64, suffix string) string { buf.WriteString(safePath) buf.WriteByte('-') buf.Write(hashB) - defer bufPool.Put(buf) + bufPool.Put(buf) + return buf.String() +} + +func hashKeyParent(obj *FileObject) string { + var currObj *FileObject + currObj = obj.Parent + transHash := currObj.Transforms.Hash() + buf := bufPool.Get().(*bytes.Buffer) + for currObj.HasParent() { + buf.Reset() + hSum := currObj.Transforms.Hash().Sum(nil) + buf.WriteString(currObj.Key) + buf.Write(hSum) + transHash.Sum(buf.Bytes()) + currObj = currObj.Parent + } + hashB := transHash.Sum(nil) + safePath := strings.Replace(currObj.key, "/", "-", -1) + + buf.Reset() + buf.WriteByte('/') + buf.WriteString(safePath) + buf.WriteByte('/') + buf.WriteString(hex.EncodeToString(hashB)) + bufPool.Put(buf) return buf.String() } diff --git a/pkg/processor/plugins/compress_test.go b/pkg/processor/plugins/compress_test.go index 60ca42c..ffc60f2 100644 --- a/pkg/processor/plugins/compress_test.go +++ b/pkg/processor/plugins/compress_test.go @@ -4,8 +4,8 @@ import ( "bytes" "compress/gzip" "github.com/aldor007/mort/pkg/response" - "github.com/stretchr/testify/assert" brEnc "github.com/google/brotli/go/cbrotli" + "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" "net/http" "net/http/httptest" @@ -245,7 +245,7 @@ func TestCompressBrotliType(t *testing.T) { var buf bytes.Buffer - br := brEnc.NewWriter(&buf, brEnc.WriterOptions{Quality:4}) + br := brEnc.NewWriter(&buf, brEnc.WriterOptions{Quality: 4}) br.Write(body) br.Close() diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 7fb8d4f..a788061 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -399,7 +399,7 @@ func prepareMetadata(obj *object.FileObject, metaHeaders http.Header) map[string case "s3": keyLower := strings.ToLower(k) if strings.HasPrefix(keyLower, "x-amz-meta") || keyLower == "content-type" { - metadata[strings.Replace(strings.ToLower(k), "x-amz-meta-", "", 1)] = v[0] + metadata[strings.Replace(keyLower, "x-amz-meta-", "", 1)] = v[0] } default: keyLower := strings.ToLower(k) @@ -432,10 +432,11 @@ func parseMetadata(obj *object.FileObject, metadata map[string]interface{}, res switch k { case "cache-control", "content-type": res.Set(k, v.(string)) + default: + res.Set(strings.Join([]string{"x-amz-meta", k}, "-"), v.(string)) } - res.Set(strings.Join([]string{"x-amz-meta", k}, "-"), v.(string)) } }