From 2cb047181db11ff27d65648eb02af42c4349261f Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Thu, 22 Feb 2024 13:04:59 +0400 Subject: [PATCH] object: Limit header length to 16KB Previously, NeoFS used 4MB as object header's length limit. The value originally resulted from the default max gRPC message length. Now header length can be up to 16KB only. To ensure the safety of data uploaded before the restriction was introduced, this limit does not apply to intra-container replication. Refs https://github.com/nspcc-dev/neofs-api/issues/262. Signed-off-by: Leonard Lyubich --- CHANGELOG.md | 1 + cmd/neofs-cli/modules/util/acl.go | 2 +- go.mod | 14 +++++++------- go.sum | 28 +++++++++++++-------------- pkg/services/object/acl/v2/service.go | 14 +++++++++++++- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 232d2cccf7c..8c57d4be9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Changelog for NeoFS Node - Storage nodes no longer reject GET w/ TTL=1 requests to read complex objects (#2447) ### Changed +- Storage nodes no longer accept objects with header larger than 16KB (#2749) ### Removed diff --git a/cmd/neofs-cli/modules/util/acl.go b/cmd/neofs-cli/modules/util/acl.go index 12949042b3e..bea8d2693af 100644 --- a/cmd/neofs-cli/modules/util/acl.go +++ b/cmd/neofs-cli/modules/util/acl.go @@ -134,7 +134,7 @@ func eaclFiltersToString(fs []eacl.Filter) string { } _, _ = tw.Write([]byte(f.Key())) - + //nolint:exhaustive switch f.Matcher() { case eacl.MatchStringEqual: _, _ = tw.Write([]byte("\t==\t")) diff --git a/go.mod b/go.mod index b878d8c0f49..0fe525c42e7 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/chzyer/readline v1.5.1 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/go-github/v39 v39.2.0 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.2 + github.com/klauspost/compress v1.17.6 github.com/mitchellh/go-homedir v1.1.0 github.com/mr-tron/base58 v1.2.0 github.com/multiformats/go-multiaddr v0.12.0 @@ -17,9 +17,9 @@ require ( github.com/nspcc-dev/hrw/v2 v2.0.0 github.com/nspcc-dev/locode-db v0.5.0 github.com/nspcc-dev/neo-go v0.105.1 - github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 + github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe github.com/nspcc-dev/neofs-contract v0.19.1 - github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240130073207-03ed6db7e1cd + github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240227143352-846741cb85a3 github.com/nspcc-dev/tzhash v1.7.1 github.com/olekukonko/tablewriter v0.0.5 github.com/panjf2000/ants/v2 v2.8.2 @@ -33,8 +33,8 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.3.0 - golang.org/x/sys v0.15.0 - golang.org/x/term v0.15.0 + golang.org/x/sys v0.17.0 + golang.org/x/term v0.17.0 google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 gopkg.in/yaml.v3 v3.0.1 @@ -97,7 +97,7 @@ require ( github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.5 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.19.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect diff --git a/go.sum b/go.sum index 6f213fdd73e..4fc9995fc77 100644 --- a/go.sum +++ b/go.sum @@ -159,8 +159,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -186,8 +186,8 @@ github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svo github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -251,14 +251,14 @@ github.com/nspcc-dev/neo-go v0.105.1 h1:r0b2yIwLBi+ARBKU94gHL9oTFEB/XMJ0YlS2HN9Q github.com/nspcc-dev/neo-go v0.105.1/go.mod h1:GNh0cRALV/cuj+/xg2ZHDsrFbqcInqG7jjhqsLEnlNc= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231127165613-b35f351f0ba0 h1:N+dMIBmteXjJpkH6UZ7HmNftuFxkqszfGLbhsEctnv0= github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20231127165613-b35f351f0ba0/go.mod h1:J/Mk6+nKeKSW4wygkZQFLQ6SkLOSGX5Ga0RuuuktEag= -github.com/nspcc-dev/neofs-api-go/v2 v2.14.0 h1:jhuN8Ldqz7WApvUJRFY0bjRXE1R3iCkboMX5QVZhHVk= -github.com/nspcc-dev/neofs-api-go/v2 v2.14.0/go.mod h1:DRIr0Ic1s+6QgdqmNFNLIqMqd7lNMJfYwkczlm1hDtM= +github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe h1:Hoq88+PWS6tNnX4Y0jxE0C8wvxPI8UlVnCs2ZJDEy4Y= +github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240213170208-cfca09b5acbe/go.mod h1:eaffSBIGhXUIMYvRBGXmlgQRLyyCWlzOft9jGYlqwrw= github.com/nspcc-dev/neofs-contract v0.19.1 h1:U1Uh+MlzfkalO0kRJ2pADZyHrmAOroC6KLFjdWnTNR0= github.com/nspcc-dev/neofs-contract v0.19.1/go.mod h1:ZOGouuwuHpgvYkx/LCGufGncIzEUhYEO18LL4cWEbyw= github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4= github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240130073207-03ed6db7e1cd h1:kRIn6i7BTa55ae4cH+UcqRfH//XC20mSC4E9WcWxkmM= -github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240130073207-03ed6db7e1cd/go.mod h1:2PKUuH7kQaAmQ/USBgmiD/k08ssnSvayor6JAFhrC1c= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240227143352-846741cb85a3 h1:ADjuSMRdYW4FVtOAaFUi+imycOXu1Ej7yIVK37grDtY= +github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20240227143352-846741cb85a3/go.mod h1:GED7qcgut188OQYY7lak/cGN8DHtjqQbrXdSt/82QNE= github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE= github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso= github.com/nspcc-dev/tzhash v1.7.1 h1:6zmexLqdTF/ssbUAh7XJS7RxgKWaw28kdNpE/4UFdEU= @@ -365,8 +365,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -508,11 +508,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/services/object/acl/v2/service.go b/pkg/services/object/acl/v2/service.go index 89f0b9f434a..33b3ee1c610 100644 --- a/pkg/services/object/acl/v2/service.go +++ b/pkg/services/object/acl/v2/service.go @@ -12,6 +12,7 @@ import ( apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" "github.com/nspcc-dev/neofs-sdk-go/container/acl" cid "github.com/nspcc-dev/neofs-sdk-go/container/id" + objectsdk "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" sessionSDK "github.com/nspcc-dev/neofs-sdk-go/session" "github.com/nspcc-dev/neofs-sdk-go/user" @@ -524,15 +525,26 @@ func (p putStreamBasicChecker) Send(request *objectV2.PutRequest) error { return err } + replication := reqInfo.requestRole == acl.RoleContainer && request.GetMetaHeader().GetTTL() == 1 if tombstone { // the only exception when writing tombstone should not be treated as deletion // is intra-container replication: container nodes must be able to replicate // such objects while deleting is prohibited - if reqInfo.requestRole == acl.RoleContainer && request.GetMetaHeader().GetTTL() == 1 { + if replication { reqInfo.operation = acl.OpObjectPut } } + if !replication { + // header length is unchecked for replication because introducing a restriction + // should not prevent the replication of objects created before. + // See also https://github.com/nspcc-dev/neofs-api/issues/293 + hdrLen := part.GetHeader().StableSize() + if hdrLen > objectsdk.MaxHeaderLen { + return fmt.Errorf("object header length exceeds the limit: %d>%d", hdrLen, objectsdk.MaxHeaderLen) + } + } + reqInfo.obj = obj if !p.source.checker.CheckBasicACL(reqInfo) || !p.source.checker.StickyBitCheck(reqInfo, idOwner) {