From 7c9bd9810f6cc94bfcae811e7466e3a8bd482692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 24 Oct 2019 17:36:42 +0200 Subject: [PATCH 1/2] on import: allow specifying namespace prefix, trim '/files/' from files.jsonl (#336) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- cmd/reva/import.go | 13 ++++++++++++- pkg/storage/migrate/metadata.go | 13 +++++++++---- pkg/storage/migrate/shares.go | 4 ++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/reva/import.go b/cmd/reva/import.go index 217c52dfd8..73d4ef9ede 100644 --- a/cmd/reva/import.go +++ b/cmd/reva/import.go @@ -22,6 +22,7 @@ import ( "fmt" "log" "os" + "path" "github.com/cs3org/reva/pkg/storage/migrate" ) @@ -30,6 +31,9 @@ func importCommand() *command { cmd := newCommand("import") cmd.Description = func() string { return "import metadata" } cmd.Usage = func() string { return "Usage: import [-flags] " } + + namespaceFlag := cmd.String("n", "/", "CS3 namespace prefix") + cmd.Action = func() error { if cmd.NArg() < 1 { fmt.Println(cmd.Usage()) @@ -42,7 +46,14 @@ func importCommand() *command { if err != nil { return err } - if err := migrate.ImportShares(ctx, client, exportPath); err != nil { + + ns := path.Join("/", *namespaceFlag) + + if err := migrate.ImportMetadata(ctx, client, exportPath, ns); err != nil { + log.Fatal(err) + return err + } + if err := migrate.ImportShares(ctx, client, exportPath, ns); err != nil { log.Fatal(err) return err } diff --git a/pkg/storage/migrate/metadata.go b/pkg/storage/migrate/metadata.go index a4721d115a..92baa89896 100644 --- a/pkg/storage/migrate/metadata.go +++ b/pkg/storage/migrate/metadata.go @@ -26,6 +26,7 @@ import ( "os" "path" "strconv" + "strings" gatewayv0alphapb "github.com/cs3org/go-cs3apis/cs3/gateway/v0alpha" rpcpb "github.com/cs3org/go-cs3apis/cs3/rpc" @@ -43,7 +44,7 @@ type metaData struct { //ImportMetadata from a files.jsonl file in exportPath. The files must already be present on the storage //Will set etag and mtime -func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string) error { +func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string, ns string) error { filesJSONL, err := os.Open(path.Join(exportPath, "files.jsonl")) if err != nil { @@ -70,7 +71,7 @@ func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceC //TODO permissions? is done via share? but this is owner permissions if len(m) > 0 { - resourcePath := path.Join("/", path.Base(exportPath), fileData.Path) + resourcePath := path.Join(ns, path.Base(exportPath), strings.TrimPrefix(fileData.Path, "/files/")) samReq := &storageproviderv0alphapb.SetArbitraryMetadataRequest{ Ref: &storageproviderv0alphapb.Reference{ Spec: &storageproviderv0alphapb.Reference_Path{Path: resourcePath}, @@ -80,11 +81,15 @@ func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceC }, } samResp, err := client.SetArbitraryMetadata(ctx, samReq) + if err != nil { + log.Fatal(err) + } if samResp.Status.Code == rpcpb.Code_CODE_NOT_FOUND { log.Print("File does not exist on target system, skipping metadata import: " + resourcePath) - } else if err != nil { - log.Fatal(err) + } + if samResp.Status.Code != rpcpb.Code_CODE_OK { + log.Print("Error importing metadata, skipping metadata import: " + resourcePath + ", " + samResp.Status.Message) } } else { log.Print("no etag or mtime for : " + fileData.Path) diff --git a/pkg/storage/migrate/shares.go b/pkg/storage/migrate/shares.go index f7952f51b6..b807e777ee 100644 --- a/pkg/storage/migrate/shares.go +++ b/pkg/storage/migrate/shares.go @@ -49,7 +49,7 @@ type share struct { } //ImportShares from a shares.jsonl file in exportPath. The files must already be present on the storage -func ImportShares(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string) error { +func ImportShares(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string, ns string) error { sharesJSONL, err := os.Open(path.Join(exportPath, "shares.jsonl")) if err != nil { @@ -66,7 +66,7 @@ func ImportShares(ctx context.Context, client gatewayv0alphapb.GatewayServiceCli } //Stat file, skip share creation if it does not exist on the target system - resourcePath := path.Join("/", path.Base(exportPath), shareData.Path) + resourcePath := path.Join(ns, path.Base(exportPath), shareData.Path) statReq := &storageproviderv0alphapb.StatRequest{ Ref: &storageproviderv0alphapb.Reference{ Spec: &storageproviderv0alphapb.Reference_Path{Path: resourcePath}, From 83f1f461c8b87c107373fb1aaa4e08fef7e7031a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 24 Oct 2019 17:57:47 +0200 Subject: [PATCH 2/2] build(deps): bump github.com/aws/aws-sdk-go from 1.25.17 to 1.25.18 (#332) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.25.17 to 1.25.18. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.25.17...v1.25.18) Signed-off-by: dependabot-preview[bot] --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 50493c11ca..67d82cf8ea 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( contrib.go.opencensus.io/exporter/jaeger v0.1.0 contrib.go.opencensus.io/exporter/prometheus v0.1.0 github.com/BurntSushi/toml v0.3.1 - github.com/aws/aws-sdk-go v1.25.17 + github.com/aws/aws-sdk-go v1.25.18 github.com/cheggaaa/pb v1.0.28 github.com/coreos/go-oidc v2.1.0+incompatible github.com/cs3org/go-cs3apis v0.0.0-20191015062230-f4cde4c589eb diff --git a/go.sum b/go.sum index 7c0e45c0f8..d249acc683 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.25.17 h1:qIyijzjkBhVzM2+vnsP53T2zoTHeoRRnfQR2kEkBStA= github.com/aws/aws-sdk-go v1.25.17/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.25.18 h1:fMEkpli4r+FS4xZRqjgjYHP+uKaSwfk2MOcDUpLYwIE= +github.com/aws/aws-sdk-go v1.25.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98=