Merge remote-tracking branch 'origin/master' into ci-release

cs3org · Oct 24, 2019 · 639970f · 639970f
2 parents 1c80fca + 83f1f46
commit 639970f
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 8 deletions.
diff --git a/cmd/reva/import.go b/cmd/reva/import.go
@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 
 	"github.com/cs3org/reva/pkg/storage/migrate"
 )
@@ -30,6 +31,9 @@ func importCommand() *command {
 	cmd := newCommand("import")
 	cmd.Description = func() string { return "import metadata" }
 	cmd.Usage = func() string { return "Usage: import [-flags] <user export folder>" }
+
+	namespaceFlag := cmd.String("n", "/", "CS3 namespace prefix")
+
 	cmd.Action = func() error {
 		if cmd.NArg() < 1 {
 			fmt.Println(cmd.Usage())
@@ -42,7 +46,14 @@ func importCommand() *command {
 		if err != nil {
 			return err
 		}
-		if err := migrate.ImportShares(ctx, client, exportPath); err != nil {
+
+		ns := path.Join("/", *namespaceFlag)
+
+		if err := migrate.ImportMetadata(ctx, client, exportPath, ns); err != nil {
+			log.Fatal(err)
+			return err
+		}
+		if err := migrate.ImportShares(ctx, client, exportPath, ns); err != nil {
 			log.Fatal(err)
 			return err
 		}

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ require (
 	contrib.go.opencensus.io/exporter/jaeger v0.1.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	github.com/BurntSushi/toml v0.3.1
-	github.com/aws/aws-sdk-go v1.25.17
+	github.com/aws/aws-sdk-go v1.25.18
 	github.com/cheggaaa/pb v1.0.28
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/cs3org/go-cs3apis v0.0.0-20191015062230-f4cde4c589eb

diff --git a/go.sum b/go.sum
@@ -20,6 +20,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.25.17 h1:qIyijzjkBhVzM2+vnsP53T2zoTHeoRRnfQR2kEkBStA=
 github.com/aws/aws-sdk-go v1.25.17/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.25.18 h1:fMEkpli4r+FS4xZRqjgjYHP+uKaSwfk2MOcDUpLYwIE=
+github.com/aws/aws-sdk-go v1.25.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98=

diff --git a/pkg/storage/migrate/metadata.go b/pkg/storage/migrate/metadata.go
@@ -26,6 +26,7 @@ import (
 	"os"
 	"path"
 	"strconv"
+	"strings"
 
 	gatewayv0alphapb "github.com/cs3org/go-cs3apis/cs3/gateway/v0alpha"
 	rpcpb "github.com/cs3org/go-cs3apis/cs3/rpc"
@@ -43,7 +44,7 @@ type metaData struct {
 
 //ImportMetadata from a files.jsonl file in exportPath. The files must already be present on the storage
 //Will set etag and mtime
-func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string) error {
+func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string, ns string) error {
 
 	filesJSONL, err := os.Open(path.Join(exportPath, "files.jsonl"))
 	if err != nil {
@@ -70,7 +71,7 @@ func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceC
 		//TODO permissions? is done via share? but this is owner permissions
 
 		if len(m) > 0 {
-			resourcePath := path.Join("/", path.Base(exportPath), fileData.Path)
+			resourcePath := path.Join(ns, path.Base(exportPath), strings.TrimPrefix(fileData.Path, "/files/"))
 			samReq := &storageproviderv0alphapb.SetArbitraryMetadataRequest{
 				Ref: &storageproviderv0alphapb.Reference{
 					Spec: &storageproviderv0alphapb.Reference_Path{Path: resourcePath},
@@ -80,11 +81,15 @@ func ImportMetadata(ctx context.Context, client gatewayv0alphapb.GatewayServiceC
 				},
 			}
 			samResp, err := client.SetArbitraryMetadata(ctx, samReq)
+			if err != nil {
+				log.Fatal(err)
+			}
 
 			if samResp.Status.Code == rpcpb.Code_CODE_NOT_FOUND {
 				log.Print("File does not exist on target system, skipping metadata import: " + resourcePath)
-			} else if err != nil {
-				log.Fatal(err)
+			}
+			if samResp.Status.Code != rpcpb.Code_CODE_OK {
+				log.Print("Error importing metadata, skipping metadata import: " + resourcePath + ", " + samResp.Status.Message)
 			}
 		} else {
 			log.Print("no etag or mtime for : " + fileData.Path)

diff --git a/pkg/storage/migrate/shares.go b/pkg/storage/migrate/shares.go
@@ -49,7 +49,7 @@ type share struct {
 }
 
 //ImportShares from a shares.jsonl file in exportPath. The files must already be present on the storage
-func ImportShares(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string) error {
+func ImportShares(ctx context.Context, client gatewayv0alphapb.GatewayServiceClient, exportPath string, ns string) error {
 
 	sharesJSONL, err := os.Open(path.Join(exportPath, "shares.jsonl"))
 	if err != nil {
@@ -66,7 +66,7 @@ func ImportShares(ctx context.Context, client gatewayv0alphapb.GatewayServiceCli
 		}
 
 		//Stat file, skip share creation if it does not exist on the target system
-		resourcePath := path.Join("/", path.Base(exportPath), shareData.Path)
+		resourcePath := path.Join(ns, path.Base(exportPath), shareData.Path)
 		statReq := &storageproviderv0alphapb.StatRequest{
 			Ref: &storageproviderv0alphapb.Reference{
 				Spec: &storageproviderv0alphapb.Reference_Path{Path: resourcePath},