diff --git a/.gitignore b/.gitignore index f1c181ec9c..c0f3915ecc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,11 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out + +# Ignore builds +cmd/revad/revad +cmd/reva/reva + + +# For Mac OS +.DS_Store diff --git a/README.md b/README.md index eb00ac2deb..99526ce8e8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,5 @@ -# reva -Cloud Storage Sync and Share Interoperability Platform +# REVA + +Cloud Storage Sync & Share Interoperability Platform + + diff --git a/cmd/reva/app-provider-get-iframe.go b/cmd/reva/app-provider-get-iframe.go new file mode 100644 index 0000000000..a15d9cf2d9 --- /dev/null +++ b/cmd/reva/app-provider-get-iframe.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "fmt" + "os" + + appproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/appprovider/v0alpha" + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" +) + +func appProviderGetIFrameCommand() *command { + cmd := newCommand("app-provider-get-iframe") + cmd.Description = func() string { + return "find iframe UI provider for filename" + } + cmd.Action = func() error { + if cmd.NArg() < 3 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + appProvider := cmd.Args()[0] + fn := cmd.Args()[1] + token := cmd.Args()[2] + req := &appproviderv0alphapb.GetIFrameRequest{ + Filename: fn, + AccessToken: token, + } + + client, err := getAppProviderClient(appProvider) + if err != nil { + return err + } + ctx := context.Background() + res, err := client.GetIFrame(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + fmt.Printf("Load in your browser the following iframe to edit the resource: %s", res.IframeLocation) + return nil + } + return cmd +} diff --git a/cmd/reva/app-registry-find.go b/cmd/reva/app-registry-find.go new file mode 100644 index 0000000000..1d1dd8a099 --- /dev/null +++ b/cmd/reva/app-registry-find.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "mime" + "os" + "path" + + appregistryv0alphapb "github.com/cernbox/go-cs3apis/cs3/appregistry/v0alpha" + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" +) + +func appRegistryFindCommand() *command { + cmd := newCommand("app-registry-find") + cmd.Description = func() string { + return "find applicaton provider for file extension or mimetype" + } + cmd.Action = func() error { + if cmd.NArg() == 0 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + fn := cmd.Args()[0] + ext := path.Ext(fn) + mime := mime.TypeByExtension(ext) + req := &appregistryv0alphapb.FindRequest{ + FilenameExtension: ext, + FilenameMimetype: mime, + } + + client, err := getAppRegistryClient() + if err != nil { + return err + } + ctx := context.Background() + res, err := client.Find(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + fmt.Printf("application provider can be found at %s\n", res.AppProviderInfo.Location) + return nil + } + return cmd +} diff --git a/cmd/reva/broker-find.go b/cmd/reva/broker-find.go new file mode 100644 index 0000000000..31fc8a267e --- /dev/null +++ b/cmd/reva/broker-find.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "fmt" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storagebrokerv0alphapb "github.com/cernbox/go-cs3apis/cs3/storagebroker/v0alpha" +) + +func brokerFindCommand() *command { + cmd := newCommand("broker-find") + cmd.Description = func() string { + return "find storage provider for path" + } + cmd.Action = func() error { + fn := "/" + if cmd.NArg() >= 1 { + fn = cmd.Args()[0] + } + + req := &storagebrokerv0alphapb.FindRequest{ + Filename: fn, + } + client, err := getStorageBrokerClient() + if err != nil { + return err + } + ctx := context.Background() + res, err := client.Find(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + fmt.Printf("resource can be found at %s\n", res.ProviderInfo.Location) + return nil + } + return cmd +} diff --git a/cmd/reva/command.go b/cmd/reva/command.go new file mode 100644 index 0000000000..9577d0cd92 --- /dev/null +++ b/cmd/reva/command.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "fmt" +) + +// command is the representation to create commands +type command struct { + *flag.FlagSet + Name string + Action func() error + Usage func() string + Description func() string +} + +// newCommand creates a new command +func newCommand(name string) *command { + fs := flag.NewFlagSet(name, flag.ExitOnError) + cmd := &command{ + Name: name, + Usage: func() string { + return fmt.Sprintf("Usage: %s", name) + }, + Action: func() error { + fmt.Println("Hello REVA") + return nil + }, + Description: func() string { + return "TODO description" + }, + FlagSet: fs, + } + return cmd +} diff --git a/cmd/reva/common.go b/cmd/reva/common.go new file mode 100644 index 0000000000..465acc7f42 --- /dev/null +++ b/cmd/reva/common.go @@ -0,0 +1,83 @@ +package main + +import ( + "bufio" + "encoding/json" + "io/ioutil" + gouser "os/user" + "path" + "strings" + + "golang.org/x/crypto/ssh/terminal" +) + +func getConfigFile() string { + user, err := gouser.Current() + if err != nil { + panic(err) + } + + return path.Join(user.HomeDir, ".reva.config") +} + +func getTokenFile() string { + user, err := gouser.Current() + if err != nil { + panic(err) + } + + return path.Join(user.HomeDir, ".reva-token") +} + +func writeToken(token string) { + ioutil.WriteFile(getTokenFile(), []byte(token), 0600) +} + +func readToken() (string, error) { + data, err := ioutil.ReadFile(getTokenFile()) + if err != nil { + return "", err + } + return string(data), nil +} + +func readConfig() (*config, error) { + data, err := ioutil.ReadFile(getConfigFile()) + if err != nil { + return nil, err + } + + c := &config{} + if err := json.Unmarshal(data, c); err != nil { + return nil, err + } + + return c, nil +} + +func writeConfig(c *config) error { + data, err := json.Marshal(c) + if err != nil { + return err + } + return ioutil.WriteFile(getConfigFile(), data, 0600) +} + +type config struct { + Host string `json:"host"` +} + +func read(r *bufio.Reader) (string, error) { + text, err := r.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimSpace(text), nil +} +func readPassword(fd int) (string, error) { + bytePassword, err := terminal.ReadPassword(fd) + if err != nil { + return "", err + } + return strings.TrimSpace(string(bytePassword)), nil +} diff --git a/cmd/reva/configure.go b/cmd/reva/configure.go new file mode 100644 index 0000000000..2a4a4eaac2 --- /dev/null +++ b/cmd/reva/configure.go @@ -0,0 +1,26 @@ +package main + +import ( + "bufio" + "fmt" + "os" +) + +var configureCommand = func() *command { + cmd := newCommand("configure") + cmd.Description = func() string { return "configure the reva client" } + cmd.Action = func() error { + reader := bufio.NewReader(os.Stdin) + fmt.Print("host: ") + text, err := read(reader) + if err != nil { + return err + } + + c := &config{Host: text} + writeConfig(c) + fmt.Println("config saved in ", getConfigFile()) + return nil + } + return cmd +} diff --git a/cmd/reva/download.go b/cmd/reva/download.go new file mode 100644 index 0000000000..d1e51787cd --- /dev/null +++ b/cmd/reva/download.go @@ -0,0 +1,91 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" + "github.com/cheggaaa/pb" +) + +func downloadCommand() *command { + cmd := newCommand("download") + cmd.Description = func() string { return "download a remote file into the local filesystem" } + cmd.Action = func() error { + fn := "/" + if cmd.NArg() < 3 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + provider := cmd.Args()[0] + fn = cmd.Args()[1] + target := cmd.Args()[2] + + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req1 := &storageproviderv0alphapb.StatRequest{Filename: fn} + ctx := context.Background() + res1, err := client.Stat(ctx, req1) + if err != nil { + return err + } + if res1.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res1.Status) + } + + md := res1.Metadata + + fd, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + + req2 := &storageproviderv0alphapb.ReadRequest{Filename: fn} + ctx = context.Background() + stream, err := client.Read(ctx, req2) + if err != nil { + return err + } + + bar := pb.New(int(md.Size)).SetUnits(pb.U_BYTES) + bar.Start() + var reader io.Reader + for { + res, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + dc := res.DataChunk + + if dc != nil { + if dc.Length > 0 { + reader = bytes.NewReader(dc.Data) + reader = bar.NewProxyReader(reader) + + _, err := io.CopyN(fd, reader, int64(dc.Length)) + if err != nil { + return err + } + } + } + } + bar.Finish() + return nil + + } + return cmd +} diff --git a/cmd/reva/grpc.go b/cmd/reva/grpc.go new file mode 100644 index 0000000000..a2e3e4006c --- /dev/null +++ b/cmd/reva/grpc.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + + "github.com/pkg/errors" + + appproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/appprovider/v0alpha" + appregistryv0alphapb "github.com/cernbox/go-cs3apis/cs3/appregistry/v0alpha" + authv0alphapb "github.com/cernbox/go-cs3apis/cs3/auth/v0alpha" + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storagebrokerv0alphapb "github.com/cernbox/go-cs3apis/cs3/storagebroker/v0alpha" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" + + "google.golang.org/grpc" +) + +func getAppProviderClient(host string) (appproviderv0alphapb.AppProviderServiceClient, error) { + conn, err := getConnToHost(host) + if err != nil { + return nil, err + } + return appproviderv0alphapb.NewAppProviderServiceClient(conn), nil +} +func getStorageBrokerClient() (storagebrokerv0alphapb.StorageBrokerServiceClient, error) { + conn, err := getConn() + if err != nil { + return nil, err + } + return storagebrokerv0alphapb.NewStorageBrokerServiceClient(conn), nil +} + +func getAppRegistryClient() (appregistryv0alphapb.AppRegistryServiceClient, error) { + conn, err := getConn() + if err != nil { + return nil, err + } + return appregistryv0alphapb.NewAppRegistryServiceClient(conn), nil +} + +func getStorageProviderClient(host string) (storageproviderv0alphapb.StorageProviderServiceClient, error) { + conn, err := getConnToHost(host) + if err != nil { + return nil, err + } + return storageproviderv0alphapb.NewStorageProviderServiceClient(conn), nil +} + +func getAuthClient() (authv0alphapb.AuthServiceClient, error) { + conn, err := getConn() + if err != nil { + return nil, err + } + return authv0alphapb.NewAuthServiceClient(conn), nil +} + +func getConn() (*grpc.ClientConn, error) { + return grpc.Dial(conf.Host, grpc.WithInsecure()) +} + +func getConnToHost(host string) (*grpc.ClientConn, error) { + return grpc.Dial(host, grpc.WithInsecure()) +} + +func formatError(status *rpcpb.Status) error { + switch status.Code { + case rpcpb.Code_CODE_NOT_FOUND: + return errors.New("error: not found") + + default: + return errors.New(fmt.Sprintf("apierror: code=%v msg=%s", status.Code, status.Message)) + } +} diff --git a/cmd/reva/login.go b/cmd/reva/login.go new file mode 100644 index 0000000000..f34f35ce4a --- /dev/null +++ b/cmd/reva/login.go @@ -0,0 +1,64 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "os" + + authv0alphapb "github.com/cernbox/go-cs3apis/cs3/auth/v0alpha" + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" +) + +var loginCommand = func() *command { + cmd := newCommand("login") + cmd.Description = func() string { return "login into the reva server" } + cmd.Action = func() error { + var username, password string + if cmd.NArg() >= 2 { + username = cmd.Args()[0] + password = cmd.Args()[1] + } else { + reader := bufio.NewReader(os.Stdin) + fmt.Print("username: ") + usernameInput, err := read(reader) + if err != nil { + return err + } + + fmt.Print("password: ") + passwordInput, err := readPassword(0) + if err != nil { + return err + } + + username = usernameInput + password = passwordInput + } + + client, err := getAuthClient() + if err != nil { + return err + } + + req := &authv0alphapb.GenerateAccessTokenRequest{ + Username: username, + Password: password, + } + + ctx := context.Background() + res, err := client.GenerateAccessToken(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + writeToken(res.AccessToken) + fmt.Println("OK") + return nil + } + return cmd +} diff --git a/cmd/reva/ls.go b/cmd/reva/ls.go new file mode 100644 index 0000000000..3ece66462f --- /dev/null +++ b/cmd/reva/ls.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" +) + +func lsCommand() *command { + cmd := newCommand("ls") + cmd.Description = func() string { return "list a folder contents" } + longFlag := cmd.Bool("l", false, "long listing") + cmd.Action = func() error { + if cmd.NArg() < 2 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + provider := cmd.Args()[0] + fn := cmd.Args()[1] + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req := &storageproviderv0alphapb.ListRequest{ + Filename: fn, + } + + ctx := context.Background() + stream, err := client.List(ctx, req) + if err != nil { + return err + } + + mds := []*storageproviderv0alphapb.Metadata{} + for { + res, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return err + } + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + mds = append(mds, res.Metadata) + } + + for _, md := range mds { + if *longFlag { + fmt.Printf("%+v %d %d %s\n", md.Permissions, md.Mtime, md.Size, md.Filename) + } else { + fmt.Println(md.Filename) + } + } + return nil + } + return cmd +} diff --git a/cmd/reva/main.go b/cmd/reva/main.go new file mode 100644 index 0000000000..917e1394d3 --- /dev/null +++ b/cmd/reva/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "os" + "strings" +) + +var ( + conf *config +) + +func main() { + + cmds := []*command{ + configureCommand(), + loginCommand(), + whoamiCommand(), + lsCommand(), + statCommand(), + uploadCommand(), + downloadCommand(), + rmCommand(), + moveCommand(), + mkdirCommand(), + brokerFindCommand(), + appRegistryFindCommand(), + appProviderGetIFrameCommand(), + } + + mainUsage := createMainUsage(cmds) + + // Verify that a subcommand has been provided + // os.Arg[0] is the main command + // os.Arg[1] will be the subcommand + if len(os.Args) < 2 { + fmt.Println(mainUsage) + os.Exit(1) + } + + // Verify a configuration file exists. + // If if does not, create one + c, err := readConfig() + if err != nil && os.Args[1] != "configure" { + fmt.Println("reva is not initialized, run \"reva configure\"") + os.Exit(1) + } else { + if os.Args[1] != "configure" { + conf = c + } + } + + // Run command + action := os.Args[1] + for _, v := range cmds { + if v.Name == action { + v.Parse(os.Args[2:]) + err := v.Action() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + os.Exit(0) + } + } + + fmt.Println(mainUsage) + os.Exit(1) +} + +func createMainUsage(cmds []*command) string { + n := 0 + for _, cmd := range cmds { + l := len(cmd.Name) + if l > n { + n = l + } + } + + usage := "Command line interface to REVA\n\n" + for _, cmd := range cmds { + usage += fmt.Sprintf("%s%s%s\n", cmd.Name, strings.Repeat(" ", 4+(n-len(cmd.Name))), cmd.Description()) + } + usage += "\nAuthors: hugo.gonzalez.labrador@cern.ch" + usage += "\nCopyright: CERN IT Storage Group" + return usage +} diff --git a/cmd/reva/mkdir.go b/cmd/reva/mkdir.go new file mode 100644 index 0000000000..1aaba35089 --- /dev/null +++ b/cmd/reva/mkdir.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "fmt" + "os" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" +) + +func mkdirCommand() *command { + cmd := newCommand("mkdir") + cmd.Description = func() string { return "creates a folder" } + cmd.Action = func() error { + if cmd.NArg() < 2 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + fn := cmd.Args()[0] + provider := cmd.Args()[1] + + ctx := context.Background() + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req := &storageproviderv0alphapb.CreateDirectoryRequest{Filename: fn} + res, err := client.CreateDirectory(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + return nil + } + return cmd +} diff --git a/cmd/reva/mv.go b/cmd/reva/mv.go new file mode 100644 index 0000000000..cc5e34caf9 --- /dev/null +++ b/cmd/reva/mv.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "fmt" + "os" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" +) + +func moveCommand() *command { + cmd := newCommand("mv") + cmd.Description = func() string { return "moves/rename a file/folder" } + cmd.Action = func() error { + if cmd.NArg() < 3 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + provider := cmd.Args()[0] + src := cmd.Args()[1] + dst := cmd.Args()[2] + + ctx := context.Background() + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req := &storageproviderv0alphapb.MoveRequest{SourceFilename: src, TargetFilename: dst} + res, err := client.Move(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + return nil + } + return cmd +} diff --git a/cmd/reva/rm.go b/cmd/reva/rm.go new file mode 100644 index 0000000000..5b02238c54 --- /dev/null +++ b/cmd/reva/rm.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "fmt" + "os" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" +) + +func rmCommand() *command { + cmd := newCommand("rm") + cmd.Description = func() string { return "removes a file or folder" } + cmd.Action = func() error { + if cmd.NArg() < 2 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + provider := cmd.Args()[0] + fn := cmd.Args()[1] + ctx := context.Background() + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req := &storageproviderv0alphapb.DeleteRequest{Filename: fn} + res, err := client.Delete(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + return nil + } + return cmd +} diff --git a/cmd/reva/stat.go b/cmd/reva/stat.go new file mode 100644 index 0000000000..1981add916 --- /dev/null +++ b/cmd/reva/stat.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "fmt" + "os" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" +) + +func statCommand() *command { + cmd := newCommand("stat") + cmd.Description = func() string { return "get the metadata for a file or folder" } + cmd.Action = func() error { + if cmd.NArg() < 2 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + provider := cmd.Args()[0] + fn := cmd.Args()[1] + ctx := context.Background() + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req := &storageproviderv0alphapb.StatRequest{Filename: fn} + res, err := client.Stat(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + fmt.Println(res.Metadata) + return nil + } + return cmd +} diff --git a/cmd/reva/upload.go b/cmd/reva/upload.go new file mode 100644 index 0000000000..354a1f6268 --- /dev/null +++ b/cmd/reva/upload.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "crypto/md5" + "fmt" + "io" + "os" + + "github.com/cheggaaa/pb" + + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" +) + +func uploadCommand() *command { + cmd := newCommand("upload") + cmd.Description = func() string { return "upload a local file to the remote server" } + cmd.Action = func() error { + if cmd.NArg() < 3 { + fmt.Println(cmd.Usage()) + os.Exit(1) + } + + provider := cmd.Args()[0] + fn := cmd.Args()[1] + target := cmd.Args()[2] + + fd, err := os.Open(fn) + if err != nil { + return err + } + md, err := fd.Stat() + if err != nil { + return err + } + defer fd.Close() + + client, err := getStorageProviderClient(provider) + if err != nil { + return err + } + + req1 := &storageproviderv0alphapb.StartWriteSessionRequest{} + ctx := context.Background() + res1, err := client.StartWriteSession(ctx, req1) + if err != nil { + return err + } + + if res1.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res1.Status) + } + + sessID := res1.SessionId + fmt.Println("Write session ID: ", sessID) + + ctx = context.Background() + stream, err := client.Write(ctx) + if err != nil { + return err + } + + bar := pb.New(int(md.Size())).SetUnits(pb.U_BYTES) + xs := md5.New() + nchunks, offset := 0, 0 + // TODO(labkode): change buffer size in configuration + bufferSize := 1024 * 1024 * 3 + buffer := make([]byte, bufferSize) + writer := io.MultiWriter(xs, bar) + bar.Start() + for { + n, err := fd.Read(buffer) + if n > 0 { + writer.Write(buffer[:n]) + req := &storageproviderv0alphapb.WriteRequest{ + Data: buffer[:n], + Length: uint64(n), + Offset: uint64(offset), + SessionId: sessID, + } + if err := stream.Send(req); err != nil { + return err + } + nchunks++ + offset += n + } + if err == io.EOF { + break + } + if err != nil { + return err + } + } + + bar.Finish() + res2, err := stream.CloseAndRecv() + if err != nil { + return err + } + + if res2.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res2.Status) + } + + //wb := res2.WrittenBytes + + //fmt.Println("Written bytes: ", wb, " NumChunks: ", nchunks, " MD5: ", fmt.Sprintf("%x", xs.Sum(nil))) + + fmt.Println("Closing write session ...") + req3 := &storageproviderv0alphapb.FinishWriteSessionRequest{ + Filename: target, + SessionId: sessID, + Checksum: fmt.Sprintf("md5:%x", xs.Sum(nil)), + } + ctx = context.Background() + res3, err := client.FinishWriteSession(ctx, req3) + if err != nil { + return err + } + + if res3.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res3.Status) + } + + fmt.Println("Upload succeed") + return nil + } + return cmd +} diff --git a/cmd/reva/whoami.go b/cmd/reva/whoami.go new file mode 100644 index 0000000000..975125e7e4 --- /dev/null +++ b/cmd/reva/whoami.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "fmt" + "os" + + authv0alphapb "github.com/cernbox/go-cs3apis/cs3/auth/v0alpha" + rpcpb "github.com/cernbox/go-cs3apis/cs3/rpc" +) + +func whoamiCommand() *command { + cmd := newCommand("whoami") + cmd.Description = func() string { return "tells who you are" } + tokenFlag := cmd.String("token", "", "access token to use") + + cmd.Action = func() error { + if cmd.NArg() != 0 { + cmd.PrintDefaults() + os.Exit(1) + } + var token string + if *tokenFlag != "" { + token = *tokenFlag + } else { + // read token from file + t, err := readToken() + if err != nil { + fmt.Println("the token file cannot be readed from file ", getTokenFile()) + fmt.Println("make sure you have login before with \"reva login\"") + return err + } + token = t + } + + client, err := getAuthClient() + if err != nil { + return err + } + + req := &authv0alphapb.WhoAmIRequest{AccessToken: token} + + ctx := context.Background() + res, err := client.WhoAmI(ctx, req) + if err != nil { + return err + } + + if res.Status.Code != rpcpb.Code_CODE_OK { + return formatError(res.Status) + } + + me := res.User + fmt.Printf("username: %s\ndisplay_name: %s\nmail: %s\ngroups: %v\n", me.Username, me.DisplayName, me.Mail, me.Groups) + return nil + } + return cmd +} diff --git a/cmd/revad/config/config.go b/cmd/revad/config/config.go new file mode 100644 index 0000000000..9e21c12196 --- /dev/null +++ b/cmd/revad/config/config.go @@ -0,0 +1,27 @@ +package config + +import ( + "github.com/spf13/viper" +) + +var v *viper.Viper + +func init() { + v = viper.New() +} + +func SetFile(fn string) { + v.SetConfigFile(fn) +} + +func Read() error { + return v.ReadInConfig() +} + +func Get(key string) map[string]interface{} { + return v.GetStringMap(key) +} + +func Dump() map[string]interface{} { + return v.AllSettings() +} diff --git a/cmd/revad/grace/grace.go b/cmd/revad/grace/grace.go new file mode 100644 index 0000000000..8088e6f9c2 --- /dev/null +++ b/cmd/revad/grace/grace.go @@ -0,0 +1,287 @@ +package grace + +import ( + "context" + "fmt" + "io/ioutil" + "net" + "os" + "os/signal" + "path/filepath" + "strconv" + "syscall" + "time" + + "github.com/cernbox/reva/pkg/err" + "github.com/cernbox/reva/pkg/log" +) + +var ( + ctx = context.Background() + logger = log.New("grace") + errors = err.New("grace") + graceful = os.Getenv("GRACEFUL") == "true" + parentPID = os.Getppid() + listeners = []net.Listener{} + srvrs = []Server{} + pidFile string + childrenPID = []int{} +) + +func Exit(errc int) { + err := removePIDFile() + if err != nil { + logger.Error(ctx, err) + } else { + logger.Println(ctx, "pidfile got removed") + } + + os.Exit(errc) +} + +func getPIDFromFile(fn string) (int, error) { + piddata, err := ioutil.ReadFile(fn) + if err != nil { + return 0, err + } + // Convert the file contents to an integer. + pid, err := strconv.Atoi(string(piddata)) + if err != nil { + return 0, err + } + return pid, nil +} + +// Write a pid file, but first make sure it doesn't exist with a running pid. +func WritePIDFile(fn string) error { + // Read in the pid file as a slice of bytes. + if piddata, err := ioutil.ReadFile(fn); err == nil { + // Convert the file contents to an integer. + if pid, err := strconv.Atoi(string(piddata)); err == nil { + // Look for the pid in the process list. + if process, err := os.FindProcess(pid); err == nil { + // Send the process a signal zero kill. + if err := process.Signal(syscall.Signal(0)); err == nil { + if !graceful { + // We only get an error if the pid isn't running, or it's not ours. + return fmt.Errorf("pid already running: %d", pid) + } + + if pid != parentPID { // overwrite only if parent pid is pidfile + // We only get an error if the pid isn't running, or it's not ours. + return fmt.Errorf("pid %d is not this process parent", pid) + } + } else { + logger.Error(ctx, err) + } + } else { + logger.Error(ctx, err) + } + } else { + logger.Error(ctx, err) + } + } else { + logger.Error(ctx, err) + } + + // If we get here, then the pidfile didn't exist or we are are in graceful reload and thus we overwrite + // or the pid in it doesn't belong to the user running this app. + err := ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d", os.Getpid())), 0664) + if err != nil { + return err + } + logger.Printf(ctx, "pid file written to %s", fn) + pidFile = fn + return nil +} + +func newListener(network, addr string) (net.Listener, error) { + return net.Listen(network, addr) +} + +// return grpc listener first and http listener second. +func GetListeners(servers []Server) ([]net.Listener, error) { + srvrs = servers + lns := []net.Listener{} + if graceful { + logger.Println(ctx, "graceful restart, inheriting parent ln fds for grpc and http") + count := 3 + for _, s := range servers { + network, addr := s.Network(), s.Address() + fd := os.NewFile(uintptr(count), "") // 3 because ExtraFile passed to new process + count++ + ln, err := net.FileListener(fd) + if err != nil { + logger.Error(ctx, err) + // create new fd + ln, err := newListener(network, addr) + if err != nil { + return nil, err + } + lns = append(lns, ln) + } else { + lns = append(lns, ln) + } + + } + // kill parent + logger.Printf(ctx, "killing parent pid gracefully with SIGQUIT: %d", parentPID) + syscall.Kill(parentPID, syscall.SIGQUIT) + listeners = lns + return lns, nil + } else { + // create two listeners for grpc and http + for _, s := range servers { + network, addr := s.Network(), s.Address() + ln, err := newListener(network, addr) + if err != nil { + return nil, err + } + lns = append(lns, ln) + + } + listeners = lns + return lns, nil + } +} + +type Server interface { + Stop() error + GracefulStop() error + Network() string + Address() string +} + +func removePIDFile() error { + // only remove PID file if the PID written is us + filePID, err := getPIDFromFile(pidFile) + if err != nil { + return err + } + + if filePID != os.Getpid() { + return fmt.Errorf("pid in pidfile is different from running pid") + } + + return os.Remove(pidFile) +} + +func TrapSignals() { + signalCh := make(chan os.Signal, 1024) + signal.Notify(signalCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT) + for { + select { + case s := <-signalCh: + logger.Printf(ctx, "%v signal received", s) + switch s { + case syscall.SIGHUP: + logger.Println(ctx, "preparing for a hot-reload, forking child process...") + // Fork a child process. + listeners := getListeners() + p, err := forkChild(listeners...) + if err != nil { + logger.Println(ctx, "unable to fork child process: ", err) + } else { + logger.Printf(ctx, "child forked with new pid %d", p.Pid) + childrenPID = append(childrenPID, p.Pid) + } + + case syscall.SIGQUIT: + logger.Println(ctx, "preparing for a graceful shutdown with deadline of 10 seconds") + go func() { + count := 10 + for range time.Tick(time.Second) { + logger.Printf(ctx, "shuting down in %d seconds", count-1) + count-- + if count <= 0 { + logger.Println(ctx, "deadline reached before draining active conns, hard stoping ...") + for _, s := range srvrs { + s.Stop() + logger.Printf(ctx, "fd to %s:%s abruptly closed", s.Network(), s.Address()) + } + Exit(1) + } + } + }() + for _, s := range srvrs { + logger.Printf(ctx, "fd to %s:%s gracefully closed ", s.Network(), s.Address()) + s.GracefulStop() + } + logger.Println(ctx, "exit with error code 0") + Exit(0) + case syscall.SIGINT, syscall.SIGTERM: + logger.Println(ctx, "preparing for hard shutdown, aborting all conns") + for _, s := range srvrs { + logger.Printf(ctx, "fd to %s:%s abruptly closed", s.Network(), s.Address()) + err := s.Stop() + if err != nil { + err = errors.Wrap(err, "error stopping server") + logger.Error(ctx, err) + } + } + Exit(0) + } + } + } +} + +func getListenerFile(ln net.Listener) (*os.File, error) { + switch t := ln.(type) { + case *net.TCPListener: + return t.File() + case *net.UnixListener: + return t.File() + } + return nil, fmt.Errorf("unsupported listener: %T", ln) +} + +func forkChild(lns ...net.Listener) (*os.Process, error) { + // Get the file descriptor for the listener and marshal the metadata to pass + // to the child in the environment. + fds := []*os.File{} + for _, ln := range lns { + fd, err := getListenerFile(ln) + if err != nil { + return nil, err + } + fds = append(fds, fd) + } + + // Pass stdin, stdout, and stderr along with the listener file to the child + files := []*os.File{ + os.Stdin, + os.Stdout, + os.Stderr, + } + files = append(files, fds...) + + // Get current environment and add in the listener to it. + environment := append(os.Environ(), "GRACEFUL=true") + + // Get current process name and directory. + execName, err := os.Executable() + if err != nil { + return nil, err + } + execDir := filepath.Dir(execName) + + // Spawn child process. + p, err := os.StartProcess(execName, os.Args, &os.ProcAttr{ + Dir: execDir, + Env: environment, + Files: files, + Sys: &syscall.SysProcAttr{}, + }) + + // TODO(labkode): if the process dies (because config changed and is wrong + // we need to return an error + if err != nil { + return nil, err + } + + return p, nil +} + +func getListeners() []net.Listener { + return listeners +} diff --git a/cmd/revad/grpcsvr/grpcsvr.go b/cmd/revad/grpcsvr/grpcsvr.go new file mode 100644 index 0000000000..b25ce8f039 --- /dev/null +++ b/cmd/revad/grpcsvr/grpcsvr.go @@ -0,0 +1,178 @@ +package grpcsvr + +import ( + "context" + "fmt" + "net" + + storagebrokerv0alphapb "github.com/cernbox/go-cs3apis/cs3/storagebroker/v0alpha" + + appproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/appprovider/v0alpha" + + appregistryv0alphapb "github.com/cernbox/go-cs3apis/cs3/appregistry/v0alpha" + authv0alphapb "github.com/cernbox/go-cs3apis/cs3/auth/v0alpha" + storageproviderv0alphapb "github.com/cernbox/go-cs3apis/cs3/storageprovider/v0alpha" + + "github.com/cernbox/reva/pkg/err" + "github.com/cernbox/reva/pkg/log" + "github.com/cernbox/reva/services/grpcsvc/appprovidersvc" + "github.com/cernbox/reva/services/grpcsvc/appregistrysvc" + + "github.com/cernbox/reva/services/grpcsvc/authsvc" + "github.com/cernbox/reva/services/grpcsvc/interceptors" + "github.com/cernbox/reva/services/grpcsvc/storagebrokersvc" + "github.com/cernbox/reva/services/grpcsvc/storageprovidersvc" + + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + + "github.com/mitchellh/mapstructure" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +var ( + ctx = context.Background() + logger = log.New("grpcsvr") + errors = err.New("grpcsvr") +) + +type config struct { + Network string `mapstructure:"network"` + Address string `mapstructure:"address"` + ShutdownDeadline int `mapstructure:"shutdown_deadline"` + EnabledServices []string `mapstructure:"enabled_services"` + StorageProviderSvc map[string]interface{} `mapstructure:"storage_provider_svc"` + AuthSvc map[string]interface{} `mapstructure:"auth_svc"` + StorageBrokerSvc map[string]interface{} `mapstructure:"storage_broker_svc"` + AppRegistrySvc map[string]interface{} `mapstructure:"app_registry_svc"` + AppProviderSvc map[string]interface{} `mapstructure:"app_provider_svc"` +} + +type Server struct { + s *grpc.Server + conf *config + listener net.Listener +} + +func New(m map[string]interface{}) (*Server, error) { + conf := &config{} + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + opts := getOpts() + s := grpc.NewServer(opts...) + + return &Server{s: s, conf: conf}, nil +} + +func (s *Server) Start(ln net.Listener) error { + if err := s.registerServices(); err != nil { + err = errors.Wrap(err, "unable to register service") + return err + } + + s.listener = ln + + err := s.s.Serve(s.listener) + if err != nil { + err = errors.Wrap(err, "serve failed") + return err + } else { + return nil + } +} + +func (s *Server) Stop() error { + s.s.Stop() + return nil +} + +func (s *Server) GracefulStop() error { + s.s.GracefulStop() + return nil +} + +func (s *Server) Network() string { + return s.conf.Network +} + +func (s *Server) Address() string { + return s.conf.Address +} + +func (s *Server) registerServices() error { + enabled := []string{} + for _, k := range s.conf.EnabledServices { + switch k { + case "storage_provider_svc": + svc, err := storageprovidersvc.New(s.conf.StorageProviderSvc) + if err != nil { + return errors.Wrap(err, "unable to register service "+k) + } + storageproviderv0alphapb.RegisterStorageProviderServiceServer(s.s, svc) + enabled = append(enabled, k) + case "auth_svc": + svc, err := authsvc.New(s.conf.AuthSvc) + if err != nil { + return errors.Wrap(err, "unable to register service "+k) + } + authv0alphapb.RegisterAuthServiceServer(s.s, svc) + enabled = append(enabled, k) + + case "storage_broker_svc": + svc, err := storagebrokersvc.New(s.conf.StorageBrokerSvc) + if err != nil { + return errors.Wrap(err, "unable to register service "+k) + } + storagebrokerv0alphapb.RegisterStorageBrokerServiceServer(s.s, svc) + enabled = append(enabled, k) + case "app_registry_svc": + svc, err := appregistrysvc.New(s.conf.AppRegistrySvc) + if err != nil { + return errors.Wrap(err, "unable to register service "+k) + } + appregistryv0alphapb.RegisterAppRegistryServiceServer(s.s, svc) + enabled = append(enabled, k) + case "app_provider_svc": + svc, err := appprovidersvc.New(s.conf.AppProviderSvc) + if err != nil { + return errors.Wrap(err, "unable to register service "+k) + } + appproviderv0alphapb.RegisterAppProviderServiceServer(s.s, svc) + enabled = append(enabled, k) + } + } + if len(enabled) == 0 { + logger.Println(ctx, "no services enabled") + } else { + for k := range enabled { + logger.Printf(ctx, "grpc service enabled: %s", enabled[k]) + } + } + return nil +} + +func getOpts() []grpc.ServerOption { + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor( + grpc_middleware.ChainUnaryServer( + grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandlerContext(recoveryFunc)), + interceptors.TraceUnaryServerInterceptor(), + interceptors.LogUnaryServerInterceptor(), + grpc_prometheus.UnaryServerInterceptor)), + grpc.StreamInterceptor( + grpc_middleware.ChainStreamServer( + grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandlerContext(recoveryFunc)), + interceptors.TraceStreamServerInterceptor(), + grpc_prometheus.StreamServerInterceptor)), + } + return opts +} + +func recoveryFunc(ctx context.Context, p interface{}) (err error) { + logger.Panic(ctx, fmt.Sprintf("%+v", p)) + return grpc.Errorf(codes.Internal, "%s", p) +} diff --git a/cmd/revad/httpsvr/httpsvr.go b/cmd/revad/httpsvr/httpsvr.go new file mode 100644 index 0000000000..f31f2f3b89 --- /dev/null +++ b/cmd/revad/httpsvr/httpsvr.go @@ -0,0 +1,144 @@ +package httpsvr + +import ( + "context" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/cernbox/reva/services/httpsvc" + + "github.com/cernbox/reva/pkg/err" + "github.com/cernbox/reva/pkg/log" + "github.com/cernbox/reva/services/httpsvc/handlers" + "github.com/cernbox/reva/services/httpsvc/iframeuisvc" + "github.com/cernbox/reva/services/httpsvc/ocdavsvc" + "github.com/cernbox/reva/services/httpsvc/prometheussvc" + "github.com/cernbox/reva/services/httpsvc/webuisvc" + + "github.com/mitchellh/mapstructure" +) + +var ( + ctx = context.Background() + logger = log.New("httpsvr") + errors = err.New("httpsvr") +) + +type config struct { + Network string `mapstructure:"network"` + Address string `mapstructure:"address"` + EnabledServices []string `mapstructure:"enabled_services"` + WebUISvc map[string]interface{} `mapstructure:"webui_svc"` + OCDAVSvc map[string]interface{} `mapstructure:"ocdav_svc"` + PromSvc map[string]interface{} `mapstructure:"prometheus_svc"` + IFrameUISvc map[string]interface{} `mapstructure:"iframe_ui_svc"` +} + +// Server contains the server info. +type Server struct { + httpServer *http.Server + conf *config + listener net.Listener + svcs map[string]http.Handler +} + +// New returns a new server +func New(m map[string]interface{}) (*Server, error) { + conf := &config{} + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + httpServer := &http.Server{} + return &Server{httpServer: httpServer, conf: conf}, nil +} + +// Start starts the server +func (s *Server) Start(ln net.Listener) error { + if err := s.registerServices(); err != nil { + return err + } + + s.httpServer.Handler = s.getHandler() + s.listener = ln + err := s.httpServer.Serve(s.listener) + if err == nil || err == http.ErrServerClosed { + return nil + } + return err +} + +func (s *Server) Stop() error { + // TODO(labkode): set ctx deadline to zero + ctx, _ = context.WithTimeout(ctx, time.Second) + return s.httpServer.Shutdown(ctx) +} + +func (s *Server) Network() string { + return s.conf.Network +} + +func (s *Server) Address() string { + return s.conf.Address +} + +func (s *Server) GracefulStop() error { + return s.httpServer.Shutdown(ctx) +} + +func (s *Server) registerServices() error { + svcs := map[string]http.Handler{} + var svc httpsvc.Service + var err error + for _, k := range s.conf.EnabledServices { + switch k { + case "webui_svc": + svc, err = webuisvc.New(s.conf.WebUISvc) + case "ocdav_svc": + svc, err = ocdavsvc.New(s.conf.OCDAVSvc) + case "prometheus_svc": + svc, err = prometheussvc.New(s.conf.PromSvc) + case "iframe_ui_svc": + svc, err = iframeuisvc.New(s.conf.IFrameUISvc) + } + + if err != nil { + return errors.Wrap(err, "unable to register service "+k) + } + svcs[svc.Prefix()] = svc.Handler() + } + + if len(svcs) == 0 { + logger.Println(ctx, "no services enabled") + } else { + for k := range s.conf.EnabledServices { + logger.Printf(ctx, "http service enabled: %s", s.conf.EnabledServices[k]) + } + } + + // instrument services with prometheus + for prefix, h := range svcs { + + svcs[prefix] = prometheus.InstrumentHandler(prefix, h) + } + s.svcs = svcs + return nil +} + +func (s *Server) getHandler() http.Handler { + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var head string + head, r.URL.Path = httpsvc.ShiftPath(r.URL.Path) + //logger.Println(r.Context(), "http routing: head=", head, " tail=", r.URL.Path) + + if h, ok := s.svcs[head]; ok { + h.ServeHTTP(w, r) + return + } + w.WriteHeader(http.StatusNotFound) + }) + return handlers.TraceHandler(handlers.LogHandler(logger, h)) +} diff --git a/cmd/revad/main.go b/cmd/revad/main.go new file mode 100644 index 0000000000..2ad5433c13 --- /dev/null +++ b/cmd/revad/main.go @@ -0,0 +1,210 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "runtime" + "strconv" + "strings" + + "github.com/cernbox/reva/pkg/err" + "github.com/cernbox/reva/pkg/log" + + "github.com/cernbox/reva/cmd/revad/config" + "github.com/cernbox/reva/cmd/revad/grace" + "github.com/cernbox/reva/cmd/revad/grpcsvr" + "github.com/cernbox/reva/cmd/revad/httpsvr" + + "github.com/mitchellh/mapstructure" +) + +var ( + errors = err.New("main") + logger = log.New("main") + ctx = context.Background() + conf *coreConfig + + versionFlag = flag.Bool("v", false, "show version and exit") + testFlag = flag.Bool("t", false, "test configuration and exit") + signalFlag = flag.String("s", "", "send signal to a master process: stop, quit, reopen, reload") + fileFlag = flag.String("c", "/etc/revad/revad.toml", "set configuration file") + pidFlag = flag.String("p", "/var/run/revad.pid", "pid file") + + // provided at compile time + GitCommit, GitBranch, GitState, GitSummary, BuildDate, Version string +) + +func init() { + checkFlags() + writePIDFile() + readConfig() + log.Out = getLogOutput(conf.LogFile) + log.Mode = conf.LogMode + if err := log.EnableAll(); err != nil { + fmt.Fprintln(os.Stderr, err) + grace.Exit(1) + } +} + +func main() { + tweakCPU() + printLoggedPkgs() + + grpcSvr := getGRPCServer() + httpSvr := getHTTPServer() + servers := []grace.Server{grpcSvr, httpSvr} + listeners, err := grace.GetListeners(servers) + if err != nil { + logger.Error(ctx, err) + grace.Exit(1) + } + + go func() { + if err := grpcSvr.Start(listeners[0]); err != nil { + err = errors.Wrap(err, "error starting grpc server") + logger.Error(ctx, err) + grace.Exit(1) + } + }() + + go func() { + if err := httpSvr.Start(listeners[1]); err != nil { + err = errors.Wrap(err, "error starting http server") + logger.Error(ctx, err) + grace.Exit(1) + } + }() + + grace.TrapSignals() +} + +func getGRPCServer() *grpcsvr.Server { + s, err := grpcsvr.New(config.Get("grpc")) + if err != nil { + logger.Error(ctx, err) + grace.Exit(1) + } + return s +} + +func getHTTPServer() *httpsvr.Server { + s, err := httpsvr.New(config.Get("http")) + if err != nil { + logger.Error(ctx, err) + grace.Exit(1) + } + return s +} + +func checkFlags() { + flag.Parse() + + if *versionFlag { + msg := "Version: %s\n" + msg += "GitCommit: %s\n" + msg += "GitBranch: %s\n" + msg += "GitSummary: %s\n" + msg += "BuildDate: %s\n" + fmt.Printf(msg, Version, GitCommit, GitBranch, GitSummary, BuildDate) + grace.Exit(1) + } + + if *fileFlag != "" { + config.SetFile(*fileFlag) + } + + if *testFlag { + err := config.Read() + if err != nil { + fmt.Println("unable to read configuration file: ", *fileFlag, err) + grace.Exit(1) + } + grace.Exit(0) + } + + if *signalFlag != "" { + fmt.Println("signaling master process") + grace.Exit(1) + } +} + +func readConfig() { + err := config.Read() + if err != nil { + fmt.Println("unable to read configuration file:", *fileFlag, err) + grace.Exit(1) + } + + // get core config + + conf = &coreConfig{} + if err := mapstructure.Decode(config.Get("core"), conf); err != nil { + fmt.Fprintln(os.Stderr, "unable to parse core config:", err) + grace.Exit(1) + } +} + +// tweakCPU parses string cpu and sets GOMAXPROCS +// according to its value. It accepts either +// a number (e.g. 3) or a percent (e.g. 50%). +func tweakCPU() error { + cpu := conf.MaxCPUs + var numCPU int + + availCPU := runtime.NumCPU() + + if strings.HasSuffix(cpu, "%") { + // Percent + var percent float32 + pctStr := cpu[:len(cpu)-1] + pctInt, err := strconv.Atoi(pctStr) + if err != nil || pctInt < 1 || pctInt > 100 { + return errors.New("invalid CPU value: percentage must be between 1-100") + } + percent = float32(pctInt) / 100 + numCPU = int(float32(availCPU) * percent) + } else { + // Number + num, err := strconv.Atoi(cpu) + if err != nil || num < 1 { + return errors.New("invalid CPU value: provide a number or percent greater than 0") + } + numCPU = num + } + + if numCPU > availCPU { + numCPU = availCPU + } + + logger.Printf(ctx, "running on %d cpus", numCPU) + runtime.GOMAXPROCS(numCPU) + return nil +} + +func writePIDFile() { + err := grace.WritePIDFile(*pidFlag) + if err != nil { + fmt.Fprintln(os.Stderr, err) + grace.Exit(1) + } +} + +type coreConfig struct { + MaxCPUs string `mapstructure:"max_cpus"` + LogFile string `mapstructure:"log_file"` + LogMode string `mapstructure:"log_mode"` +} + +func getLogOutput(val string) io.Writer { + return os.Stderr +} + +func printLoggedPkgs() { + pkgs := log.ListEnabledPackages() + for k := range pkgs { + logger.Printf(ctx, "logging enabled for package: %s", pkgs[k]) + } +} diff --git a/cmd/revad/revad.service b/cmd/revad/revad.service new file mode 100644 index 0000000000..df0dbdaedb --- /dev/null +++ b/cmd/revad/revad.service @@ -0,0 +1,16 @@ +[Unit] +Description=revad +After=syslog.target + +[Service] +Type=simple +User=root +Group=root +ExecStart=/usr/local/bin/revad +StandardOutput=syslog +StandardError=syslog +LimitNOFILE=49152 + +[Install] +WantedBy=multi-user.target + diff --git a/cmd/revad/revad.toml b/cmd/revad/revad.toml new file mode 100644 index 0000000000..e915782537 --- /dev/null +++ b/cmd/revad/revad.toml @@ -0,0 +1,74 @@ +[core] +log_file = "stderr" +log_mode = "dev" +max_cpus = "32" + +[grpc] +network = "tcp" +address = "0.0.0.0:9999" +access_log = "stderr" +tls_enabled = true +tls_cert = "/etc/gridsecurity/host.cert" +tls_key = "/etc/gridsecurity/host.key" +enabled_services = ["storage_provider_svc", "auth_svc", "storage_broker_svc", "app_registry_svc", "app_provider_svc"] + +[http] +enabled_services = ["prometheus_svc", "webui_svc", "ocdav_svc", "iframe_ui_svc"] +network = "tcp" +address = "0.0.0.0:9998" + +[grpc.storage_provider_svc] +driver = "local" +mount_path = "/" +mount_id = "123e4567-e89b-12d3-a456-426655440000" + +[grpc.auth_svc.auth_manager] +driver = "demo" + +[grpc.auth_svc.token_manager] +driver = "jwt" + +[grpc.auth_svc.user_manager] +driver = "demo" + +[grpc.storage_provider_svc.eos] +mgm = "root://nowhere.org" +root_uid = 0 +root_gid = 0 + +[grpc.storage_provider_svc.local] +root = "/var/tmp/owncloud/data" + +[grpc.storage_broker_svc] +driver = "static" + +[grpc.storage_broker_svc.static.rules] +"/" = "localhost:9999" +"123e4567-e89b-12d3-a456-426655440000" = "localhost:9999" + +[grpc.app_registry_svc] +driver = "static" + +[grpc.app_registry_svc.static.rules] +".txt" = "localhost:9999" +"txt/plain" = "localhost:9999" + +[grpc.app_provider_svc] +driver = "demo" + +[grpc.app_provider_svc.demo] +iframe_ui_provider = "http://localhost:9998/iframeuisvc" + +[http.prometheus_svc] +prefix = "metrics" + +[http.webui_svc] +prefix = "ui" + +[http.ocdav_svc] +prefix = "owncloud" +chunk_folder = "/var/tmp/owncloud/chunks" +storageprovidersvc = "localhost:9999" + +[http.iframe_ui_svc] +prefix = "iframeuisvc" diff --git a/pkg/app/app.go b/pkg/app/app.go new file mode 100644 index 0000000000..39667df29c --- /dev/null +++ b/pkg/app/app.go @@ -0,0 +1,21 @@ +package app + +import "context" + +// Registry is the interface that application registries implement +// for discovering application providers +type Registry interface { + FindProvider(ctx context.Context, ext, mimetype string) (*ProviderInfo, error) +} + +// ProviderInfo contains the information +// about a Application Provider +type ProviderInfo struct { + Location string +} + +// Provider is the interface that application providers implement +// for providing the iframe location to a iframe UI Provider +type Provider interface { + GetIFrame(ctx context.Context, fn, mimetype, token string) (string, error) +} diff --git a/pkg/app/provider/demo/demo.go b/pkg/app/provider/demo/demo.go new file mode 100644 index 0000000000..615230822e --- /dev/null +++ b/pkg/app/provider/demo/demo.go @@ -0,0 +1,49 @@ +package demo + +import ( + "context" + "fmt" + + "github.com/cernbox/reva/pkg/app" + "github.com/cernbox/reva/pkg/log" + + "github.com/mitchellh/mapstructure" +) + +var logger = log.New("demo") + +type provider struct { + iframeUIProvider string +} + +func (p *provider) GetIFrame(ctx context.Context, filename, mimetype, token string) (string, error) { + msg := fmt.Sprintf("