Refine API (#74)

* Refine API Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * add cli Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * add gird demo Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * add GKEDemo Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix test Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * add mnist-models.yaml Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix docs Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * improve GKEdemo docs Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * add more docs Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix test Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * enable get worker from trialid ana add getParameterList from studyid Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix doc Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix test Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com> * fix Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com>
kubeflow · May 15, 2018 · 0a95175 · 0a95175
1 parent 06d6ea5
commit 0a95175
Show file tree

Hide file tree

Showing 125 changed files with 6,499 additions and 7,578 deletions.
diff --git a/README.md b/README.md
@@ -38,7 +38,6 @@ Each component communicates with others via GRPC and the API is defined at `api/
 - vizier: main components.
     - vizier-core : API server of vizier.
     - vizier-db
-- dlk-manager : a interface of kubernetes.
 - suggestion : implementation of each exploration algorithm.
     - vizier-suggestion-random
     - vizier-suggestion-grid

diff --git a/cmd/cli/get-model.go b/cmd/cli/get-model.go
@@ -52,68 +52,28 @@ func getModel(cmd *cobra.Command, opt *getModelOpt) {
 		return
 	}
 	defer conn.Close()
-	var soverviews []*api.StudyOverview
 	c := api.NewManagerClient(conn)
 	// Search study if Study ID or name is set
-	if len(opt.args) > 0 {
-		// Search specified study in running studies
-		req := &api.GetStudiesRequest{}
-		r, err := c.GetStudies(context.Background(), req)
-		if err != nil {
-			log.Fatalf("GetStudy failed: %v", err)
-			return
-		}
-		if len(r.StudyInfos) > 0 {
-			for _, si := range r.StudyInfos {
-				if len(opt.args) > 0 {
-					if utf8.RuneCountInString(opt.args[0]) >= 7 {
-						if strings.HasPrefix(si.StudyId, opt.args[0]) {
-							soverviews = append(soverviews, &api.StudyOverview{
-								Name:  si.Name,
-								Owner: si.Owner,
-							})
-							break
-						}
-					}
-					if si.Name == opt.args[0] {
-						soverviews = append(soverviews, &api.StudyOverview{
-							Name:  si.Name,
-							Owner: si.Owner,
-						})
-						break
-					}
-				} else {
-					soverviews = append(soverviews, &api.StudyOverview{
-						Name:  si.Name,
-						Owner: si.Owner,
-					})
-				}
-			}
-		}
+	req := &api.GetStudyListRequest{}
+	r, err := c.GetStudyList(context.Background(), req)
+	if err != nil {
+		log.Fatalf("GetModels failed: %v", err)
 	}
-	if len(soverviews) == 0 {
-		// Search specified study from ModelDB
-		sreq := &api.GetSavedStudiesRequest{}
-		sr, err := c.GetSavedStudies(context.Background(), sreq)
-		if err != nil {
-			log.Fatalf("GetStudy failed: %v", err)
-			return
-		}
-		if len(sr.Studies) == 0 {
-			log.Fatalf("No Studies are saved.")
-			return
-		}
-		for _, s := range sr.Studies {
-			if len(opt.args) > 0 {
-				if opt.args[0] == s.Name {
-					soverviews = append(soverviews, s)
+	if len(r.StudyOverviews) == 0 {
+		log.Println("No Study fond")
+		return
+	}
+	for _, si := range r.StudyOverviews {
+		if len(opt.args) > 0 {
+			if utf8.RuneCountInString(opt.args[0]) >= 7 {
+				if !strings.HasPrefix(si.Id, opt.args[0]) {
+					break
 				}
-			} else {
-				soverviews = append(soverviews, s)
+			}
+			if si.Name != opt.args[0] {
+				break
 			}
 		}
-	}
-	for _, si := range soverviews {
 		// Search Models from ModelDB
 		mreq := &api.GetSavedModelsRequest{StudyName: si.Name}
 		mr, err := c.GetSavedModels(context.Background(), mreq)
@@ -127,11 +87,11 @@ func getModel(cmd *cobra.Command, opt *getModelOpt) {
 		if opt.detail {
 			for _, m := range mr.Models {
 				if len(opt.args) > 1 {
-					if !strings.HasPrefix(m.TrialId, opt.args[1]) {
+					if !strings.HasPrefix(m.WorkerId, opt.args[1]) {
 						continue
 					}
 				}
-				fmt.Printf("TrialID :%v\n", m.TrialId)
+				fmt.Printf("WorkerID :%v\n", m.WorkerId)
 				fmt.Printf("Model Path: %s\n", m.ModelPath)
 				fmt.Println("Parameters:")
 				for _, p := range m.Parameters {
@@ -148,12 +108,12 @@ func getModel(cmd *cobra.Command, opt *getModelOpt) {
 			fmt.Fprintln(w, "TrialID\tParamNum\tMetricsNum")
 			for _, m := range mr.Models {
 				if len(opt.args) > 1 {
-					if !strings.HasPrefix(m.TrialId, opt.args[1]) {
+					if !strings.HasPrefix(m.WorkerId, opt.args[1]) {
 						continue
 					}
 				}
 				fmt.Fprintf(w, "%s\t%d\t%d\n",
-					string([]rune(m.TrialId)[:7]),
+					string([]rune(m.WorkerId)[:7]),
 					len(m.Parameters),
 					len(m.Metrics),
 				)

diff --git a/cmd/cli/get-study.go b/cmd/cli/get-study.go
@@ -44,40 +44,39 @@ func getStudy(cmd *cobra.Command, args []string) {
 	defer conn.Close()
 
 	c := api.NewManagerClient(conn)
-	req := &api.GetStudiesRequest{}
-	r, err := c.GetStudies(context.Background(), req)
+	req := &api.GetStudyListRequest{}
+	r, err := c.GetStudyList(context.Background(), req)
 	if err != nil {
 		log.Fatalf("GetStudy failed: %v", err)
 		return
 	}
-	var sis []*api.StudyInfo
+	result := []*api.StudyOverview{}
 	// Search study if Study ID or name is set
 	if len(args) > 0 {
-		for _, si := range r.StudyInfos {
+		for _, si := range r.StudyOverviews {
 			if utf8.RuneCountInString(args[0]) >= 7 {
-				if strings.HasPrefix(si.StudyId, args[0]) {
-					sis = append(sis, si)
+				if strings.HasPrefix(si.Id, args[0]) {
+					result = append(result, si)
 					break
 				}
 			}
 			if si.Name == args[0] {
-				sis = append(sis, si)
+				result = append(result, si)
 				break
 			}
 		}
 	} else {
-		sis = r.StudyInfos
+		result = r.StudyOverviews
 	}
 	w := new(tabwriter.Writer)
 	w.Init(os.Stdout, 0, 8, 0, '\t', tabwriter.TabIndent)
-	fmt.Fprintln(w, "StudyID\tName\tOwner\tRunning\tCompleted")
-	for _, si := range sis {
-		fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n",
-			string([]rune(si.StudyId)[:7]),
+	fmt.Fprintln(w, "StudyID\tName\tOwner")
+	for _, si := range result {
+		fmt.Fprintf(w, "%s\t%s\t%s\n",
+			string([]rune(si.Id)[:7]),
 			si.Name,
 			si.Owner,
-			si.RunningTrialNum,
-			si.CompletedTrialNum)
+		)
 	}
 	w.Flush()
 }
diff --git a/cmd/cli/pull-study.go b/cmd/cli/pull-study.go
@@ -0,0 +1,105 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/kubeflow/katib/pkg/api"
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type pullStudyOpt struct {
+	outfile string
+	args    []string
+}
+
+//NewCommandPullStudy generate pull studies cmd
+func NewCommandPullStudy() *cobra.Command {
+	var opt pullStudyOpt
+	cmd := &cobra.Command{
+		Use:   "studies",
+		Args:  cobra.ExactArgs(1),
+		Short: "Export a Study and its Models lnfo",
+		Long:  `Export Information of a Study and its Models to yaml format`,
+		Run: func(cmd *cobra.Command, args []string) {
+			opt.args = args
+			pullStudy(cmd, &opt)
+		},
+		Aliases: []string{"st"},
+	}
+	cmd.Flags().StringVarP(&opt.outfile, "output", "o", "", "File path to export")
+	return cmd
+}
+
+func pullStudy(cmd *cobra.Command, opt *pullStudyOpt) {
+	//check and get persistent flag volume
+	var pf *PersistentFlags
+	pf, err := CheckPersistentFlags()
+	if err != nil {
+		log.Fatalf("Fail to Check Flags: %v", err)
+	}
+	conn, err := grpc.Dial(pf.server, grpc.WithInsecure())
+	if err != nil {
+		log.Fatalf("could not connect: %v", err)
+	}
+	defer conn.Close()
+
+	c := api.NewManagerClient(conn)
+	listreq := &api.GetStudyListRequest{}
+	listr, err := c.GetStudyList(context.Background(), listreq)
+	if err != nil {
+		log.Fatalf("GetStudy failed: %v", err)
+		return
+	}
+	studyId := ""
+	// Search study by Study ID or name
+	for _, si := range listr.StudyOverviews {
+		if utf8.RuneCountInString(opt.args[0]) >= 7 {
+			if strings.HasPrefix(si.Id, opt.args[0]) {
+				studyId = si.Id
+				break
+			}
+		}
+		if si.Name == opt.args[0] {
+			studyId = si.Id
+			break
+		}
+	}
+	if studyId == "" {
+		log.Fatalf("Study %s is not found", opt.args[0])
+	}
+	req := &api.GetStudyRequest{
+		StudyId: studyId,
+	}
+	r, err := c.GetStudy(context.Background(), req)
+	if err != nil {
+		log.Fatalf("GetStudy failed: %v", err)
+	}
+	mreq := &api.GetSavedModelsRequest{
+		StudyName: r.StudyConfig.Name,
+	}
+	mr, err := c.GetSavedModels(context.Background(), mreq)
+	if err != nil {
+		log.Fatalf("GetModel failed: %v", err)
+	}
+	sd := StudyData{
+		StudyConf: r.StudyConfig,
+		Models:    mr.Models,
+	}
+	yst, err := yaml.Marshal(sd)
+	if err != nil {
+		log.Fatalf("Failed to Marshal: %v", err)
+	}
+	if opt.outfile != "" {
+		ioutil.WriteFile(opt.outfile, yst, os.ModePerm)
+	} else {
+		fmt.Println(string(yst))
+	}
+}
diff --git a/cmd/cli/pull.go b/cmd/cli/pull.go
@@ -0,0 +1,19 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+//NewCommandPull generate run cmd
+func NewCommandPull() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "pull",
+		Short: "Pull a resource from a file or from stdin.",
+		Long:  `YAML or JSON formats are accepted.`,
+	}
+
+	cmd.AddCommand(NewCommandPullStudy())
+	//	cmd.AddCommand(NewCommandPullModel())
+
+	return cmd
+}
diff --git a/cmd/cli/push-model.go b/cmd/cli/push-model.go
@@ -14,7 +14,7 @@ import (
 )
 
 type pushModelOpt struct {
-	conf string
+	file string
 	args []string
 }
 
@@ -26,13 +26,13 @@ func NewCommandPushModel() *cobra.Command {
 		Args:    cobra.MaximumNArgs(1),
 		Short:   "Push a model Info from a file or from stdin",
 		Long:    "YAML or JSON formats are accepted.",
-		Aliases: []string{"st"},
+		Aliases: []string{"md"},
 		Run: func(cmd *cobra.Command, args []string) {
 			opt.args = args
 			pushModel(cmd, &opt)
 		},
 	}
-	cmd.Flags().StringVarP(&opt.conf, "config", "f", "", "File path of study config")
+	cmd.Flags().StringVarP(&opt.file, "file", "f", "", "File path of model config file")
 	return cmd
 }
 
@@ -44,9 +44,9 @@ func pushModel(cmd *cobra.Command, opt *pushModelOpt) {
 		log.Fatalf("Fail to Check Flags: %v", err)
 		return
 	}
-	var req api.SaveModelRequest
-	if opt.conf != "" {
-		buf, _ := ioutil.ReadFile(opt.conf)
+	var req []*api.SaveModelRequest
+	if opt.file != "" {
+		buf, _ := ioutil.ReadFile(opt.file)
 		err = yaml.Unmarshal(buf, &req)
 		if err != nil {
 			log.Fatalf("Fail to Purse config: %v", err)
@@ -70,9 +70,11 @@ func pushModel(cmd *cobra.Command, opt *pushModelOpt) {
 	}
 	defer conn.Close()
 	c := api.NewManagerClient(conn)
-	_, err = c.SaveModel(context.Background(), &req)
-	if err != nil {
-		log.Fatalf("PushModel failed: %v", err)
+	for _, m := range req {
+		_, err = c.SaveModel(context.Background(), m)
+		if err != nil {
+			log.Fatalf("PushModel failed: %v", err)
+		}
+		fmt.Printf("Model %v is Pushed.\n", m.Model.WorkerId)
 	}
-	fmt.Printf("Model %v is Pushed.\n", req.Model.TrialId)
 }