Skip to content

Commit

Permalink
feat: build out cli (#36)
Browse files Browse the repository at this point in the history
* feat: extract cli connection params

* feat: replications cli, misc. improvements

* feat: add provider commands

* feat: provider list, jq formattable output

* feat: add dataset cmd

* feat: fix issue incorrectly setting self service

* feat: cleanup

* feat: wallet list

* feat: update docs, add wallet assoc cmd

* feat: use http methods everywhere

* feat: cleanup & refactor

* feat: wip ptolemy doc

---------

Co-authored-by: Jason Cihelka <jcihelka@isotechnics.com>
  • Loading branch information
jcace and Jason Cihelka authored Mar 29, 2023
1 parent 7ecbb88 commit 257d2fa
Show file tree
Hide file tree
Showing 16 changed files with 585 additions and 51 deletions.
14 changes: 7 additions & 7 deletions api/dataset.go → api/datasets.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ import (
"github.com/labstack/echo/v4"
)

func ConfigureDatasetRouter(e *echo.Group, dldm *core.DeltaDM) {
dataset := e.Group("/datasets")
func ConfigureDatasetsRouter(e *echo.Group, dldm *core.DeltaDM) {
datasets := e.Group("/datasets")

dataset.Use(dldm.AS.AuthMiddleware)
datasets.Use(dldm.AS.AuthMiddleware)

dataset.GET("", func(c echo.Context) error {
datasets.GET("", func(c echo.Context) error {
var ds []core.Dataset

dldm.DB.Model(&core.Dataset{}).Preload("Wallet").Find(&ds)
Expand All @@ -35,7 +35,7 @@ func ConfigureDatasetRouter(e *echo.Group, dldm *core.DeltaDM) {
return c.JSON(200, ds)
})

dataset.POST("", func(c echo.Context) error {
datasets.POST("", func(c echo.Context) error {
var ads core.Dataset

if err := c.Bind(&ads); err != nil {
Expand Down Expand Up @@ -67,7 +67,7 @@ func ConfigureDatasetRouter(e *echo.Group, dldm *core.DeltaDM) {
return c.JSON(200, ads)
})

dataset.GET("/content/:dataset", func(c echo.Context) error {
datasets.GET("/content/:dataset", func(c echo.Context) error {
var content []core.Content
var dataset core.Dataset

Expand All @@ -87,7 +87,7 @@ func ConfigureDatasetRouter(e *echo.Group, dldm *core.DeltaDM) {
return c.JSON(200, content)
})

dataset.POST("/content/:dataset", func(c echo.Context) error {
datasets.POST("/content/:dataset", func(c echo.Context) error {
var content []core.Content
var dataset core.Dataset
results := struct {
Expand Down
14 changes: 12 additions & 2 deletions api/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ import (
"github.com/labstack/echo/v4"
)

type ProviderPutBody struct {
ActorName string `json:"actor_name"`
AllowSelfService string `json:"allow_self_service"`
}

func ConfigureProvidersRouter(e *echo.Group, dldm *core.DeltaDM) {
providers := e.Group("/providers")

Expand Down Expand Up @@ -59,7 +64,7 @@ func ConfigureProvidersRouter(e *echo.Group, dldm *core.DeltaDM) {
return fmt.Errorf("provider id not specified")
}

var p core.Provider
var p ProviderPutBody

if err := c.Bind(&p); err != nil {
return err
Expand All @@ -75,7 +80,12 @@ func ConfigureProvidersRouter(e *echo.Group, dldm *core.DeltaDM) {
if p.ActorName != "" {
existing.ActorName = p.ActorName
}
existing.AllowSelfService = p.AllowSelfService

if p.AllowSelfService == "on" {
existing.AllowSelfService = true
} else if p.AllowSelfService == "off" {
existing.AllowSelfService = false
}

res = dldm.DB.Save(&existing)
if res.Error != nil {
Expand Down
19 changes: 17 additions & 2 deletions api/replications.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ type PostReplicationBody struct {
Dataset *string `json:"dataset,omitempty"`
NumDeals *uint `json:"num_deals,omitempty"`
// NumTib *int `json:"num_tib,omitempty"`
PricePerDeal float64 `json:"price_per_deal,omitempty"`
// PricePerDeal float64 `json:"price_per_deal,omitempty"`
}

func ConfigureReplicationsRouter(e *echo.Group, dldm *core.DeltaDM) {
Expand Down Expand Up @@ -186,6 +186,21 @@ func handlePostReplications(c echo.Context, dldm *core.DeltaDM) error {
return fmt.Errorf("must specify num_deals")
}

var providerExists bool
err := dldm.DB.Model(core.Provider{}).
Select("count(*) > 0").
Where("actor_id = ?", d.Provider).
Find(&providerExists).
Error

if err != nil {
return fmt.Errorf("could not check if provider %s exists: %s", d.Provider, err)
}

if !providerExists {
return fmt.Errorf("provider %s does not exist in ddm. please add it first", d.Provider)
}

// TODO: Support num_tib to allow specifying the amount of data to replicate

toReplicate, err := findUnreplicatedContentForProvider(dldm.DB, d.Provider, d.Dataset, d.NumDeals)
Expand Down Expand Up @@ -213,7 +228,7 @@ func handlePostReplications(c echo.Context, dldm *core.DeltaDM) error {
Size: c.Size,
SkipIpniAnnounce: !c.Indexed,
RemoveUnsealedCopies: !c.Unsealed,
DurationInDays: c.DealDuration - 3, // TODO: Allow specifying duration, with default
DurationInDays: c.DealDuration, // TODO: Allow specifying duration, with default
StartEpochAtDays: 3,
PieceCommitment: core.PieceCommitment{
PieceCid: c.CommP,
Expand Down
2 changes: 1 addition & 1 deletion api/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func InitializeEchoRouterConfig(dldm *core.DeltaDM) {

apiGroup := e.Group("/api/v1")

ConfigureDatasetRouter(apiGroup, dldm)
ConfigureDatasetsRouter(apiGroup, dldm)
ConfigureProvidersRouter(apiGroup, dldm)
ConfigureReplicationsRouter(apiGroup, dldm)
ConfigureWalletsRouter(apiGroup, dldm)
Expand Down
2 changes: 1 addition & 1 deletion api/selfservice.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func handleSelfServiceByCid(c echo.Context, dldm *core.DeltaDM) error {
Size: cnt.Size,
SkipIpniAnnounce: !ds.Indexed,
RemoveUnsealedCopies: !ds.Unsealed,
DurationInDays: ds.DealDuration - delayDays,
DurationInDays: ds.DealDuration,
StartEpochAtDays: delayDays,
PieceCommitment: core.PieceCommitment{
PieceCid: cnt.CommP,
Expand Down
45 changes: 41 additions & 4 deletions cmd/cmdProcessor.go → cmd/cmd-processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,43 @@ import (
"fmt"
"io/ioutil"
"net/http"
"os"

"github.com/urfave/cli/v2"
)

var CLIConnectFlags = []cli.Flag{
&cli.StringFlag{
Name: "ddm-api-info",
Usage: "DDM API connection info",
EnvVars: []string{"DDM_API_INFO"},
Hidden: true,
},
&cli.StringFlag{
Name: "delta-auth",
Usage: "delta auth token",
EnvVars: []string{"DELTA_AUTH"},
Hidden: true,
},
}

type CmdProcessor struct {
ddmUrl string
ddmAuthKey string
}

func NewCmdProcessor(ddmUrl string, ddmAuthKey string) (*CmdProcessor, error) {
func NewCmdProcessor(c *cli.Context) (*CmdProcessor, error) {
ddmUrl := getFlagOrEnvVar(c, "ddm-api-info", "DDM_API_INFO", "http://localhost:1314")
ddmAuthKey := getFlagOrEnvVar(c, "delta-auth", "DELTA_AUTH", "")

if ddmAuthKey == "" {
return nil, fmt.Errorf("DELTA_AUTH env variable or --delta-auth flag is required")
}

err := healthCheck(ddmUrl, ddmAuthKey)

if err != nil {
return nil, err
return nil, fmt.Errorf("unable to communicate with ddm daemon: %s", err)
}

return &CmdProcessor{
Expand All @@ -25,9 +50,21 @@ func NewCmdProcessor(ddmUrl string, ddmAuthKey string) (*CmdProcessor, error) {
}, nil
}

// If the flag is set, use it. If not, check the environment variable. If that's not set, use the default value
func getFlagOrEnvVar(c *cli.Context, flagName, envVarName, defaultValue string) string {
value := c.String(flagName)
if value == "" {
value = os.Getenv(envVarName)
if value == "" {
value = defaultValue
}
}
return value
}

// Verify that DDM API is reachable
func healthCheck(ddmUrl string, ddmAuthKey string) error {
req, err := http.NewRequest("GET", ddmUrl+"/api/v1/health", nil)
req, err := http.NewRequest(http.MethodGet, ddmUrl+"/api/v1/health", nil)
if err != nil {
return fmt.Errorf("could not construct http request %v", err)
}
Expand All @@ -54,7 +91,7 @@ func healthCheck(ddmUrl string, ddmAuthKey string) error {
return err
}

func (c *CmdProcessor) ddmRequest(method string, url string, raw []byte) ([]byte, func() error, error) {
func (c *CmdProcessor) MakeRequest(method string, url string, raw []byte) ([]byte, func() error, error) {
req, err := http.NewRequest(method, c.ddmUrl+url, bytes.NewBuffer(raw))
if err != nil {
return nil, nil, fmt.Errorf("could not construct http request %v", err)
Expand Down
110 changes: 110 additions & 0 deletions cmd/dataset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package cmd

import (
"encoding/json"
"fmt"
"net/http"

"github.com/application-research/delta-dm/core"
"github.com/application-research/delta-dm/util"
"github.com/urfave/cli/v2"
)

func DatasetCmd() []*cli.Command {
var datasetName string
var replicationQuota uint64
var dealDuration uint64

// add a command to run API node
var datasetCmds []*cli.Command
datasetCmd := &cli.Command{
Name: "dataset",
Usage: "Dataset Commands",
Subcommands: []*cli.Command{
{
Name: "add",
Usage: "add dataset",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "name",
Aliases: []string{"n"},
Usage: "dataset name (slug)",
Destination: &datasetName,
Required: true,
},
&cli.Uint64Flag{
Name: "replicaion-quota",
Aliases: []string{"q"},
Usage: "replication quota - how many times the dataset may be replicated",
DefaultText: "6",
Value: 6,
Destination: &replicationQuota,
},
&cli.Uint64Flag{
Name: "duration",
Aliases: []string{"d"},
Usage: "deal duration - how long (in days) should deals for this dataset last",
DefaultText: "540",
Value: 540,
Destination: &dealDuration,
},
},
Action: func(c *cli.Context) error {
cmd, err := NewCmdProcessor(c)
if err != nil {
return err
}

if !util.ValidateDatasetName(datasetName) {
return fmt.Errorf("invalid dataset name. must contain only lowercase letters, numbers and hyphens. must begin and end with a letter. must not contain consecutive hyphens")
}

body := core.Dataset{
Name: datasetName,
ReplicationQuota: replicationQuota,
DealDuration: dealDuration,
}

b, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("unable to construct request body %s", err)
}

res, closer, err := cmd.MakeRequest(http.MethodPost, "/api/v1/datasets", b)
if err != nil {
return fmt.Errorf("unable to make request %s", err)
}
defer closer()

fmt.Printf("%s", string(res))

return nil
},
},
{
Name: "list",
Usage: "list datasets",
Action: func(c *cli.Context) error {
cmd, err := NewCmdProcessor(c)
if err != nil {
return err
}

res, closer, err := cmd.MakeRequest(http.MethodGet, "/api/v1/datasets", nil)
if err != nil {
return fmt.Errorf("unable to make request %s", err)
}
defer closer()

fmt.Printf("%s", string(res))

return nil
},
},
},
}

datasetCmds = append(datasetCmds, datasetCmd)

return datasetCmds
}
Loading

0 comments on commit 257d2fa

Please sign in to comment.