issue#967: Customizable number of failures before snap disable the task

Updating the example tasks using the new additional failure field

issue#967: Define the failure default value when defining the command line flag

issue#967: Adding default constant value for failure according to code review.

Fix the constant declaration to match the rest of coding style

Renamed failure to max_failure according to the code review

issue#967: Change the failure type so it can accept negative value to represent infinite retry

.gitignore

@@ -37,3 +37,5 @@ tags
 
 # OSX stuff
 .DS_Store
+
+*.cov

cmd/snapctl/commands.go

@@ -49,6 +49,7 @@ var (
 						flTaskSchedDuration,
 						flTaskSchedNoStart,
 						flTaskDeadline,
+						flTaskFailures,
 					},
 				},
 				{

cmd/snapctl/flags.go

@@ -21,6 +21,10 @@ package main
 
 import "github.com/codegangsta/cli"
 
+const (
+	DefaultMaxFailures = 10
+)
+
 var (
 
 	// Main flags
@@ -122,6 +126,11 @@ var (
 		Name:  "deadline",
 		Usage: "The deadline for the task to be killed after started if the task runs too long (All tasks default to 5s)",
 	}
+	flTaskFailures = cli.IntFlag{
+		Name:  "max-failures",
+		Usage: "The number of consecutive failures before snap disable the task (Default 10 consective failures)",
+		Value: DefaultMaxFailures,
+	}
 
 	// metric
 	flMetricVersion = cli.IntFlag{

cmd/snapctl/task.go

@@ -78,11 +78,12 @@ func trunc(n int) string {
 }
 
 type task struct {
-	Version  int
-	Schedule *client.Schedule
-	Workflow *wmap.WorkflowMap
-	Name     string
-	Deadline string
+	Version     int
+	Schedule    *client.Schedule
+	Workflow    *wmap.WorkflowMap
+	Name        string
+	Deadline    string
+	MaxFailures int `json:"max-failures"`
 }
 
 func createTask(ctx *cli.Context) error {
@@ -134,7 +135,14 @@ func createTaskUsingTaskManifest(ctx *cli.Context) error {
 		fmt.Println("Invalid version provided")
 		return errCritical
 	}
-	r := pClient.CreateTask(t.Schedule, t.Workflow, t.Name, t.Deadline, !ctx.IsSet("no-start"))
+
+	// If the number of failures does not specific, default value is 10
+	if t.MaxFailures == 0 {
+		fmt.Println("If the number of maximum failures is not specified, use default value of", DefaultMaxFailures)
+		t.MaxFailures = DefaultMaxFailures
+	}
+
+	r := pClient.CreateTask(t.Schedule, t.Workflow, t.Name, t.Deadline, !ctx.IsSet("no-start"), t.MaxFailures)
 
 	if r.Err != nil {
 		errors := strings.Split(r.Err.Error(), " -- ")
@@ -202,6 +210,7 @@ func createTaskUsingWFManifest(ctx *cli.Context) error {
 
 	// Deadline for a task
 	dl := ctx.String("deadline")
+	failure := uint(ctx.Int("failure"))
 
 	var sch *client.Schedule
 	// None of these mean it is a simple schedule
@@ -270,7 +279,7 @@ func createTaskUsingWFManifest(ctx *cli.Context) error {
 		}
 	}
 	// Create task
-	r := pClient.CreateTask(sch, wf, name, dl, !ctx.IsSet("no-start"))
+	r := pClient.CreateTask(sch, wf, name, dl, !ctx.IsSet("no-start"), failure)
 	if r.Err != nil {
 		errors := strings.Split(r.Err.Error(), " -- ")
 		fmt.Println("Error creating task:")

core/task.go

@@ -117,7 +117,7 @@ func TaskDeadlineDuration(v time.Duration) TaskOption {
 // TaskStopOnFailure sets the tasks stopOnFailure
 // The stopOnFailure is the number of consecutive task failures that will
 // trigger disabling the task
-func OptionStopOnFailure(v uint) TaskOption {
+func OptionStopOnFailure(v int) TaskOption {
 	return func(t Task) TaskOption {
 		previous := t.GetStopOnFailure()
 		t.SetStopOnFailure(v)
@@ -156,11 +156,12 @@ type TaskErrors interface {
 }
 
 type TaskCreationRequest struct {
-	Name     string            `json:"name"`
-	Deadline string            `json:"deadline"`
-	Workflow *wmap.WorkflowMap `json:"workflow"`
-	Schedule Schedule          `json:"schedule"`
-	Start    bool              `json:"start"`
+	Name        string            `json:"name"`
+	Deadline    string            `json:"deadline"`
+	Workflow    *wmap.WorkflowMap `json:"workflow"`
+	Schedule    Schedule          `json:"schedule"`
+	Start       bool              `json:"start"`
+	MaxFailures int               `json:"max-failures"`
 }
 
 // Function used to create a task according to content (1st parameter)

docs/TASKS.md

@@ -15,6 +15,7 @@ The manifest can be divided into two parts: Header and Workflow.
   schedule:
     type: "simple"
     interval: "1s"
+  max-failures: 10
 ```
 
 #### Version
@@ -32,9 +33,15 @@ The schedule describes the schedule type and interval for running the task.  The
         "type": "cron",
         "interval" : "0 30 * * * *"
     },
+    "max-failures": 10,
 ```
 More on cron expressions can be found here: https://godoc.org/github.com/robfig/cron
 
+#### Failure
+By default, snap will disable a task if there is 10 consecutive errors from any plugins within the workflow.  The configuration
+can be changed by specifying the number of failures value in the task header.
+
+
 For more on tasks, visit [`SNAPCTL.md`](SNAPCTL.md).
 
 ### The Workflow

examples/tasks/ceph-file.json

@@ -4,6 +4,7 @@
         "type": "simple",
         "interval": "1s"
     },
+    "max-failure": 10,
     "workflow": {
         "collect": {
             "metrics": {

examples/tasks/distributed-mock-file.json

@@ -4,6 +4,7 @@
         "type": "simple",
         "interval": "1s"
     },
+    "max-failure": 10,
     "workflow": {
         "collect": {
             "metrics": {

examples/tasks/mock-file.json

@@ -4,6 +4,7 @@
         "type": "simple",
         "interval": "1s"
     },
+    "max-failure": 10,
     "workflow": {
         "collect": {
             "metrics": {

examples/tasks/mock-file.yaml

@@ -3,6 +3,7 @@
   schedule: 
     type: "simple"
     interval: "1s"
+  max-failure: 10
   workflow: 
     collect: 
       metrics: 

examples/tasks/psutil-file.yaml

@@ -3,6 +3,7 @@
   schedule:
     type: "simple"
     interval: "1s"
+  max-failure: 10
   workflow:
     collect:
       metrics:

examples/tasks/psutil-file_no-processor.yaml

@@ -3,6 +3,7 @@
   schedule:
     type: "simple"
     interval: "1s"
+  max-failure: 10
   workflow: 
     collect:
       metrics:

examples/tasks/psutil-influx.json

@@ -4,6 +4,7 @@
         "type": "simple",
         "interval": "1s"
     },
+    "max-failure": 10,
     "workflow": {
         "collect": {
             "metrics": {

mgmt/rest/client/task.go

@@ -48,14 +48,15 @@ type Schedule struct {
 // If the startTask flag is true, the newly created task is started after the creation.
 // Otherwise, it's in the Stopped state. CreateTask is accomplished through a POST HTTP JSON request.
 // A ScheduledTask is returned if it succeeds, otherwise an error is returned.
-func (c *Client) CreateTask(s *Schedule, wf *wmap.WorkflowMap, name string, deadline string, startTask bool) *CreateTaskResult {
+func (c *Client) CreateTask(s *Schedule, wf *wmap.WorkflowMap, name string, deadline string, startTask bool, maxFailures int) *CreateTaskResult {
 	t := core.TaskCreationRequest{
 		Schedule: core.Schedule{
 			Type:     s.Type,
 			Interval: s.Interval,
 		},
-		Workflow: wf,
-		Start:    startTask,
+		Workflow:    wf,
+		Start:       startTask,
+		MaxFailures: maxFailures,
 	}
 	// Add start and/or stop timestamps if they exist
 	if s.StartTime != nil {

mgmt/rest/task.go

@@ -43,12 +43,66 @@ var (
 	ErrTaskDisabledNotRunnable = errors.New("Task is disabled. Cannot be started")
 )
 
+<<<<<<< 0995570155aeaa57088aedc64b522204046a9c3c
+=======
+type configItem struct {
+	Key   string      `json:"key"`
+	Value interface{} `json:"value"`
+}
+
+type task struct {
+	ID                 uint64                  `json:"id"`
+	Config             map[string][]configItem `json:"config"`
+	Name               string                  `json:"name"`
+	Deadline           string                  `json:"deadline"`
+	Workflow           wmap.WorkflowMap        `json:"workflow"`
+	Schedule           cschedule.Schedule      `json:"schedule"`
+	CreationTime       time.Time               `json:"creation_timestamp,omitempty"`
+	LastRunTime        time.Time               `json:"last_run_timestamp,omitempty"`
+	HitCount           uint                    `json:"hit_count,omitempty"`
+	MissCount          uint                    `json:"miss_count,omitempty"`
+	FailedCount        uint                    `json:"failed_count,omitempty"`
+	LastFailureMessage string                  `json:"last_failure_message,omitempty"`
+	State              string                  `json:"task_state"`
+	MaxFailure         uint                    `json:"max_failure"`
+}
+
+>>>>>>> issue#967: Customizable number of failures before snap disable the task
 func (s *Server) addTask(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 	task, err := core.CreateTaskFromContent(r.Body, nil, s.mt.CreateTask)
 	if err != nil {
 		respond(500, rbody.FromError(err), w)
 		return
 	}
+<<<<<<< 0995570155aeaa57088aedc64b522204046a9c3c
+=======
+
+	var opts []core.TaskOption
+	if tr.Deadline != "" {
+		dl, err := time.ParseDuration(tr.Deadline)
+		if err != nil {
+			respond(500, rbody.FromError(err), w)
+			return
+		}
+		opts = append(opts, core.TaskDeadlineDuration(dl))
+	}
+
+	if tr.Name != "" {
+		opts = append(opts, core.SetTaskName(tr.Name))
+	}
+	opts = append(opts, core.OptionStopOnFailure(tr.MaxFailures))
+
+	task, errs := s.mt.CreateTask(sch, tr.Workflow, tr.Start, opts...)
+	if errs != nil && len(errs.Errors()) != 0 {
+		var errMsg string
+		for _, e := range errs.Errors() {
+			errMsg = errMsg + e.Error() + " -- "
+		}
+		respond(500, rbody.FromError(errors.New(errMsg[:len(errMsg)-4])), w)
+		return
+	}
+
+>>>>>>> issue#967: Customizable number of failures before snap disable the task
 	taskB := rbody.AddSchedulerTaskFromTask(task)
 	taskB.Href = taskURI(r.Host, task)
 	respond(201, taskB, w)