Add jitter_percent option on fake

Signed-off-by: Xabier Larrakoetxea <me@slok.dev>

Author: slok

plugins/fake/README.md

@@ -7,6 +7,7 @@ You can get more information on the [SRE workbook `Alert on burn rate`][examples
 ## Options
 
 - `burn_rate`: (**Required**) A number that tells the burn rate factor (e.g: `1`, `2`, `10`...).
+- `jitter_percent`: (**Optional**) A percent number that will add/remove jitter on the burned rate.
 
 ## Metric requirements
 
@@ -64,4 +65,15 @@ sli:
       burn_rate: "1000"
 ```
 
+### 1x speed `30d` window, consumed in `30d` using jitter
+
+```yaml
+sli:
+  plugin:
+    id: "sloth-common/fake"
+    options:
+      burn_rate: "1"
+      jitter_percent: "10"
+```
+
 [examples-sre-book]: https://sre.google/workbook/alerting-on-slos/#burn_rates_and_time_to_complete_budget_ex

plugins/fake/plugin.go

@@ -13,6 +13,7 @@ const (
 
 // SLIPlugin will return a query that will fake a burning error budget at the desired speed using
 // `burn_rate` option.
+// The plugins also accepts a `jitter_percent` that will add/remove a jitter in the range of that jitter percent.
 func SLIPlugin(ctx context.Context, meta, labels, options map[string]string) (string, error) {
 	objective, err := getSLOObjective(meta)
 	if err != nil {
@@ -24,16 +25,28 @@ func SLIPlugin(ctx context.Context, meta, labels, options map[string]string) (st
 		return "", fmt.Errorf("could not get burn rate: %w", err)
 	}
 
+	jitter, err := getJitterPercent(options)
+	if err != nil {
+		return "", fmt.Errorf("could not get jitter percent: %w", err)
+	}
+
 	// Get the error budget.
 	errorBudgetPerc := 100 - objective
 	errorBudgetRatio := errorBudgetPerc / 100
 
 	// Apply factor (burn rate) to error budget.
 	expectedSLIError := errorBudgetRatio * burnRate
 
+	// Create regular query and if no jitter required then regular query
 	query := fmt.Sprintf(`max_over_time(vector(%f)[{{.window}}:])`, expectedSLIError)
+	if jitter == 0 {
+		return query, nil
+	}
 
-	return query, nil
+	// Create jitter (this is the best random I could came up with) and rest to the regular query.
+	jitterQuery := fmt.Sprintf(`(%f * (%f - ((time() * minute() * hour() * day_of_week() * month()) %% %f))) / 100`, expectedSLIError, jitter, jitter*2)
+
+	return fmt.Sprintf("(%s) - (%s)", query, jitterQuery), nil
 }
 
 func getBurnRate(options map[string]string) (float64, error) {
@@ -50,6 +63,20 @@ func getBurnRate(options map[string]string) (float64, error) {
 	return burnRate, nil
 }
 
+func getJitterPercent(options map[string]string) (float64, error) {
+	jitterPercentS, ok := options["jitter_percent"]
+	if !ok {
+		return 0, nil
+	}
+
+	jitterPercent, err := strconv.ParseFloat(jitterPercentS, 64)
+	if err != nil {
+		return 0, fmt.Errorf("not a valid jitter_percent, can't parse to float64: %w", err)
+	}
+
+	return jitterPercent, nil
+}
+
 func getSLOObjective(meta map[string]string) (float64, error) {
 	objectiveS, ok := meta["objective"]
 	if !ok {

plugins/fake/plugin_test.go

@@ -64,6 +64,18 @@ func TestSLIPlugin(t *testing.T) {
 			options:  map[string]string{"burn_rate": "0.5"},
 			expQuery: `max_over_time(vector(0.010000)[{{.window}}:])`,
 		},
+
+		"Having objective, burn rate and jitter, should return a correct query (speed 1, +-10%).": {
+			meta:     map[string]string{"objective": "99"},
+			options:  map[string]string{"burn_rate": "1", "jitter_percent": "10"},
+			expQuery: `(max_over_time(vector(0.010000)[{{.window}}:])) - ((0.010000 * (10.000000 - ((time() * minute() * hour() * day_of_week() * month()) % 20.000000))) / 100)`,
+		},
+
+		"Having objective, burn rate and jitter, should return a correct query (speed ~2 +-50%).": {
+			meta:     map[string]string{"objective": "99.9"},
+			options:  map[string]string{"burn_rate": "2", "jitter_percent": "50"},
+			expQuery: `(max_over_time(vector(0.002000)[{{.window}}:])) - ((0.002000 * (50.000000 - ((time() * minute() * hour() * day_of_week() * month()) % 100.000000))) / 100)`,
+		},
 	}
 
 	for name, test := range tests {

scripts/check/integration-test.sh

@@ -6,15 +6,4 @@ IFS=$'\t\n'
 
 command -v sloth >/dev/null 2>&1 || { echo 'please install sloth'; exit 1; }
 
-set +f # Allow asterisk expansion.
-
-# Load all plugins and try generating SLOs without error, for now this is good enough 
-# for integration tests along with each plugin unit tests.
-for file in ./test/integration/*.yml; do
-    fname=$(basename "$file")
-    echo -n "[TEST] [${file}] Generating SLOs..."
-    sloth generate -p ./plugins -i "${file}" --no-log > /dev/null
-    echo ": OK"
-done
-
-set -f
+sloth validate -p ./plugins -i ./test/integration/ --debug

test/integration/fake.yml

@@ -26,3 +26,17 @@ slos:
         disable: true
       ticket_alert:
         disable: true
+
+  - name: "test-jitter"
+    objective: 99
+    sli:
+      plugin:
+        id: "sloth-common/fake"
+        options:
+          burn_rate: "1000"
+          jitter_percent: "10"
+    alerting:
+      page_alert:
+        disable: true
+      ticket_alert:
+        disable: true