From 379451098a1be384839b5845ffd16a5435a34ebf Mon Sep 17 00:00:00 2001 From: Nikolay Edigaryev Date: Thu, 1 Aug 2024 18:41:22 +0400 Subject: [PATCH] Separate DataDog event enrichment depending on the payload type (#11) * Pick up the timestamp value from X-Cirrus-Timestamp * Separate DataDog event enrichment depending on the payload type * CI: run "go test ./..." * Enrich build and task's initializer and audit_event's actor --- .cirrus.yml | 8 ++ go.mod | 4 + go.sum | 1 + internal/command/datadog/datadog.go | 79 +++++-------------- .../command/datadog/payload/auditevent.go | 47 +++++++++++ .../datadog/payload/auditevent_test.go | 36 +++++++++ .../command/datadog/payload/auditeventdata.go | 28 +++++++ .../command/datadog/payload/buildortask.go | 69 ++++++++++++++++ .../datadog/payload/buildortask_test.go | 67 ++++++++++++++++ internal/command/datadog/payload/common.go | 58 ++++++++++++++ internal/command/datadog/payload/payload.go | 11 +++ .../datadog/payload/testdata/audit_event.json | 15 ++++ .../datadog/payload/testdata/build.json | 24 ++++++ .../datadog/payload/testdata/task.json | 36 +++++++++ 14 files changed, 422 insertions(+), 61 deletions(-) create mode 100644 internal/command/datadog/payload/auditevent.go create mode 100644 internal/command/datadog/payload/auditevent_test.go create mode 100644 internal/command/datadog/payload/auditeventdata.go create mode 100644 internal/command/datadog/payload/buildortask.go create mode 100644 internal/command/datadog/payload/buildortask_test.go create mode 100644 internal/command/datadog/payload/common.go create mode 100644 internal/command/datadog/payload/payload.go create mode 100644 internal/command/datadog/payload/testdata/audit_event.json create mode 100644 internal/command/datadog/payload/testdata/build.json create mode 100644 internal/command/datadog/payload/testdata/task.json diff --git a/.cirrus.yml b/.cirrus.yml index e87697b..c736044 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,3 +1,11 @@ +container: + image: golang:latest + +task: + name: Test + test_script: + - go test ./... + task: name: Release Binaries only_if: $CIRRUS_TAG != '' diff --git a/go.mod b/go.mod index f62755d..590d748 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,14 @@ require ( github.com/deckarep/golang-set/v2 v2.6.0 github.com/labstack/echo/v4 v4.12.0 github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.27.0 ) require ( github.com/DataDog/zstd v1.5.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -24,6 +26,7 @@ require ( github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect @@ -33,4 +36,5 @@ require ( golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index dac03f2..88b3d16 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/command/datadog/datadog.go b/internal/command/datadog/datadog.go index 2d0c8cb..e24d0db 100644 --- a/internal/command/datadog/datadog.go +++ b/internal/command/datadog/datadog.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "github.com/brpaz/echozap" + payloadpkg "github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload" "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" mapset "github.com/deckarep/golang-set/v2" "github.com/labstack/echo/v4" @@ -33,25 +34,6 @@ var ( ErrSignatureVerificationFailed = errors.New("event signature verification failed") ) -type commonWebhookFields struct { - Action *string - Timestamp *int64 - Actor struct { - ID *int64 - } - Repository struct { - ID *int64 - Owner *string - Name *string - } - Build struct { - ID *int64 - } - Task struct { - ID *int64 - } -} - func NewCommand() *cobra.Command { cmd := &cobra.Command{ Use: "datadog", @@ -182,7 +164,23 @@ func processWebhookEvent( } // Enrich the event with tags - enrichEventWithTags(body, evt, logger) + var payload payloadpkg.Payload + + switch presentedEventType { + case "audit_event": + payload = &payloadpkg.AuditEvent{} + case "build", "task": + payload = &payloadpkg.BuildOrTask{} + } + + if payload != nil { + if err = json.Unmarshal(body, payload); err != nil { + logger.Warnf("failed to enrich Datadog event with tags: "+ + "failed to parse the webhook event of type %q as JSON: %v", presentedEventType, err) + } else { + payload.Enrich(ctx.Request().Header, evt, logger) + } + } // Datadog silently discards log events submitted with a // timestamp that is more than 18 hours in the past, sigh. @@ -227,44 +225,3 @@ func verifyEvent(ctx echo.Context, body []byte) error { return nil } - -func enrichEventWithTags(body []byte, evt *datadogsender.Event, logger *zap.SugaredLogger) { - var commonWebhookFields commonWebhookFields - - if err := json.Unmarshal(body, &commonWebhookFields); err != nil { - logger.Warnf("failed to enrich Datadog event with tags: "+ - "failed to parse the webhook event as JSON: %v", err) - - return - } - - if value := commonWebhookFields.Action; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("action:%s", *value)) - } - - if timestamp := commonWebhookFields.Timestamp; timestamp != nil { - evt.Timestamp = time.UnixMilli(*timestamp).UTC() - } - - if value := commonWebhookFields.Actor.ID; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("actor_id:%d", *value)) - } - - if value := commonWebhookFields.Repository.ID; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("repository_id:%d", *value)) - } - if value := commonWebhookFields.Repository.Owner; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("repository_owner:%s", *value)) - } - if value := commonWebhookFields.Repository.Name; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("repository_name:%s", *value)) - } - - if value := commonWebhookFields.Build.ID; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("build_id:%d", *value)) - } - - if value := commonWebhookFields.Task.ID; value != nil { - evt.Tags = append(evt.Tags, fmt.Sprintf("task_id:%d", *value)) - } -} diff --git a/internal/command/datadog/payload/auditevent.go b/internal/command/datadog/payload/auditevent.go new file mode 100644 index 0000000..efc53a5 --- /dev/null +++ b/internal/command/datadog/payload/auditevent.go @@ -0,0 +1,47 @@ +package payload + +import ( + "encoding/json" + "fmt" + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "go.uber.org/zap" + "net/http" +) + +type AuditEvent struct { + Data *string `json:"data"` + + Actor struct { + Username *string `json:"username"` + } `json:"actor"` + + ActorLocationIP *string `json:"actorLocationIp"` + + common +} + +func (auditEvent AuditEvent) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) { + auditEvent.common.Enrich(header, evt, logger) + + if data := auditEvent.Data; data != nil { + var auditEventData auditEventData + + if err := json.Unmarshal([]byte(*data), &auditEventData); err != nil { + logger.Warnf("failed to unmarshal audit event's data: %v", err) + + return + } else { + auditEventData.Enrich(header, evt, logger) + } + } + + actorUsername := "api" + if value := auditEvent.Actor.Username; value != nil { + actorUsername = *value + } + evt.Tags = append(evt.Tags, fmt.Sprintf("actor_username:%s", actorUsername)) + + if value := auditEvent.ActorLocationIP; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("actor_location_ip:%s", *value)) + } +} diff --git a/internal/command/datadog/payload/auditevent_test.go b/internal/command/datadog/payload/auditevent_test.go new file mode 100644 index 0000000..43e133b --- /dev/null +++ b/internal/command/datadog/payload/auditevent_test.go @@ -0,0 +1,36 @@ +package payload_test + +import ( + "encoding/json" + "github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload" + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" +) + +func TestEnrichAuditEvent(t *testing.T) { + body, err := os.ReadFile(filepath.Join("testdata", "audit_event.json")) + require.NoError(t, err) + + evt := &datadogsender.Event{} + + payload := payload.AuditEvent{} + require.NoError(t, json.Unmarshal(body, &payload)) + payload.Enrich(http.Header{ + "X-Cirrus-Timestamp": []string{strconv.FormatInt(time.Now().UnixMilli(), 10)}, + }, evt, zap.S()) + require.WithinDuration(t, time.Now(), evt.Timestamp, time.Second) + require.Equal(t, []string{ + "action:created", + "type:graphql.mutation", + "data.mutationName:GenerateNewScopedAccessToken", + "actor_username:edigaryev", + "actor_location_ip:1.2.3.4", + }, evt.Tags) +} diff --git a/internal/command/datadog/payload/auditeventdata.go b/internal/command/datadog/payload/auditeventdata.go new file mode 100644 index 0000000..bae0c30 --- /dev/null +++ b/internal/command/datadog/payload/auditeventdata.go @@ -0,0 +1,28 @@ +package payload + +import ( + "fmt" + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "go.uber.org/zap" + "net/http" +) + +type auditEventData struct { + MutationName *string `json:"mutationName"` + BuildID *string `json:"buildId"` + TaskID *string `json:"taskId"` +} + +func (auditEventData auditEventData) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) { + if mutationName := auditEventData.MutationName; mutationName != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("data.mutationName:%s", *mutationName)) + } + + if buildID := auditEventData.BuildID; buildID != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("data.buildId:%s", *buildID)) + } + + if taskID := auditEventData.TaskID; taskID != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("data.taskId:%s", *taskID)) + } +} diff --git a/internal/command/datadog/payload/buildortask.go b/internal/command/datadog/payload/buildortask.go new file mode 100644 index 0000000..3f1e729 --- /dev/null +++ b/internal/command/datadog/payload/buildortask.go @@ -0,0 +1,69 @@ +package payload + +import ( + "fmt" + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "go.uber.org/zap" + "net/http" + "strings" +) + +type BuildOrTask struct { + Build struct { + ID *int64 `json:"id"` + Status *string `json:"status"` + Branch *string `json:"branch"` + PullRequest *int64 `json:"pullRequest"` + User struct { + Username *string `json:"username"` + } `json:"user"` + } + Task struct { + ID *int64 `json:"id"` + Name *string `json:"name"` + Status *string `json:"status"` + InstanceType *string `json:"instanceType"` + UniqueLabels []string `json:"uniqueLabels"` + } + + common +} + +func (buildOrTask BuildOrTask) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) { + buildOrTask.common.Enrich(header, evt, logger) + + if value := buildOrTask.Build.ID; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("build_id:%d", *value)) + } + if value := buildOrTask.Build.Status; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("build_status:%s", *value)) + } + if value := buildOrTask.Build.Branch; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("build_branch:%s", *value)) + } + if value := buildOrTask.Build.PullRequest; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("build_pull_request:%d", *value)) + } + + initializerUsername := "api" + if value := buildOrTask.Build.User.Username; value != nil { + initializerUsername = *value + } + evt.Tags = append(evt.Tags, fmt.Sprintf("initializer_username:%s", initializerUsername)) + + if value := buildOrTask.Task.ID; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("task_id:%d", *value)) + } + if value := buildOrTask.Task.Name; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("task_name:%s", *value)) + } + if value := buildOrTask.Task.Status; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("task_status:%s", *value)) + } + if value := buildOrTask.Task.InstanceType; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("task_instance_type:%s", *value)) + } + if value := buildOrTask.Task.UniqueLabels; len(value) > 0 { + evt.Tags = append(evt.Tags, fmt.Sprintf("task_unique_labels:%s", strings.Join(value, ","))) + } +} diff --git a/internal/command/datadog/payload/buildortask_test.go b/internal/command/datadog/payload/buildortask_test.go new file mode 100644 index 0000000..30a9dd8 --- /dev/null +++ b/internal/command/datadog/payload/buildortask_test.go @@ -0,0 +1,67 @@ +package payload_test + +import ( + "encoding/json" + "github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload" + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" +) + +func TestEnrichBuild(t *testing.T) { + body, err := os.ReadFile(filepath.Join("testdata", "build.json")) + require.NoError(t, err) + + evt := &datadogsender.Event{} + + payload := payload.BuildOrTask{} + require.NoError(t, json.Unmarshal(body, &payload)) + payload.Enrich(http.Header{ + "X-Cirrus-Timestamp": []string{strconv.FormatInt(time.Now().UnixMilli(), 10)}, + }, evt, zap.S()) + require.WithinDuration(t, time.Now(), evt.Timestamp, time.Second) + require.Equal(t, []string{ + "action:updated", + "repository_id:5129885287448576", + "repository_owner:edigaryev", + "repository_name:awesome-system-calls", + "build_id:5082236150611968", + "build_status:EXECUTING", + "build_branch:main", + "initializer_username:edigaryev", + }, evt.Tags) +} + +func TestEnrichTask(t *testing.T) { + body, err := os.ReadFile(filepath.Join("testdata", "task.json")) + require.NoError(t, err) + + evt := &datadogsender.Event{} + + payload := payload.BuildOrTask{} + require.NoError(t, json.Unmarshal(body, &payload)) + payload.Enrich(http.Header{ + "X-Cirrus-Timestamp": []string{strconv.FormatInt(time.Now().UnixMilli(), 10)}, + }, evt, zap.S()) + require.WithinDuration(t, time.Now(), evt.Timestamp, time.Second) + require.Equal(t, []string{ + "action:created", + "repository_id:5129885287448576", + "repository_owner:edigaryev", + "repository_name:awesome-system-calls", + "build_id:5082236150611968", + "build_status:EXECUTING", + "build_branch:main", + "initializer_username:edigaryev", + "task_id:6017965227769856", + "task_name:Lint (cargo fmt)", + "task_status:EXECUTING", + "task_instance_type:CommunityContainer", + }, evt.Tags) +} diff --git a/internal/command/datadog/payload/common.go b/internal/command/datadog/payload/common.go new file mode 100644 index 0000000..9c88a39 --- /dev/null +++ b/internal/command/datadog/payload/common.go @@ -0,0 +1,58 @@ +package payload + +import ( + "fmt" + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "go.uber.org/zap" + "net/http" + "strconv" + "time" +) + +type common struct { + Action *string `json:"action"` + Type *string `json:"type"` + Timestamp *int64 `json:"timestamp"` + Actor struct { + ID *int64 `json:"id"` + } + Repository struct { + ID *int64 `json:"id"` + Owner *string `json:"owner"` + Name *string `json:"name"` + } +} + +func (common common) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) { + if value := common.Action; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("action:%s", *value)) + } + + if t := common.Type; t != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("type:%s", *t)) + } + + if rawTimestamp := header.Get("X-Cirrus-Timestamp"); rawTimestamp != "" { + timestamp, err := strconv.ParseInt(rawTimestamp, 10, 64) + if err != nil { + logger.Warnf("failed to parse \"X-Cirrus-Timestamp\" timestamp value %q: %v", + rawTimestamp, err) + } else { + evt.Timestamp = time.UnixMilli(timestamp) + } + } + + if value := common.Actor.ID; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("actor_id:%d", *value)) + } + + if value := common.Repository.ID; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("repository_id:%d", *value)) + } + if value := common.Repository.Owner; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("repository_owner:%s", *value)) + } + if value := common.Repository.Name; value != nil { + evt.Tags = append(evt.Tags, fmt.Sprintf("repository_name:%s", *value)) + } +} diff --git a/internal/command/datadog/payload/payload.go b/internal/command/datadog/payload/payload.go new file mode 100644 index 0000000..9411248 --- /dev/null +++ b/internal/command/datadog/payload/payload.go @@ -0,0 +1,11 @@ +package payload + +import ( + "github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender" + "go.uber.org/zap" + "net/http" +) + +type Payload interface { + Enrich(http.Header, *datadogsender.Event, *zap.SugaredLogger) +} diff --git a/internal/command/datadog/payload/testdata/audit_event.json b/internal/command/datadog/payload/testdata/audit_event.json new file mode 100644 index 0000000..953b3c1 --- /dev/null +++ b/internal/command/datadog/payload/testdata/audit_event.json @@ -0,0 +1,15 @@ +{ + "id": "bb2bde61-24e6-475c-a8a7-3f03bedcbd61", + "type": "graphql.mutation", + "timestamp": 1722518406287, + "data": "{\n \"mutationName\": \"GenerateNewScopedAccessToken\",\n \"platform\": \"github\",\n \"ownerUid\": \"85709\",\n \"durationSeconds\": 0,\n \"permission\": \"READ\",\n \"repositoryNames\": [\n \"awesome-system-calls\"\n ],\n \"clientMutationId\": \"generate-scoped-token-85709awesome-system-calls\"\n}", + "actor": { + "id": 5702969901449216, + "username": "edigaryev" + }, + "actorLocationIp": "1.2.3.4", + "action": "created", + "repository": {}, + "build": {}, + "task": {} +} diff --git a/internal/command/datadog/payload/testdata/build.json b/internal/command/datadog/payload/testdata/build.json new file mode 100644 index 0000000..3353360 --- /dev/null +++ b/internal/command/datadog/payload/testdata/build.json @@ -0,0 +1,24 @@ +{ + "old_status": "COMPLETED", + "action": "updated", + "repository": { + "id": 5129885287448576, + "owner": "edigaryev", + "name": "awesome-system-calls", + "isPrivate": false + }, + "build": { + "id": 5082236150611968, + "branch": "main", + "durationInSeconds": 18, + "changeIdInRepo": "1a7a425b71fd28274b739c9d410ca966aedbcd63", + "changeTimestamp": 1722406690000, + "changeMessageTitle": "Periodic update (#7)", + "status": "EXECUTING", + "user": { + "id": 5702969901449216, + "username": "edigaryev" + } + }, + "task": {} +} diff --git a/internal/command/datadog/payload/testdata/task.json b/internal/command/datadog/payload/testdata/task.json new file mode 100644 index 0000000..90f5956 --- /dev/null +++ b/internal/command/datadog/payload/testdata/task.json @@ -0,0 +1,36 @@ +{ + "action": "created", + "repository": { + "id": 5129885287448576, + "owner": "edigaryev", + "name": "awesome-system-calls", + "isPrivate": false + }, + "build": { + "id": 5082236150611968, + "branch": "main", + "durationInSeconds": 18, + "changeIdInRepo": "1a7a425b71fd28274b739c9d410ca966aedbcd63", + "changeTimestamp": 1722406690000, + "changeMessageTitle": "Periodic update (#7)", + "status": "EXECUTING", + "user": { + "id": 5702969901449216, + "username": "edigaryev" + } + }, + "task": { + "id": 6017965227769856, + "name": "Lint (cargo fmt)", + "nameAlias": "lint", + "status": "EXECUTING", + "statusTimestamp": 1722408869403, + "creationTimestamp": 1722408865412, + "durationInSeconds": 1, + "uniqueLabels": [], + "automaticReRun": false, + "automaticallyReRunnable": false, + "instanceType": "CommunityContainer", + "notifications": [] + } +}