Skip to content

Commit

Permalink
Separate DataDog event enrichment depending on the payload type (#11)
Browse files Browse the repository at this point in the history
* Pick up the timestamp value from X-Cirrus-Timestamp

* Separate DataDog event enrichment depending on the payload type

* CI: run "go test ./..."

* Enrich build and task's initializer and audit_event's actor
  • Loading branch information
edigaryev authored Aug 1, 2024
1 parent b2e7ec7 commit 3794510
Show file tree
Hide file tree
Showing 14 changed files with 422 additions and 61 deletions.
8 changes: 8 additions & 0 deletions .cirrus.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
container:
image: golang:latest

task:
name: Test
test_script:
- go test ./...

task:
name: Release Binaries
only_if: $CIRRUS_TAG != ''
Expand Down
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,22 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0
github.com/labstack/echo/v4 v4.12.0
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.8.4
go.uber.org/zap v1.27.0
)

require (
github.com/DataDog/zstd v1.5.5 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/labstack/gommon v0.4.2 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
Expand All @@ -33,4 +36,5 @@ require (
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
79 changes: 18 additions & 61 deletions internal/command/datadog/datadog.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"github.com/brpaz/echozap"
payloadpkg "github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload"
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
mapset "github.com/deckarep/golang-set/v2"
"github.com/labstack/echo/v4"
Expand All @@ -33,25 +34,6 @@ var (
ErrSignatureVerificationFailed = errors.New("event signature verification failed")
)

type commonWebhookFields struct {
Action *string
Timestamp *int64
Actor struct {
ID *int64
}
Repository struct {
ID *int64
Owner *string
Name *string
}
Build struct {
ID *int64
}
Task struct {
ID *int64
}
}

func NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "datadog",
Expand Down Expand Up @@ -182,7 +164,23 @@ func processWebhookEvent(
}

// Enrich the event with tags
enrichEventWithTags(body, evt, logger)
var payload payloadpkg.Payload

switch presentedEventType {
case "audit_event":
payload = &payloadpkg.AuditEvent{}
case "build", "task":
payload = &payloadpkg.BuildOrTask{}
}

if payload != nil {
if err = json.Unmarshal(body, payload); err != nil {
logger.Warnf("failed to enrich Datadog event with tags: "+
"failed to parse the webhook event of type %q as JSON: %v", presentedEventType, err)
} else {
payload.Enrich(ctx.Request().Header, evt, logger)
}
}

// Datadog silently discards log events submitted with a
// timestamp that is more than 18 hours in the past, sigh.
Expand Down Expand Up @@ -227,44 +225,3 @@ func verifyEvent(ctx echo.Context, body []byte) error {

return nil
}

func enrichEventWithTags(body []byte, evt *datadogsender.Event, logger *zap.SugaredLogger) {
var commonWebhookFields commonWebhookFields

if err := json.Unmarshal(body, &commonWebhookFields); err != nil {
logger.Warnf("failed to enrich Datadog event with tags: "+
"failed to parse the webhook event as JSON: %v", err)

return
}

if value := commonWebhookFields.Action; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("action:%s", *value))
}

if timestamp := commonWebhookFields.Timestamp; timestamp != nil {
evt.Timestamp = time.UnixMilli(*timestamp).UTC()
}

if value := commonWebhookFields.Actor.ID; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("actor_id:%d", *value))
}

if value := commonWebhookFields.Repository.ID; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("repository_id:%d", *value))
}
if value := commonWebhookFields.Repository.Owner; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("repository_owner:%s", *value))
}
if value := commonWebhookFields.Repository.Name; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("repository_name:%s", *value))
}

if value := commonWebhookFields.Build.ID; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("build_id:%d", *value))
}

if value := commonWebhookFields.Task.ID; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("task_id:%d", *value))
}
}
47 changes: 47 additions & 0 deletions internal/command/datadog/payload/auditevent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package payload

import (
"encoding/json"
"fmt"
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
"go.uber.org/zap"
"net/http"
)

type AuditEvent struct {
Data *string `json:"data"`

Actor struct {
Username *string `json:"username"`
} `json:"actor"`

ActorLocationIP *string `json:"actorLocationIp"`

common
}

func (auditEvent AuditEvent) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) {
auditEvent.common.Enrich(header, evt, logger)

if data := auditEvent.Data; data != nil {
var auditEventData auditEventData

if err := json.Unmarshal([]byte(*data), &auditEventData); err != nil {
logger.Warnf("failed to unmarshal audit event's data: %v", err)

return
} else {
auditEventData.Enrich(header, evt, logger)
}
}

actorUsername := "api"
if value := auditEvent.Actor.Username; value != nil {
actorUsername = *value
}
evt.Tags = append(evt.Tags, fmt.Sprintf("actor_username:%s", actorUsername))

if value := auditEvent.ActorLocationIP; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("actor_location_ip:%s", *value))
}
}
36 changes: 36 additions & 0 deletions internal/command/datadog/payload/auditevent_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package payload_test

import (
"encoding/json"
"github.com/cirruslabs/cirrus-webhooks-server/internal/command/datadog/payload"
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"net/http"
"os"
"path/filepath"
"strconv"
"testing"
"time"
)

func TestEnrichAuditEvent(t *testing.T) {
body, err := os.ReadFile(filepath.Join("testdata", "audit_event.json"))
require.NoError(t, err)

evt := &datadogsender.Event{}

payload := payload.AuditEvent{}
require.NoError(t, json.Unmarshal(body, &payload))
payload.Enrich(http.Header{
"X-Cirrus-Timestamp": []string{strconv.FormatInt(time.Now().UnixMilli(), 10)},
}, evt, zap.S())
require.WithinDuration(t, time.Now(), evt.Timestamp, time.Second)
require.Equal(t, []string{
"action:created",
"type:graphql.mutation",
"data.mutationName:GenerateNewScopedAccessToken",
"actor_username:edigaryev",
"actor_location_ip:1.2.3.4",
}, evt.Tags)
}
28 changes: 28 additions & 0 deletions internal/command/datadog/payload/auditeventdata.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package payload

import (
"fmt"
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
"go.uber.org/zap"
"net/http"
)

type auditEventData struct {
MutationName *string `json:"mutationName"`
BuildID *string `json:"buildId"`
TaskID *string `json:"taskId"`
}

func (auditEventData auditEventData) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) {
if mutationName := auditEventData.MutationName; mutationName != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("data.mutationName:%s", *mutationName))
}

if buildID := auditEventData.BuildID; buildID != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("data.buildId:%s", *buildID))
}

if taskID := auditEventData.TaskID; taskID != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("data.taskId:%s", *taskID))
}
}
69 changes: 69 additions & 0 deletions internal/command/datadog/payload/buildortask.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package payload

import (
"fmt"
"github.com/cirruslabs/cirrus-webhooks-server/internal/datadogsender"
"go.uber.org/zap"
"net/http"
"strings"
)

type BuildOrTask struct {
Build struct {
ID *int64 `json:"id"`
Status *string `json:"status"`
Branch *string `json:"branch"`
PullRequest *int64 `json:"pullRequest"`
User struct {
Username *string `json:"username"`
} `json:"user"`
}
Task struct {
ID *int64 `json:"id"`
Name *string `json:"name"`
Status *string `json:"status"`
InstanceType *string `json:"instanceType"`
UniqueLabels []string `json:"uniqueLabels"`
}

common
}

func (buildOrTask BuildOrTask) Enrich(header http.Header, evt *datadogsender.Event, logger *zap.SugaredLogger) {
buildOrTask.common.Enrich(header, evt, logger)

if value := buildOrTask.Build.ID; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("build_id:%d", *value))
}
if value := buildOrTask.Build.Status; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("build_status:%s", *value))
}
if value := buildOrTask.Build.Branch; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("build_branch:%s", *value))
}
if value := buildOrTask.Build.PullRequest; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("build_pull_request:%d", *value))
}

initializerUsername := "api"
if value := buildOrTask.Build.User.Username; value != nil {
initializerUsername = *value
}
evt.Tags = append(evt.Tags, fmt.Sprintf("initializer_username:%s", initializerUsername))

if value := buildOrTask.Task.ID; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("task_id:%d", *value))
}
if value := buildOrTask.Task.Name; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("task_name:%s", *value))
}
if value := buildOrTask.Task.Status; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("task_status:%s", *value))
}
if value := buildOrTask.Task.InstanceType; value != nil {
evt.Tags = append(evt.Tags, fmt.Sprintf("task_instance_type:%s", *value))
}
if value := buildOrTask.Task.UniqueLabels; len(value) > 0 {
evt.Tags = append(evt.Tags, fmt.Sprintf("task_unique_labels:%s", strings.Join(value, ",")))
}
}
Loading

0 comments on commit 3794510

Please sign in to comment.