diff --git a/go.mod b/go.mod index d92ee3d345e..88f3e8bed22 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( github.com/mennanov/fieldmask-utils v1.1.0 github.com/opencontainers/runtime-spec v1.2.0 github.com/pelletier/go-toml v1.9.5 - github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.6.0 github.com/sirupsen/logrus v1.9.3 @@ -149,6 +148,7 @@ require ( github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.48.0 // indirect diff --git a/pkg/api/ops/ops.go b/pkg/api/ops/ops.go index 6a8a6ed459b..29d681dafa8 100644 --- a/pkg/api/ops/ops.go +++ b/pkg/api/ops/ops.go @@ -3,6 +3,12 @@ package ops +import ( + "fmt" + + "github.com/cilium/tetragon/pkg/logger" +) + const ( MSG_OP_UNDEF = 0 // MSG_OP_EXECVE event indicates a process was created. The 'PID' @@ -65,21 +71,36 @@ const ( MsgOpKfreeSkb = 11 MsgOpGenericKprobe = 13 MsgOpGeneric_Tracepoint = 14 + MsgOpGenericUprobe = 15 + MsgOpClone = 23 + MsgOpData = 24 + MsgOpCgroup = 25 + MsgOpLoader = 26 MsgOpTest = 254 ) +var OpCodeStrings = map[OpCode]string{ + MsgOpUndef: "Undef", + MsgOpExecve: "Execve", + MsgOpExit: "Exit", + MsgOpKfreeSkb: "KfreeSkb", + MsgOpGenericKprobe: "GenericKprobe", + MsgOpGeneric_Tracepoint: "GenericTracepoint", + MsgOpGenericUprobe: "GenericUprobe", + MsgOpClone: "Clone", + MsgOpData: "Data", + MsgOpCgroup: "Cgroup", + MsgOpLoader: "Loader", + MsgOpTest: "Test", +} + func (op OpCode) String() string { - return [...]string{ - 0: "Undef", - 5: "Execve", - 7: "Exit", - 13: "GenericKprobe", - 14: "GenericTracepoint", - 23: "Clone", - 24: "Data", - 25: "Cgroup", - 254: "Test", - }[op] + s, ok := OpCodeStrings[op] + if !ok { + logger.GetLogger().WithField("opcode", op).Info("Unknown OpCode. This is a bug, please report it to Tetragon developers.") + return fmt.Sprintf("Unknown(%d)", op) + } + return s } func (op CgroupOpCode) String() string { diff --git a/pkg/eventcache/eventcache.go b/pkg/eventcache/eventcache.go index b5488609c44..a62e32734ea 100644 --- a/pkg/eventcache/eventcache.go +++ b/pkg/eventcache/eventcache.go @@ -141,11 +141,11 @@ func (ec *Cache) handleEvents() { continue } if errors.Is(err, ErrFailedToGetParentInfo) { - eventcachemetrics.ParentInfoError(notify.EventTypeString(event.event)).Inc() + eventcachemetrics.ParentInfoError(notify.EventType(event.event)).Inc() } else if errors.Is(err, ErrFailedToGetProcessInfo) { - eventcachemetrics.ProcessInfoError(notify.EventTypeString(event.event)).Inc() + eventcachemetrics.ProcessInfoError(notify.EventType(event.event)).Inc() } else if errors.Is(err, ErrFailedToGetPodInfo) { - eventcachemetrics.PodInfoError(notify.EventTypeString(event.event)).Inc() + eventcachemetrics.PodInfoError(notify.EventType(event.event)).Inc() } } diff --git a/pkg/grpc/exec/exec.go b/pkg/grpc/exec/exec.go index 3c4bff7b72a..c007476972a 100644 --- a/pkg/grpc/exec/exec.go +++ b/pkg/grpc/exec/exec.go @@ -70,7 +70,7 @@ func GetProcessExec(event *MsgExecveEventUnix, useCache bool) *tetragon.ProcessE } if tetragonProcess.Pid == nil { - eventcachemetrics.EventCacheError("GetProcessExec: nil Process.Pid").Inc() + eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc() return nil } @@ -394,7 +394,7 @@ func GetProcessExit(event *MsgExitEventUnix) *tetragon.ProcessExit { } if tetragonProcess.Pid == nil { - eventcachemetrics.EventCacheError("GetProcessExit: nil Process.Pid").Inc() + eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc() return nil } diff --git a/pkg/grpc/tracing/stats.go b/pkg/grpc/tracing/stats.go index b88691e5896..ada802ea0aa 100644 --- a/pkg/grpc/tracing/stats.go +++ b/pkg/grpc/tracing/stats.go @@ -19,6 +19,14 @@ var ( func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(LoaderStats) + + // Initialize metrics with labels + for _, ty := range LoaderTypeStrings { + LoaderStats.WithLabelValues(ty).Add(0) + } + + // NOTES: + // * Rename process_loader_stats metric (to e.g. process_loader_events_total) and count label (to e.g. event)? } type LoaderType int diff --git a/pkg/grpc/tracing/tracing.go b/pkg/grpc/tracing/tracing.go index 1b2239747b6..aff343b3586 100644 --- a/pkg/grpc/tracing/tracing.go +++ b/pkg/grpc/tracing/tracing.go @@ -335,7 +335,7 @@ func GetProcessKprobe(event *MsgGenericKprobeUnix) *tetragon.ProcessKprobe { } if tetragonProcess.Pid == nil { - eventcachemetrics.EventCacheError("GetProcessKprobe: nil Process.Pid").Inc() + eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc() return nil } @@ -497,7 +497,7 @@ func (msg *MsgGenericTracepointUnix) HandleMessage() *tetragon.GetEventsResponse } if tetragonProcess.Pid == nil { - eventcachemetrics.EventCacheError("GetProcessTracepoint: nil Process.Pid").Inc() + eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc() return nil } @@ -615,28 +615,26 @@ func GetProcessLoader(msg *MsgProcessLoaderUnix) *tetragon.ProcessLoader { tetragonProcess = process.UnsafeGetProcess() } + notifyEvent := &ProcessLoaderNotify{ + ProcessLoader: tetragon.ProcessLoader{ + Process: tetragonProcess, + Path: msg.Path, + Buildid: msg.Buildid, + }, + } + if tetragonProcess.Pid == nil { - eventcachemetrics.EventCacheError("GetProcessLoader: nil Process.Pid").Inc() + eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(notifyEvent)).Inc() return nil } if ec := eventcache.Get(); ec != nil && (ec.Needed(tetragonProcess) || (tetragonProcess.Pid.Value > 1)) { - tetragonEvent := &ProcessLoaderNotify{} - tetragonEvent.Process = tetragonProcess - tetragonEvent.Path = msg.Path - tetragonEvent.Buildid = msg.Buildid - ec.Add(nil, tetragonEvent, msg.Msg.Common.Ktime, msg.Msg.ProcessKey.Ktime, msg) + ec.Add(nil, notifyEvent, msg.Msg.Common.Ktime, msg.Msg.ProcessKey.Ktime, msg) return nil } - tetragonEvent := &tetragon.ProcessLoader{ - Process: tetragonProcess, - Path: msg.Path, - Buildid: msg.Buildid, - } - - return tetragonEvent + return ¬ifyEvent.ProcessLoader } func (msg *MsgProcessLoaderUnix) Notify() bool { @@ -735,7 +733,7 @@ func GetProcessUprobe(event *MsgGenericUprobeUnix) *tetragon.ProcessUprobe { } if tetragonProcess.Pid == nil { - eventcachemetrics.EventCacheError("GetProcessUprobe: nil Process.Pid").Inc() + eventcachemetrics.EventCacheError(eventcachemetrics.NilProcessPid, notify.EventType(tetragonEvent)).Inc() return nil } diff --git a/pkg/metrics/errormetrics/errormetrics.go b/pkg/metrics/errormetrics/errormetrics.go index c6c24ec0aa6..008dd5fa03c 100644 --- a/pkg/metrics/errormetrics/errormetrics.go +++ b/pkg/metrics/errormetrics/errormetrics.go @@ -5,32 +5,62 @@ package errormetrics import ( "fmt" - "strings" + "github.com/cilium/tetragon/pkg/api/ops" "github.com/cilium/tetragon/pkg/metrics/consts" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) -type ErrorType string +type ErrorType int -var ( +const ( // Process not found on get() call. - ProcessCacheMissOnGet ErrorType = "process_cache_miss_on_get" + ProcessCacheMissOnGet ErrorType = iota // Process evicted from the cache. - ProcessCacheEvicted ErrorType = "process_cache_evicted" + ProcessCacheEvicted // Process not found on remove() call. - ProcessCacheMissOnRemove ErrorType = "process_cache_miss_on_remove" + ProcessCacheMissOnRemove // Tid and Pid mismatch that could affect BPF and user space caching logic - ProcessPidTidMismatch ErrorType = "process_pid_tid_mismatch" + ProcessPidTidMismatch // An event is missing process info. - EventMissingProcessInfo ErrorType = "event_missing_process_info" + EventMissingProcessInfo // An error occurred in an event handler. - HandlerError ErrorType = "handler_error" + HandlerError // An event finalizer on Process failed - EventFinalizeProcessInfoFailed ErrorType = "event_finalize_process_info_failed" + EventFinalizeProcessInfoFailed +) + +var errorTypeLabelValues = map[ErrorType]string{ + ProcessCacheMissOnGet: "process_cache_miss_on_get", + ProcessCacheEvicted: "process_cache_evicted", + ProcessCacheMissOnRemove: "process_cache_miss_on_remove", + ProcessPidTidMismatch: "process_pid_tid_mismatch", + EventMissingProcessInfo: "event_missing_process_info", + HandlerError: "handler_error", + EventFinalizeProcessInfoFailed: "event_finalize_process_info_failed", +} + +func (e ErrorType) String() string { + return errorTypeLabelValues[e] +} + +type EventHandlerError int + +// TODO: Recognize different errors returned by individual handlers +const ( + HandlePerfUnknownOp EventHandlerError = iota + HandlePerfHandlerError ) +var eventHandlerErrorLabelValues = map[EventHandlerError]string{ + HandlePerfUnknownOp: "unknown_opcode", + HandlePerfHandlerError: "event_handler_failed", +} + +func (e EventHandlerError) String() string { + return eventHandlerErrorLabelValues[e] +} + var ( ErrorTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: consts.MetricsNamespace, @@ -50,24 +80,44 @@ var ( func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(ErrorTotal) registry.MustRegister(HandlerErrors) + + // Initialize metrics with labels + for er := range errorTypeLabelValues { + GetErrorTotal(er).Add(0) + } + for opcode := range ops.OpCodeStrings { + if opcode != ops.MsgOpUndef && opcode != ops.MsgOpTest { + GetHandlerErrors(opcode, HandlePerfHandlerError).Add(0) + } + } + // NB: We initialize only ops.MsgOpUndef here, but unknown_opcode can occur for any opcode + // that is not explicitly handled. + GetHandlerErrors(ops.MsgOpUndef, HandlePerfUnknownOp).Add(0) + + // NOTES: + // * op, msg_op, opcode - standardize on a label (+ add human-readable label) + // * error, error_type, type - standardize on a label + // * Delete errors_total{type="handler_error"} - it duplicates handler_errors_total + // * Consider further splitting errors_total + // * Rename handler_errors_total to event_handler_errors_total? } // Get a new handle on an ErrorTotal metric for an ErrorType -func GetErrorTotal(t ErrorType) prometheus.Counter { - return ErrorTotal.WithLabelValues(string(t)) +func GetErrorTotal(er ErrorType) prometheus.Counter { + return ErrorTotal.WithLabelValues(er.String()) } // Increment an ErrorTotal for an ErrorType -func ErrorTotalInc(t ErrorType) { - GetErrorTotal(t).Inc() +func ErrorTotalInc(er ErrorType) { + GetErrorTotal(er).Inc() } // Get a new handle on the HandlerErrors metric -func GetHandlerErrors(opcode int, err error) prometheus.Counter { - return HandlerErrors.WithLabelValues(fmt.Sprint(opcode), strings.ReplaceAll(fmt.Sprintf("%T", errors.Cause(err)), "*", "")) +func GetHandlerErrors(opcode ops.OpCode, er EventHandlerError) prometheus.Counter { + return HandlerErrors.WithLabelValues(fmt.Sprint(int32(opcode)), er.String()) } // Increment the HandlerErrors metric -func HandlerErrorsInc(opcode int, err error) { - GetHandlerErrors(opcode, err).Inc() +func HandlerErrorsInc(opcode ops.OpCode, er EventHandlerError) { + GetHandlerErrors(opcode, er).Inc() } diff --git a/pkg/metrics/eventcachemetrics/eventcachemetrics.go b/pkg/metrics/eventcachemetrics/eventcachemetrics.go index 67400cb3a44..93b816bcb22 100644 --- a/pkg/metrics/eventcachemetrics/eventcachemetrics.go +++ b/pkg/metrics/eventcachemetrics/eventcachemetrics.go @@ -4,16 +4,43 @@ package eventcachemetrics import ( + "github.com/cilium/tetragon/api/v1/tetragon" "github.com/cilium/tetragon/pkg/metrics/consts" "github.com/prometheus/client_golang/prometheus" ) +type CacheEntryType int + const ( - ProcessInfo = "process_info" - ParentInfo = "parent_info" - PodInfo = "pod_info" + ProcessInfo CacheEntryType = iota + ParentInfo + PodInfo ) +var cacheEntryTypeLabelValues = map[CacheEntryType]string{ + ProcessInfo: "process_info", + ParentInfo: "parent_info", + PodInfo: "pod_info", +} + +func (t CacheEntryType) String() string { + return cacheEntryTypeLabelValues[t] +} + +type CacheError int + +const ( + NilProcessPid CacheError = iota +) + +var cacheErrorLabelValues = map[CacheError]string{ + NilProcessPid: "nil_process_pid", +} + +func (e CacheError) String() string { + return cacheErrorLabelValues[e] +} + var ( processInfoErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: consts.MetricsNamespace, @@ -38,7 +65,7 @@ var ( Name: "event_cache_errors_total", Help: "The total of errors encountered while fetching process exec information from the cache.", ConstLabels: nil, - }, []string{"error"}) + }, []string{"error", "event_type"}) eventCacheRetriesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: consts.MetricsNamespace, Name: "event_cache_retries_total", @@ -59,29 +86,49 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(eventCacheErrorsTotal) registry.MustRegister(eventCacheRetriesTotal) registry.MustRegister(parentInfoErrors) + + // Initialize metrics with labels + for en := range cacheEntryTypeLabelValues { + EventCacheRetries(en).Add(0) + } + for ev := range tetragon.EventType_name { + if tetragon.EventType(ev) != tetragon.EventType_UNDEF && tetragon.EventType(ev) != tetragon.EventType_TEST { + ProcessInfoError(tetragon.EventType(ev)).Add(0) + PodInfoError(tetragon.EventType(ev)).Add(0) + ParentInfoError(tetragon.EventType(ev)).Add(0) + for er := range cacheErrorLabelValues { + EventCacheError(er, tetragon.EventType(ev)).Add(0) + } + } + } + + // NOTES: + // * error, error_type, type - standardize on a label + // * event, event_type, type - standardize on a label + // * Consider merging event cache errors metrics into one with error, event, entry labels } // Get a new handle on a processInfoErrors metric for an eventType -func ProcessInfoError(eventType string) prometheus.Counter { - return processInfoErrors.WithLabelValues(eventType) +func ProcessInfoError(eventType tetragon.EventType) prometheus.Counter { + return processInfoErrors.WithLabelValues(eventType.String()) } // Get a new handle on a podInfoErrors metric for an eventType -func PodInfoError(eventType string) prometheus.Counter { - return podInfoErrors.WithLabelValues(eventType) +func PodInfoError(eventType tetragon.EventType) prometheus.Counter { + return podInfoErrors.WithLabelValues(eventType.String()) } // Get a new handle on an eventCacheErrorsTotal metric for an error -func EventCacheError(err string) prometheus.Counter { - return eventCacheErrorsTotal.WithLabelValues(err) +func EventCacheError(er CacheError, eventType tetragon.EventType) prometheus.Counter { + return eventCacheErrorsTotal.WithLabelValues(er.String(), eventType.String()) } // Get a new handle on an eventCacheRetriesTotal metric for an entryType -func EventCacheRetries(entryType string) prometheus.Counter { - return eventCacheRetriesTotal.WithLabelValues(entryType) +func EventCacheRetries(entryType CacheEntryType) prometheus.Counter { + return eventCacheRetriesTotal.WithLabelValues(entryType.String()) } // Get a new handle on an processInfoErrors metric for an eventType -func ParentInfoError(eventType string) prometheus.Counter { - return parentInfoErrors.WithLabelValues(eventType) +func ParentInfoError(eventType tetragon.EventType) prometheus.Counter { + return parentInfoErrors.WithLabelValues(eventType.String()) } diff --git a/pkg/metrics/eventmetrics/eventmetrics.go b/pkg/metrics/eventmetrics/eventmetrics.go index f5b0040d0f8..2bf4e5b0d17 100644 --- a/pkg/metrics/eventmetrics/eventmetrics.go +++ b/pkg/metrics/eventmetrics/eventmetrics.go @@ -57,6 +57,16 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(FlagCount) registry.MustRegister(NotifyOverflowedEvents) registry.MustRegister(policyStats.ToProm()) + // custom collectors are registered independently + + // Initialize metrics with labels + for _, v := range exec.FlagStrings { + FlagCount.WithLabelValues(v).Add(0) + } + + // NOTES: + // * op, msg_op, opcode - standardize on a label (+ add human-readable label) + // * event, event_type, type - standardize on a label } func GetProcessInfo(process *tetragon.Process) (binary, pod, workload, namespace string) { diff --git a/pkg/metrics/kprobemetrics/kprobemetrics.go b/pkg/metrics/kprobemetrics/kprobemetrics.go index 2f2adc988f1..eb7df826d25 100644 --- a/pkg/metrics/kprobemetrics/kprobemetrics.go +++ b/pkg/metrics/kprobemetrics/kprobemetrics.go @@ -8,6 +8,22 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +type MergeErrorType int + +const ( + MergeErrorTypeEnter MergeErrorType = iota + MergeErrorTypeExit +) + +var mergeErrorTypeLabelValues = map[MergeErrorType]string{ + MergeErrorTypeEnter: "enter", + MergeErrorTypeExit: "exit", +} + +func (t MergeErrorType) String() string { + return mergeErrorTypeLabelValues[t] +} + var ( MergeErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: consts.MetricsNamespace, @@ -33,18 +49,21 @@ func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(MergeErrors) registry.MustRegister(MergeOkTotal) registry.MustRegister(MergePushed) + + // NOTES: + // * Consider merging ok and errors into one with status label } // Get a new handle on the mergeErrors metric for a current and previous function // name and probe type -func GetMergeErrors(currFn, currType, prevFn, prevType string) prometheus.Counter { - return MergeErrors.WithLabelValues(currFn, currType, prevFn, prevType) +func GetMergeErrors(currFn, prevFn string, currType, prevType MergeErrorType) prometheus.Counter { + return MergeErrors.WithLabelValues(currFn, prevFn, currType.String(), prevType.String()) } // Increment the mergeErrors metric for a current and previous function // name and probe type -func MergeErrorsInc(currFn, currType, prevFn, prevType string) { - GetMergeErrors(currFn, currType, prevFn, prevType).Inc() +func MergeErrorsInc(currFn, prevFn string, currType, prevType MergeErrorType) { + GetMergeErrors(currFn, prevFn, currType, prevType).Inc() } func MergeOkTotalInc() { diff --git a/pkg/metrics/mapmetrics/mapmetrics.go b/pkg/metrics/mapmetrics/mapmetrics.go index 0aeb3bc2e83..987ff40c388 100644 --- a/pkg/metrics/mapmetrics/mapmetrics.go +++ b/pkg/metrics/mapmetrics/mapmetrics.go @@ -24,6 +24,11 @@ var ( func InitMetrics(_ *prometheus.Registry) { // custom collectors are registered independently + + // NOTES: + // * Delete (move/replace) map_drops_total as it's monitoring process cache not maps + // * Rename map_in_use_gauge metric (to e.g. map_entries) and delete total label? + // * Introduce a metric for map capacity } // bpfCollector implements prometheus.Collector. It collects metrics directly from BPF maps. diff --git a/pkg/metrics/metricsconfig/initmetrics.go b/pkg/metrics/metricsconfig/initmetrics.go index 2a64f115589..c8070b8586f 100644 --- a/pkg/metrics/metricsconfig/initmetrics.go +++ b/pkg/metrics/metricsconfig/initmetrics.go @@ -13,7 +13,6 @@ import ( "github.com/cilium/tetragon/pkg/metrics/opcodemetrics" "github.com/cilium/tetragon/pkg/metrics/policyfiltermetrics" "github.com/cilium/tetragon/pkg/metrics/policystatemetrics" - "github.com/cilium/tetragon/pkg/metrics/processexecmetrics" "github.com/cilium/tetragon/pkg/metrics/ratelimitmetrics" "github.com/cilium/tetragon/pkg/metrics/ringbufmetrics" "github.com/cilium/tetragon/pkg/metrics/ringbufqueuemetrics" @@ -34,7 +33,6 @@ func InitAllMetrics(registry *prometheus.Registry) { mapmetrics.InitMetrics(registry) opcodemetrics.InitMetrics(registry) policyfiltermetrics.InitMetrics(registry) - processexecmetrics.InitMetrics(registry) ringbufmetrics.InitMetrics(registry) ringbufqueuemetrics.InitMetrics(registry) syscallmetrics.InitMetrics(registry) diff --git a/pkg/metrics/opcodemetrics/opcodemetrics.go b/pkg/metrics/opcodemetrics/opcodemetrics.go index e5d2b397afc..72cb089f4c4 100644 --- a/pkg/metrics/opcodemetrics/opcodemetrics.go +++ b/pkg/metrics/opcodemetrics/opcodemetrics.go @@ -6,6 +6,7 @@ package opcodemetrics import ( "fmt" + "github.com/cilium/tetragon/pkg/api/ops" "github.com/cilium/tetragon/pkg/metrics/consts" "github.com/prometheus/client_golang/prometheus" ) @@ -30,14 +31,26 @@ var ( func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(MsgOpsCount) registry.MustRegister(LatencyStats) + + // Initialize all metrics + for opcode := range ops.OpCodeStrings { + if opcode != ops.MsgOpUndef && opcode != ops.MsgOpTest { + GetOpTotal(opcode).Add(0) + LatencyStats.WithLabelValues(fmt.Sprint(int32(opcode))) + } + } + + // NOTES: + // * op, msg_op, opcode - standardize on a label (+ add human-readable label) + // * Rename handling_latency to handler_latency_microseconds? } // Get a new handle on a msgOpsCount metric for an OpCode -func GetOpTotal(op int) prometheus.Counter { - return MsgOpsCount.WithLabelValues(fmt.Sprint(op)) +func GetOpTotal(opcode ops.OpCode) prometheus.Counter { + return MsgOpsCount.WithLabelValues(fmt.Sprint(int32(opcode))) } // Increment an msgOpsCount for an OpCode -func OpTotalInc(op int) { - GetOpTotal(op).Inc() +func OpTotalInc(opcode ops.OpCode) { + GetOpTotal(opcode).Inc() } diff --git a/pkg/metrics/policyfiltermetrics/policyfiltermetrics.go b/pkg/metrics/policyfiltermetrics/policyfiltermetrics.go index 8c946bfb793..0f3b8693734 100644 --- a/pkg/metrics/policyfiltermetrics/policyfiltermetrics.go +++ b/pkg/metrics/policyfiltermetrics/policyfiltermetrics.go @@ -4,30 +4,72 @@ package policyfiltermetrics import ( - "fmt" - "strings" - "github.com/cilium/tetragon/pkg/metrics/consts" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) +type Subsys int + +const ( + RTHooksSubsys Subsys = iota + PodHandlersSubsys +) + +var subsysLabelValues = map[Subsys]string{ + PodHandlersSubsys: "pod-handlers", + RTHooksSubsys: "rthooks", +} + +func (s Subsys) String() string { + return subsysLabelValues[s] +} + +type Operation int + +const ( + AddPodOperation Operation = iota + UpdatePodOperation + DeletePodOperation + AddContainerOperation +) + +var operationLabelValues = map[Operation]string{ + AddPodOperation: "add", + UpdatePodOperation: "update", + DeletePodOperation: "delete", + AddContainerOperation: "add-container", +} + +func (s Operation) String() string { + return operationLabelValues[s] +} + var ( PolicyFilterOpMetrics = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: consts.MetricsNamespace, Name: "policyfilter_metrics_total", Help: "Policy filter metrics. For internal use only.", ConstLabels: nil, - }, []string{"subsys", "op", "error_type"}) + }, []string{"subsys", "op"}) ) func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(PolicyFilterOpMetrics) + + // Initialize metrics with labels + PolicyFilterOpMetrics.WithLabelValues(RTHooksSubsys.String(), AddContainerOperation.String()).Add(0) + PolicyFilterOpMetrics.WithLabelValues(PodHandlersSubsys.String(), AddPodOperation.String()).Add(0) + PolicyFilterOpMetrics.WithLabelValues(PodHandlersSubsys.String(), UpdatePodOperation.String()).Add(0) + PolicyFilterOpMetrics.WithLabelValues(PodHandlersSubsys.String(), DeletePodOperation.String()).Add(0) + + // NOTES: + // * error, error_type, type - standardize on a label + // * Don't confuse op in policyfilter_metrics_total with ops.OpCode + // * Rename policyfilter_metrics_total to get rid of _metrics? } -func OpInc(subsys, op string, err error) { +func OpInc(subsys Subsys, op Operation) { PolicyFilterOpMetrics.WithLabelValues( - subsys, op, - strings.ReplaceAll(fmt.Sprintf("%T", errors.Cause(err)), "*", ""), + subsys.String(), op.String(), ).Inc() } diff --git a/pkg/metrics/processexecmetrics/processexecmetrics.go b/pkg/metrics/processexecmetrics/processexecmetrics.go deleted file mode 100644 index 433e635335f..00000000000 --- a/pkg/metrics/processexecmetrics/processexecmetrics.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright Authors of Tetragon - -package processexecmetrics - -import ( - "github.com/cilium/tetragon/pkg/metrics/consts" - "github.com/prometheus/client_golang/prometheus" -) - -var ( - MissingParentErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: consts.MetricsNamespace, - Name: "exec_missing_parent_errors_total", - Help: "The total of times a given parent exec id could not be found in an exec event.", - ConstLabels: nil, - }, []string{"parent_exec_id"}) - SameExecIdErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: consts.MetricsNamespace, - Name: "exec_parent_child_same_id_errors_total", - Help: "The total of times an error occurs due to a parent and child process have the same exec id.", - ConstLabels: nil, - }, []string{"exec_id"}) -) - -func InitMetrics(registry *prometheus.Registry) { - registry.MustRegister(MissingParentErrors) - registry.MustRegister(SameExecIdErrors) -} - -// Get a new handle on the missingParentErrors metric for an execId -func GetMissingParent(execId string) prometheus.Counter { - return MissingParentErrors.WithLabelValues(execId) -} - -// Increment the missingParentErrors metric for an execId -func MissingParentInc(execId string) { - GetMissingParent(execId).Inc() -} - -// Get a new handle on the sameExecIdErrors metric for an execId -func GetSameExecId(execId string) prometheus.Counter { - return SameExecIdErrors.WithLabelValues(execId) -} - -// Increment the sameExecIdErrors metric for an execId -func SameExecIdInc(execId string) { - GetSameExecId(execId).Inc() -} diff --git a/pkg/metrics/syscallmetrics/syscallmetrics.go b/pkg/metrics/syscallmetrics/syscallmetrics.go index 9de85efd0d9..bf5725fbcb2 100644 --- a/pkg/metrics/syscallmetrics/syscallmetrics.go +++ b/pkg/metrics/syscallmetrics/syscallmetrics.go @@ -22,6 +22,9 @@ var ( func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(syscallStats.ToProm()) + + // NOTES: + // * Delete syscalls_total? It seems to duplicate policy_events_total. } func Handle(event interface{}) { diff --git a/pkg/metrics/watchermetrics/watchermetrics.go b/pkg/metrics/watchermetrics/watchermetrics.go index aae32f22473..2cdd1abb2a0 100644 --- a/pkg/metrics/watchermetrics/watchermetrics.go +++ b/pkg/metrics/watchermetrics/watchermetrics.go @@ -8,6 +8,22 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +type Watcher int + +// TODO: Having only one watcher type, "k8s", makes it not a useful label. +// Maybe "pod" would be more informative. +const ( + K8sWatcher Watcher = iota +) + +var watcherTypeLabelValues = map[Watcher]string{ + K8sWatcher: "k8s", +} + +func (w Watcher) String() string { + return watcherTypeLabelValues[w] +} + type ErrorType string const ( @@ -32,14 +48,21 @@ var ( func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(WatcherErrors) registry.MustRegister(WatcherEvents) + + // Initialize metrics with labels + GetWatcherEvents(K8sWatcher).Add(0) + GetWatcherErrors(K8sWatcher, FailedToGetPodError).Add(0) + + // NOTES: + // * error, error_type, type - standardize on a label } // Get a new handle on an WatcherEvents metric for a watcher type -func GetWatcherEvents(watcherType string) prometheus.Counter { - return WatcherEvents.WithLabelValues(watcherType) +func GetWatcherEvents(watcherType Watcher) prometheus.Counter { + return WatcherEvents.WithLabelValues(watcherType.String()) } // Get a new handle on an WatcherEvents metric for a watcher type -func GetWatcherErrors(watcherType string, watcherError ErrorType) prometheus.Counter { - return WatcherErrors.WithLabelValues(watcherType, string(watcherError)) +func GetWatcherErrors(watcherType Watcher, watcherError ErrorType) prometheus.Counter { + return WatcherErrors.WithLabelValues(watcherType.String(), string(watcherError)) } diff --git a/pkg/observer/data_stats.go b/pkg/observer/data_stats.go index f967058e26d..2d9832687d3 100644 --- a/pkg/observer/data_stats.go +++ b/pkg/observer/data_stats.go @@ -8,6 +8,22 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +type DataEventOp int + +const ( + DataEventOpOk DataEventOp = iota + DataEventOpBad +) + +var dataEventStrings = map[DataEventOp]string{ + DataEventOpOk: "ok", + DataEventOpBad: "bad", +} + +func (e DataEventOp) String() string { + return dataEventStrings[e] +} + var ( // Define a counter metric for data event statistics DataEventStats = prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -29,6 +45,17 @@ var ( func InitMetrics(registry *prometheus.Registry) { registry.MustRegister(DataEventStats) registry.MustRegister(DataEventSizeHist) + + // Initialize metrics with labels + for _, ev := range DataEventTypeStrings { + DataEventStats.WithLabelValues(ev).Add(0) + } + DataEventSizeHist.WithLabelValues(DataEventOpOk.String()) + DataEventSizeHist.WithLabelValues(DataEventOpBad.String()) + + // NOTES: + // * Don't confuse op in data_event_size with ops.OpCode + // * Don't confuse event in data_events_total with tetragon.EventType } type DataEventType int @@ -57,9 +84,9 @@ func DataEventMetricInc(event DataEventType) { } func DataEventMetricSizeOk(size uint32) { - DataEventSizeHist.WithLabelValues("ok").Observe(float64(size)) + DataEventSizeHist.WithLabelValues(DataEventOpOk.String()).Observe(float64(size)) } func DataEventMetricSizeBad(size uint32) { - DataEventSizeHist.WithLabelValues("bad").Observe(float64(size)) + DataEventSizeHist.WithLabelValues(DataEventOpBad.String()).Observe(float64(size)) } diff --git a/pkg/observer/observer.go b/pkg/observer/observer.go index 3d180af941d..b05dd2165f7 100644 --- a/pkg/observer/observer.go +++ b/pkg/observer/observer.go @@ -17,6 +17,7 @@ import ( "github.com/cilium/ebpf" "github.com/cilium/ebpf/perf" + "github.com/cilium/tetragon/pkg/api/ops" "github.com/cilium/tetragon/pkg/api/readyapi" "github.com/cilium/tetragon/pkg/bpf" "github.com/cilium/tetragon/pkg/logger" @@ -76,47 +77,44 @@ func (k *Observer) RemoveListener(listener Listener) { } } -type handlePerfUnknownOp struct { - op byte +type HandlePerfError struct { + kind errormetrics.EventHandlerError + err error + opcode byte } -func (e handlePerfUnknownOp) Error() string { - return fmt.Sprintf("unknown op: %d", e.op) +func (e *HandlePerfError) Error() string { + return e.err.Error() } -type handlePerfHandlerErr struct { - op byte - err error -} - -func (e *handlePerfHandlerErr) Error() string { - return fmt.Sprintf("handler for op %d failed: %s", e.op, e.err) -} - -func (e *handlePerfHandlerErr) Unwrap() error { - return e.err -} - -func (e *handlePerfHandlerErr) Cause() error { +func (e *HandlePerfError) Unwrap() error { return e.err } // HandlePerfData returns the events from raw bytes // NB: It is made public so that it can be used in testing. -func HandlePerfData(data []byte) (byte, []Event, error) { +func HandlePerfData(data []byte) (byte, []Event, *HandlePerfError) { op := data[0] r := bytes.NewReader(data) // These ops handlers are registered by RegisterEventHandlerAtInit(). handler, ok := eventHandler[op] if !ok { - return op, nil, handlePerfUnknownOp{op: op} + return op, nil, &HandlePerfError{ + kind: errormetrics.HandlePerfUnknownOp, + err: fmt.Errorf("unknown op: %d", op), + opcode: op, + } } events, err := handler(r) if err != nil { - err = &handlePerfHandlerErr{op: op, err: err} + return op, events, &HandlePerfError{ + kind: errormetrics.HandlePerfHandlerError, + err: fmt.Errorf("handler for op %d failed: %w", op, err), + opcode: op, + } } - return op, events, err + return op, events, nil } func (k *Observer) receiveEvent(data []byte) { @@ -126,18 +124,16 @@ func (k *Observer) receiveEvent(data []byte) { } op, events, err := HandlePerfData(data) - opcodemetrics.OpTotalInc(int(op)) + opcodemetrics.OpTotalInc(ops.OpCode(op)) if err != nil { // Increment error metrics errormetrics.ErrorTotalInc(errormetrics.HandlerError) - errormetrics.HandlerErrorsInc(int(op), err) - switch e := err.(type) { - case handlePerfUnknownOp: - k.log.WithField("opcode", e.op).Debug("unknown opcode ignored") - case *handlePerfHandlerErr: - k.log.WithError(e.err).WithField("opcode", e.op).Debug("error occurred in event handler") + errormetrics.HandlerErrorsInc(ops.OpCode(op), err.kind) + switch err.kind { + case errormetrics.HandlePerfUnknownOp: + k.log.WithField("opcode", err.opcode).Debug("unknown opcode ignored") default: - k.log.WithError(err).Debug("error occurred in event handler") + k.log.WithError(err).WithField("opcode", err.opcode).Debug("error occurred in event handler") } } for _, event := range events { diff --git a/pkg/policyfilter/rthooks/rthooks.go b/pkg/policyfilter/rthooks/rthooks.go index 5eb5524dbd8..69d9542ad5e 100644 --- a/pkg/policyfilter/rthooks/rthooks.go +++ b/pkg/policyfilter/rthooks/rthooks.go @@ -103,7 +103,7 @@ func createContainerHook(_ context.Context, arg *rthooks.CreateContainerArg) err if err := pfState.AddPodContainer(policyfilter.PodID(podID), namespace, pod.Labels, containerID, cgid); err != nil { log.WithError(err).Warn("failed to update policy filter, aborting hook.") } - policyfiltermetrics.OpInc("rthooks", "add-container", err) + policyfiltermetrics.OpInc(policyfiltermetrics.RTHooksSubsys, policyfiltermetrics.AddContainerOperation) return nil } diff --git a/pkg/policyfilter/state.go b/pkg/policyfilter/state.go index e032c783b59..1f29c5c3e60 100644 --- a/pkg/policyfilter/state.go +++ b/pkg/policyfilter/state.go @@ -284,7 +284,7 @@ func (m *state) updatePodHandler(pod *v1.Pod) error { "pod-id": podID, "container-ids": containerIDs, "namespace": namespace, - }).Warn("policyfilter, add pod-handler: AddPodContainer failed") + }).Warn("policyfilter: UpdatePod failed") return err } @@ -299,17 +299,17 @@ func (m *state) getPodEventHandlers() cache.ResourceEventHandlerFuncs { logger.GetLogger().Warn("policyfilter, add-pod handler: unexpected object type: %T", pod) return } - err := m.updatePodHandler(pod) - policyfiltermetrics.OpInc("pod-handlers", "add", err) + m.updatePodHandler(pod) + policyfiltermetrics.OpInc(policyfiltermetrics.PodHandlersSubsys, policyfiltermetrics.AddPodOperation) }, UpdateFunc: func(_, newObj interface{}) { pod, ok := newObj.(*v1.Pod) if !ok { - logger.GetLogger().Warn("policyfilter, update-pod: unexpected object type(s): new:%T", pod) + logger.GetLogger().Warn("policyfilter, update-pod handler: unexpected object type(s): new:%T", pod) return } - err := m.updatePodHandler(pod) - policyfiltermetrics.OpInc("pod-handlers", "update", err) + m.updatePodHandler(pod) + policyfiltermetrics.OpInc(policyfiltermetrics.PodHandlersSubsys, policyfiltermetrics.UpdatePodOperation) }, DeleteFunc: func(obj interface{}) { // Remove all containers for this pod @@ -332,7 +332,7 @@ func (m *state) getPodEventHandlers() cache.ResourceEventHandlerFuncs { "namespace": namespace, }).Warn("policyfilter, delete-pod handler: DelPod failed") } - policyfiltermetrics.OpInc("pod-handlers", "delete", err) + policyfiltermetrics.OpInc(policyfiltermetrics.PodHandlersSubsys, policyfiltermetrics.DeletePodOperation) }, } } diff --git a/pkg/process/podinfo.go b/pkg/process/podinfo.go index 00e05b53004..f15bbc0af5f 100644 --- a/pkg/process/podinfo.go +++ b/pkg/process/podinfo.go @@ -43,7 +43,7 @@ func getPodInfo( } pod, container, ok := w.FindContainer(containerID) if !ok { - watchermetrics.GetWatcherErrors("k8s", watchermetrics.FailedToGetPodError).Inc() + watchermetrics.GetWatcherErrors(watchermetrics.K8sWatcher, watchermetrics.FailedToGetPodError).Inc() logger.GetLogger().WithField("container id", containerID).Trace("failed to get pod") return nil } @@ -63,7 +63,7 @@ func getPodInfo( } } workloadObject, workloadType := GetWorkloadMetaFromPod(pod) - watchermetrics.GetWatcherEvents("k8s").Inc() + watchermetrics.GetWatcherEvents(watchermetrics.K8sWatcher).Inc() return &tetragon.Pod{ Namespace: pod.Namespace, Workload: workloadObject.Name, diff --git a/pkg/reader/exec/exec.go b/pkg/reader/exec/exec.go index c3cd5b8844b..3df72d92e48 100644 --- a/pkg/reader/exec/exec.go +++ b/pkg/reader/exec/exec.go @@ -10,71 +10,64 @@ import ( "golang.org/x/sys/unix" ) +// TODO: Harmonize these with the API docs (Flags field in tetragon.Process) +var FlagStrings = map[uint32]string{ + api.EventExecve: "execve", + // nolint We still want to support this even though it's deprecated + api.EventExecveAt: "execveat", + api.EventProcFS: "procFS", + api.EventTruncFilename: "truncFilename", + api.EventTruncArgs: "truncArgs", + api.EventTaskWalk: "taskWalk", + api.EventMiss: "miss", + api.EventNeedsAUID: "auid", + api.EventErrorFilename: "errorFilename", + api.EventErrorArgs: "errorArgs", + api.EventNoCWDSupport: "nocwd", + api.EventRootCWD: "rootcwd", + api.EventErrorCWD: "errorCWD", + api.EventClone: "clone", + api.EventErrorCgroupName: "errorCgroupName", + api.EventErrorCgroupId: "errorCgroupID", + api.EventErrorCgroupKn: "errorCgroupKn", + api.EventErrorCgroupSubsysCgrp: "errorCgroupSubsysCgrp", + api.EventErrorCgroupSubsys: "errorCgroupSubsys", + api.EventErrorCgroups: "errorCgroups", + api.EventErrorPathComponents: "errorPathResolutionCwd", +} + +var flagsOrdered = []uint32{ + api.EventExecve, + // nolint We still want to support this even though it's deprecated + api.EventExecveAt, + api.EventProcFS, + api.EventTruncFilename, + api.EventTruncArgs, + api.EventTaskWalk, + api.EventMiss, + api.EventNeedsAUID, + api.EventErrorFilename, + api.EventErrorArgs, + api.EventNoCWDSupport, + api.EventRootCWD, + api.EventErrorCWD, + api.EventClone, + api.EventErrorCgroupName, + api.EventErrorCgroupId, + api.EventErrorCgroupKn, + api.EventErrorCgroupSubsysCgrp, + api.EventErrorCgroupSubsys, + api.EventErrorCgroups, + api.EventErrorPathComponents, +} + func DecodeCommonFlags(flags uint32) []string { var s []string - if (flags & api.EventExecve) != 0 { - s = append(s, "execve") - } - // nolint We still want to support this even though it's deprecated - if (flags & api.EventExecveAt) != 0 { - s = append(s, "execveat") - } - if (flags & api.EventProcFS) != 0 { - s = append(s, "procFS") - } - if (flags & api.EventTruncFilename) != 0 { - s = append(s, "truncFilename") - } - if (flags & api.EventTruncArgs) != 0 { - s = append(s, "truncArgs") - } - if (flags & api.EventTaskWalk) != 0 { - s = append(s, "taskWalk") - } - if (flags & api.EventMiss) != 0 { - s = append(s, "miss") - } - if (flags & api.EventNeedsAUID) != 0 { - s = append(s, "auid") - } - if (flags & api.EventErrorFilename) != 0 { - s = append(s, "errorFilename") - } - if (flags & api.EventErrorArgs) != 0 { - s = append(s, "errorArgs") - } - if (flags & api.EventNoCWDSupport) != 0 { - s = append(s, "nocwd") - } - if (flags & api.EventRootCWD) != 0 { - s = append(s, "rootcwd") - } - if (flags & api.EventErrorCWD) != 0 { - s = append(s, "errorCWD") - } - if (flags & api.EventClone) != 0 { - s = append(s, "clone") - } - if (flags & api.EventErrorCgroupName) != 0 { - s = append(s, "errorCgroupName") - } - if (flags & api.EventErrorCgroupId) != 0 { - s = append(s, "errorCgroupID") - } - if (flags & api.EventErrorCgroupKn) != 0 { - s = append(s, "errorCgroupKn") - } - if (flags & api.EventErrorCgroupSubsysCgrp) != 0 { - s = append(s, "errorCgroupSubsysCgrp") - } - if (flags & api.EventErrorCgroupSubsys) != 0 { - s = append(s, "errorCgroupSubsys") - } - if (flags & api.EventErrorCgroups) != 0 { - s = append(s, "errorCgroups") - } - if (flags & api.EventErrorPathComponents) != 0 { - s = append(s, "errorPathResolutionCwd") + for _, f := range flagsOrdered { + v := FlagStrings[f] + if (flags & f) != 0 { + s = append(s, v) + } } return s } diff --git a/pkg/reader/notify/notify.go b/pkg/reader/notify/notify.go index d448274294a..9e70c25e8ec 100644 --- a/pkg/reader/notify/notify.go +++ b/pkg/reader/notify/notify.go @@ -4,9 +4,6 @@ package notify import ( - "fmt" - "strings" - "github.com/cilium/tetragon/api/v1/tetragon" "github.com/cilium/tetragon/pkg/process" ) @@ -27,11 +24,13 @@ type Event interface { Encapsulate() tetragon.IsGetEventsResponse_Event } -func EventTypeString(event Event) string { - // Get the concrete type of the event - ty := fmt.Sprintf("%T", event) - // Take only what comes after the last "." - tys := strings.Split(ty, ".") - ty = tys[len(tys)-1] - return ty +func EventType(event Event) tetragon.EventType { + if event == nil { + return tetragon.EventType_UNDEF + } + eventWrapper := event.Encapsulate() + res := &tetragon.GetEventsResponse{ + Event: eventWrapper, + } + return res.EventType() } diff --git a/pkg/sensors/tracing/generickprobe.go b/pkg/sensors/tracing/generickprobe.go index b2742393b66..a4c30a51960 100644 --- a/pkg/sensors/tracing/generickprobe.go +++ b/pkg/sensors/tracing/generickprobe.go @@ -1170,26 +1170,26 @@ func reportMergeError(curr pendingEvent, prev pendingEvent) { if curr.ev != nil { currFn = curr.ev.FuncName } - currType := "enter" + currType := kprobemetrics.MergeErrorTypeEnter if curr.returnEvent { - currType = "exit" + currType = kprobemetrics.MergeErrorTypeExit } prevFn := "UNKNOWN" if prev.ev != nil { prevFn = prev.ev.FuncName } - prevType := "enter" + prevType := kprobemetrics.MergeErrorTypeEnter if prev.returnEvent { - prevType = "exit" + prevType = kprobemetrics.MergeErrorTypeExit } - kprobemetrics.MergeErrorsInc(currFn, currType, prevFn, prevType) + kprobemetrics.MergeErrorsInc(currFn, prevFn, currType, prevType) logger.GetLogger().WithFields(logrus.Fields{ "currFn": currFn, - "currType": currType, + "currType": currType.String(), "prevFn": prevFn, - "prevType": prevType, + "prevType": prevType.String(), }).Debugf("failed to merge events") } diff --git a/pkg/testutils/perfring/perfring.go b/pkg/testutils/perfring/perfring.go index edd63c13b75..58ee4eb46b7 100644 --- a/pkg/testutils/perfring/perfring.go +++ b/pkg/testutils/perfring/perfring.go @@ -104,9 +104,9 @@ func ProcessEvents(t *testing.T, ctx context.Context, eventFn EventFn, wgStarted break } - _, events, err := observer.HandlePerfData(record.RawSample) - if err != nil { - errChan <- fmt.Errorf("error handling perfring data: %v", err) + _, events, handlerErr := observer.HandlePerfData(record.RawSample) + if handlerErr != nil { + errChan <- fmt.Errorf("error handling perfring data: %v", handlerErr) break } err = loopEvents(events, eventFn, complChecker)