From 05be1ecd60feaa1a866be4aa0fdbdf412b9806a1 Mon Sep 17 00:00:00 2001 From: Olivier Cazade Date: Wed, 7 Feb 2024 16:25:00 +0000 Subject: [PATCH] Update flp dependency and update code due to breaking changes --- controllers/flp/flp_pipeline_builder.go | 28 ++++++++--- go.mod | 2 + go.sum | 4 +- .../flowlogs-pipeline/pkg/api/api.go | 33 +++++++------ .../flowlogs-pipeline/pkg/api/encode_otlp.go | 22 ++++----- .../flowlogs-pipeline/pkg/api/encode_prom.go | 10 ++-- .../pkg/api/transform_network.go | 49 ++++++++++++++----- vendor/modules.txt | 2 +- 8 files changed, 97 insertions(+), 53 deletions(-) diff --git a/controllers/flp/flp_pipeline_builder.go b/controllers/flp/flp_pipeline_builder.go index 38fecf90a..e5c0fdffe 100644 --- a/controllers/flp/flp_pipeline_builder.go +++ b/controllers/flp/flp_pipeline_builder.go @@ -47,6 +47,8 @@ func newPipelineBuilder( } } +const openshiftNamespacesPrefixes = "openshift" + func (b *PipelineBuilder) AddProcessorStages() error { lastStage := *b.PipelineBuilderStage lastStage = b.addTransformFilter(lastStage) @@ -57,17 +59,17 @@ func (b *PipelineBuilder) AddProcessorStages() error { // enrich stage (transform) configuration enrichedStage := lastStage.TransformNetwork("enrich", api.TransformNetwork{ Rules: api.NetworkTransformRules{{ - Input: "SrcAddr", - Output: "SrcK8S", - Type: api.AddKubernetesRuleType, + Type: api.AddKubernetesRuleType, Kubernetes: &api.K8sRule{ + Input: "SrcAddr", + Output: "SrcK8S", AddZone: addZone, }, }, { - Input: "DstAddr", - Output: "DstK8S", - Type: api.AddKubernetesRuleType, + Type: api.AddKubernetesRuleType, Kubernetes: &api.K8sRule{ + Input: "DstAddr", + Output: "DstK8S", AddZone: addZone, }, }, { @@ -79,8 +81,18 @@ func (b *PipelineBuilder) AddProcessorStages() error { "SrcAddr", "DstAddr", }, - Output: "K8S_FlowLayer", - InfraPrefix: b.desired.Namespace, + Output: "K8S_FlowLayer", + InfraPrefixes: []string{b.desired.Namespace, openshiftNamespacesPrefixes}, + InfraRefs: []api.K8sReference{ + { + Name: "kubernetes", + Namespace: "default", + }, + { + Name: "openshift", + Namespace: "default", + }, + }, }, }}, DirectionInfo: api.NetworkTransformDirectionInfo{ diff --git a/go.mod b/go.mod index 4fcfe5df4..20f983ae5 100644 --- a/go.mod +++ b/go.mod @@ -84,3 +84,5 @@ require ( ) replace github.com/prometheus/common v0.46.0 => github.com/netobserv/prometheus-common v0.46.0-netobserv + +replace github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 => github.com/OlivierCazade/flowlogs-pipeline v0.0.2-0.20240202153633-a34a92595834 diff --git a/go.sum b/go.sum index 0f601e06e..737f6f227 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OlivierCazade/flowlogs-pipeline v0.0.2-0.20240202153633-a34a92595834 h1:n2tD82BulV+TuOEYEAEjnYT2fA10+7eVZizZU3xCblc= +github.com/OlivierCazade/flowlogs-pipeline v0.0.2-0.20240202153633-a34a92595834/go.mod h1:iXyrdjKesbiZeLC+mT2ma8eFXlrMqRj/o3aOnBV3YAM= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -151,8 +153,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 h1:vMJ6tEETHl81Lbzmvs7z7I7PgvyC6WEzlJym2zSWMpQ= -github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934/go.mod h1:Hfcqa5PFrELNp1B4M9e2+rG3226uOiKMi8KEOYBY0ow= github.com/netobserv/prometheus-common v0.46.0-netobserv h1:gpnbXZtSWBauwlSm0aC//mAwAUIvblW6xqNV4mnY2mE= github.com/netobserv/prometheus-common v0.46.0-netobserv/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= diff --git a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/api.go b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/api.go index 2eb99a58b..0bbc0d725 100644 --- a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/api.go +++ b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/api.go @@ -63,19 +63,22 @@ const ( // Note: items beginning with doc: "## title" are top level items that get divided into sections inside api.md. type API struct { - PromEncode PromEncode `yaml:"prom" doc:"## Prometheus encode API\nFollowing is the supported API format for prometheus encode:\n"` - KafkaEncode EncodeKafka `yaml:"kafka" doc:"## Kafka encode API\nFollowing is the supported API format for kafka encode:\n"` - S3Encode EncodeS3 `yaml:"s3" doc:"## S3 encode API\nFollowing is the supported API format for S3 encode:\n"` - IngestCollector IngestCollector `yaml:"collector" doc:"## Ingest collector API\nFollowing is the supported API format for the NetFlow / IPFIX collector:\n"` - IngestKafka IngestKafka `yaml:"kafka" doc:"## Ingest Kafka API\nFollowing is the supported API format for the kafka ingest:\n"` - IngestGRPCProto IngestGRPCProto `yaml:"grpc" doc:"## Ingest GRPC from Network Observability eBPF Agent\nFollowing is the supported API format for the Network Observability eBPF ingest:\n"` - IngestStdin IngestStdin `yaml:"stdin" doc:"## Ingest Standard Input\nFollowing is the supported API format for the standard input ingest:\n"` - TransformGeneric TransformGeneric `yaml:"generic" doc:"## Transform Generic API\nFollowing is the supported API format for generic transformations:\n"` - TransformFilter TransformFilter `yaml:"filter" doc:"## Transform Filter API\nFollowing is the supported API format for filter transformations:\n"` - TransformNetwork TransformNetwork `yaml:"network" doc:"## Transform Network API\nFollowing is the supported API format for network transformations:\n"` - WriteLoki WriteLoki `yaml:"loki" doc:"## Write Loki API\nFollowing is the supported API format for writing to loki:\n"` - WriteStdout WriteStdout `yaml:"stdout" doc:"## Write Standard Output\nFollowing is the supported API format for writing to standard output:\n"` - ExtractAggregate Aggregates `yaml:"aggregates" doc:"## Aggregate metrics API\nFollowing is the supported API format for specifying metrics aggregations:\n"` - ConnectionTracking ConnTrack `yaml:"conntrack" doc:"## Connection tracking API\nFollowing is the supported API format for specifying connection tracking:\n"` - ExtractTimebased ExtractTimebased `yaml:"timebased" doc:"## Time-based Filters API\nFollowing is the supported API format for specifying metrics time-based filters:\n"` + PromEncode PromEncode `yaml:"prom" doc:"## Prometheus encode API\nFollowing is the supported API format for prometheus encode:\n"` + KafkaEncode EncodeKafka `yaml:"kafka" doc:"## Kafka encode API\nFollowing is the supported API format for kafka encode:\n"` + S3Encode EncodeS3 `yaml:"s3" doc:"## S3 encode API\nFollowing is the supported API format for S3 encode:\n"` + IngestCollector IngestCollector `yaml:"collector" doc:"## Ingest collector API\nFollowing is the supported API format for the NetFlow / IPFIX collector:\n"` + IngestKafka IngestKafka `yaml:"kafka" doc:"## Ingest Kafka API\nFollowing is the supported API format for the kafka ingest:\n"` + IngestGRPCProto IngestGRPCProto `yaml:"grpc" doc:"## Ingest GRPC from Network Observability eBPF Agent\nFollowing is the supported API format for the Network Observability eBPF ingest:\n"` + IngestStdin IngestStdin `yaml:"stdin" doc:"## Ingest Standard Input\nFollowing is the supported API format for the standard input ingest:\n"` + TransformGeneric TransformGeneric `yaml:"generic" doc:"## Transform Generic API\nFollowing is the supported API format for generic transformations:\n"` + TransformFilter TransformFilter `yaml:"filter" doc:"## Transform Filter API\nFollowing is the supported API format for filter transformations:\n"` + TransformNetwork TransformNetwork `yaml:"network" doc:"## Transform Network API\nFollowing is the supported API format for network transformations:\n"` + WriteLoki WriteLoki `yaml:"loki" doc:"## Write Loki API\nFollowing is the supported API format for writing to loki:\n"` + WriteStdout WriteStdout `yaml:"stdout" doc:"## Write Standard Output\nFollowing is the supported API format for writing to standard output:\n"` + ExtractAggregate Aggregates `yaml:"aggregates" doc:"## Aggregate metrics API\nFollowing is the supported API format for specifying metrics aggregations:\n"` + ConnectionTracking ConnTrack `yaml:"conntrack" doc:"## Connection tracking API\nFollowing is the supported API format for specifying connection tracking:\n"` + ExtractTimebased ExtractTimebased `yaml:"timebased" doc:"## Time-based Filters API\nFollowing is the supported API format for specifying metrics time-based filters:\n"` + EncodeOtlpLogs EncodeOtlpLogs `yaml:"otlplogs" doc:"## OpenTelemetry Logs API\nFollowing is the supported API format for writing logs to an OpenTelemetry collector:\n"` + EncodeOtlpMetrics EncodeOtlpMetrics `yaml:"otlpmetrics" doc:"## OpenTelemetry Metrics API\nFollowing is the supported API format for writing metrics to an OpenTelemetry collector:\n"` + EncodeOtlpTraces EncodeOtlpTraces `yaml:"otlptraces" doc:"## OpenTelemetry Traces API\nFollowing is the supported API format for writing traces to an OpenTelemetry collector:\n"` } diff --git a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_otlp.go b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_otlp.go index 4c67878df..86e3c32c0 100644 --- a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_otlp.go +++ b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_otlp.go @@ -18,26 +18,26 @@ package api type EncodeOtlpLogs struct { - *OtlpConnectionInfo + *OtlpConnectionInfo `json:",inline" doc:"OpenTelemetry connection info; includes:"` } type EncodeOtlpTraces struct { - *OtlpConnectionInfo - SpanSplitter []string `yaml:"spanSplitter,omitempty" json:"spanSplitter,omitempty" doc:"separate span for each prefix listed"` + *OtlpConnectionInfo `json:",inline" doc:"OpenTelemetry connection info; includes:"` + SpanSplitter []string `yaml:"spanSplitter,omitempty" json:"spanSplitter,omitempty" doc:"separate span for each prefix listed"` } type EncodeOtlpMetrics struct { - *OtlpConnectionInfo - Prefix string `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix added to each metric name"` - Metrics MetricsItems `yaml:"metrics,omitempty" json:"metrics,omitempty" doc:"list of metric definitions, each includes:"` - PushTimeInterval Duration `yaml:"pushTimeInterval,omitempty" json:"pushTimeInterval,omitempty" doc:"how often should metrics be sent to collector:"` - ExpiryTime Duration `yaml:"expiryTime,omitempty" json:"expiryTime,omitempty" doc:"time duration of no-flow to wait before deleting data item"` + *OtlpConnectionInfo `json:",inline" doc:"OpenTelemetry connection info; includes:"` + Prefix string `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix added to each metric name"` + Metrics MetricsItems `yaml:"metrics,omitempty" json:"metrics,omitempty" doc:"list of metric definitions, each includes:"` + PushTimeInterval Duration `yaml:"pushTimeInterval,omitempty" json:"pushTimeInterval,omitempty" doc:"how often should metrics be sent to collector:"` + ExpiryTime Duration `yaml:"expiryTime,omitempty" json:"expiryTime,omitempty" doc:"time duration of no-flow to wait before deleting data item"` } type OtlpConnectionInfo struct { - Address string `yaml:"address,omitempty" json:"address,omitempty" doc:"endpoint address to expose"` - Port int `yaml:"port,omitempty" json:"port,omitempty" doc:"endpoint port number to expose"` - ConnectionType string `yaml:"connectionType,omitempty" json:"connectionType,omitempty" doc:"interface mechanism: either http or grpc"` + Address string `yaml:"address" json:"address" doc:"endpoint address to expose"` + Port int `yaml:"port" json:"port" doc:"endpoint port number to expose"` + ConnectionType string `yaml:"connectionType" json:"connectionType" doc:"interface mechanism: either http or grpc"` TLS *ClientTLS `yaml:"tls,omitempty" json:"tls,omitempty" doc:"TLS configuration for the endpoint"` Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty" doc:"headers to add to messages (optional)"` } diff --git a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_prom.go b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_prom.go index c635f328a..07213d613 100644 --- a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_prom.go +++ b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/encode_prom.go @@ -23,11 +23,11 @@ type PromTLSConf struct { } type PromEncode struct { - *PromConnectionInfo - Metrics MetricsItems `yaml:"metrics,omitempty" json:"metrics,omitempty" doc:"list of prometheus metric definitions, each includes:"` - Prefix string `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix added to each metric name"` - ExpiryTime Duration `yaml:"expiryTime,omitempty" json:"expiryTime,omitempty" doc:"time duration of no-flow to wait before deleting prometheus data item"` - MaxMetrics int `yaml:"maxMetrics,omitempty" json:"maxMetrics,omitempty" doc:"maximum number of metrics to report (default: unlimited)"` + *PromConnectionInfo `json:",inline,omitempty" doc:"Prometheus connection info (optional); includes:"` + Metrics MetricsItems `yaml:"metrics,omitempty" json:"metrics,omitempty" doc:"list of prometheus metric definitions, each includes:"` + Prefix string `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix added to each metric name"` + ExpiryTime Duration `yaml:"expiryTime,omitempty" json:"expiryTime,omitempty" doc:"time duration of no-flow to wait before deleting prometheus data item"` + MaxMetrics int `yaml:"maxMetrics,omitempty" json:"maxMetrics,omitempty" doc:"maximum number of metrics to report (default: unlimited)"` } type MetricEncodeOperationEnum struct { diff --git a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go index 28167bbbd..1156c6276 100644 --- a/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go +++ b/vendor/github.com/netobserv/flowlogs-pipeline/pkg/api/transform_network.go @@ -63,23 +63,50 @@ func TransformNetworkOperationName(operation string) string { } type NetworkTransformRule struct { - Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` - Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` - Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` - Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"` - Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"` - KubernetesInfra *K8sInfraRule `yaml:"kubernetes_infra,omitempty" json:"kubernetes_infra,omitempty" doc:"Kubernetes infra rule specific configuration"` - Kubernetes *K8sRule `yaml:"kubernetes,omitempty" json:"kubernetes,omitempty" doc:"Kubernetes rule specific configuration"` + Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` + KubernetesInfra *K8sInfraRule `yaml:"kubernetes_infra,omitempty" json:"kubernetes_infra,omitempty" doc:"Kubernetes infra rule configuration"` + Kubernetes *K8sRule `yaml:"kubernetes,omitempty" json:"kubernetes,omitempty" doc:"Kubernetes rule configuration"` + AddSubnet *NetworkAddSubnetRule `yaml:"add_subnet,omitempty" json:"add_subnet,omitempty" doc:"Add subnet rule configuration"` + AddLocation *NetworkGenericRule `yaml:"add_location,omitempty" json:"add_location,omitempty" doc:"Add location rule configuration"` + AddIPCategory *NetworkGenericRule `yaml:"add_ip_category,omitempty" json:"add_ip_category,omitempty" doc:"Add ip category rule configuration"` + AddService *NetworkAddServiceRule `yaml:"add_service,omitempty" json:"add_service,omitempty" doc:"Add service rule configuration"` } type K8sInfraRule struct { - Inputs []string `yaml:"inputs,omitempty" json:"inputs,omitempty" doc:"entry inputs fields"` - Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` - InfraPrefix string `yaml:"infra_prefixes,omitempty" json:"infra_prefixes,omitempty" doc:"Namespace prefixes that will be tagged as infra"` + Inputs []string `yaml:"inputs,omitempty" json:"inputs,omitempty" doc:"entry inputs fields"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + InfraPrefixes []string `yaml:"infra_prefixes,omitempty" json:"infra_prefixes,omitempty" doc:"Namespace prefixes that will be tagged as infra"` + InfraRefs []K8sReference `yaml:"infra_refs,omitempty" json:"infra_refs,omitempty" doc:"Additional object references to be tagged as infra"` +} + +type K8sReference struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the object"` + Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty" doc:"namespace of the object"` } type K8sRule struct { - AddZone bool `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"If true the rule will add the zone"` + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"` + LabelsPrefix string `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input lables, if empty labels will not be copied"` + AddZone bool `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"If true the rule will add the zone"` +} + +type NetworkGenericRule struct { + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` +} + +type NetworkAddSubnetRule struct { + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + SubnetMask string `yaml:"subnet_mask,omitempty" json:"subnet_mask,omitempty" doc:"subnet mask field"` +} + +type NetworkAddServiceRule struct { + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty" doc:"entry protocol field"` } type NetworkTransformDirectionInfo struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index da7ace649..c5765ded1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -112,7 +112,7 @@ github.com/munnerz/goautoneg # github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f ## explicit github.com/mwitkow/go-conntrack -# github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 +# github.com/netobserv/flowlogs-pipeline v0.1.11-0.20240125165124-4289f5a17934 => github.com/OlivierCazade/flowlogs-pipeline v0.0.2-0.20240202153633-a34a92595834 ## explicit; go 1.20 github.com/netobserv/flowlogs-pipeline/pkg/api github.com/netobserv/flowlogs-pipeline/pkg/config