Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NETOBSERV-764: Loki v1beta2 model follow-up #474

Merged
merged 6 commits into from
Nov 1, 2023
Merged

NETOBSERV-764: Loki v1beta2 model follow-up #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2018e03
Loki v1beta2 model follow-up
jotak Oct 23, 2023
08da2c2
fix double backticks
jotak Oct 23, 2023
871680e
Automatically deploy roles for Loki when using LokiStack
jotak Oct 24, 2023
49273cc
Add tests, new loki package
jotak Oct 24, 2023
59d09e5
download and pin yq
jotak Oct 24, 2023
b136d06
Remove lokistack sample ; lokistack mentioned in main sample
jotak Oct 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 21 additions & 21 deletions api/v1alpha1/flowcollector_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (

"github.com/netobserv/network-observability-operator/api/v1beta2"
utilconversion "github.com/netobserv/network-observability-operator/pkg/conversion"
"github.com/netobserv/network-observability-operator/pkg/helper"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiconversion "k8s.io/apimachinery/pkg/conversion"
"sigs.k8s.io/controller-runtime/pkg/conversion"
Expand All @@ -41,16 +42,6 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error {
// Manually restore data.
restored := &v1beta2.FlowCollector{}
if ok, err := utilconversion.UnmarshalData(r, restored); err != nil || !ok {
// fallback on current loki config as Manual mode if metadata are not available
dst.Spec.Loki.Mode = v1beta2.LokiModeManual
dst.Spec.Loki.Manual.IngesterURL = r.Spec.Loki.URL
dst.Spec.Loki.Manual.QuerierURL = r.Spec.Loki.QuerierURL
dst.Spec.Loki.Manual.StatusURL = r.Spec.Loki.StatusURL
dst.Spec.Loki.Manual.TenantID = r.Spec.Loki.TenantID
dst.Spec.Loki.Manual.AuthToken = r.Spec.Loki.AuthToken
if err := Convert_v1alpha1_ClientTLS_To_v1beta2_ClientTLS(&r.Spec.Loki.TLS, &dst.Spec.Loki.Manual.TLS, nil); err != nil {
return fmt.Errorf("copying v1alplha1.Loki.TLS into v1beta2.Loki.Manual.TLS: %w", err)
}
return err
}

Expand Down Expand Up @@ -82,17 +73,6 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error {

// Loki
dst.Spec.Loki.Enable = restored.Spec.Loki.Enable
dst.Spec.Loki.Mode = restored.Spec.Loki.Mode
dst.Spec.Loki.Manual = restored.Spec.Loki.Manual
if restored.Spec.Loki.Distributed != nil {
dst.Spec.Loki.Distributed = restored.Spec.Loki.Distributed
}
if restored.Spec.Loki.Monolith != nil {
dst.Spec.Loki.Monolith = restored.Spec.Loki.Monolith
}
if restored.Spec.Loki.LokiStack != nil {
dst.Spec.Loki.LokiStack = restored.Spec.Loki.LokiStack
}

// Exporters
if restored.Spec.Exporters != nil {
Expand Down Expand Up @@ -157,13 +137,33 @@ func Convert_v1beta2_FLPMetrics_To_v1alpha1_FLPMetrics(in *v1beta2.FLPMetrics, o
// we have new defined fields in v1beta2 not in v1alpha1
// nolint:golint,stylecheck,revive
func Convert_v1beta2_FlowCollectorLoki_To_v1alpha1_FlowCollectorLoki(in *v1beta2.FlowCollectorLoki, out *FlowCollectorLoki, s apiconversion.Scope) error {
manual := helper.NewLokiConfig(in)
out.URL = manual.IngesterURL
out.QuerierURL = manual.QuerierURL
out.StatusURL = manual.StatusURL
out.TenantID = manual.TenantID
out.AuthToken = manual.AuthToken
if err := Convert_v1beta2_ClientTLS_To_v1alpha1_ClientTLS(&manual.TLS, &out.TLS, nil); err != nil {
return fmt.Errorf("copying v1beta2.Loki.TLS into v1alpha1.Loki.TLS: %w", err)
}
return autoConvert_v1beta2_FlowCollectorLoki_To_v1alpha1_FlowCollectorLoki(in, out, s)
}

// This function need to be manually created because conversion-gen not able to create it intentionally because
// we have new defined fields in v1beta2 not in v1alpha1
// nolint:golint,stylecheck,revive
func Convert_v1alpha1_FlowCollectorLoki_To_v1beta2_FlowCollectorLoki(in *FlowCollectorLoki, out *v1beta2.FlowCollectorLoki, s apiconversion.Scope) error {
out.Mode = v1beta2.LokiModeManual
out.Manual = v1beta2.LokiManualParams{
IngesterURL: in.URL,
QuerierURL: in.QuerierURL,
StatusURL: in.StatusURL,
TenantID: in.TenantID,
AuthToken: in.AuthToken,
}
if err := Convert_v1alpha1_ClientTLS_To_v1beta2_ClientTLS(&in.TLS, &out.Manual.TLS, nil); err != nil {
return fmt.Errorf("copying v1alpha1.Loki.TLS into v1beta2.Loki.Manual.TLS: %w", err)
}
return autoConvert_v1alpha1_FlowCollectorLoki_To_v1beta2_FlowCollectorLoki(in, out, s)
}

Expand Down
4 changes: 2 additions & 2 deletions api/v1alpha1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

72 changes: 27 additions & 45 deletions api/v1beta1/flowcollector_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,34 +41,9 @@ func (r *FlowCollector) ConvertTo(dstRaw conversion.Hub) error {
// Manually restore data.
restored := &v1beta2.FlowCollector{}
if ok, err := utilconversion.UnmarshalData(r, restored); err != nil || !ok {
// fallback on current loki config as Manual mode if metadata are not available
dst.Spec.Loki.Mode = v1beta2.LokiModeManual
dst.Spec.Loki.Manual.IngesterURL = r.Spec.Loki.URL
dst.Spec.Loki.Manual.QuerierURL = r.Spec.Loki.QuerierURL
dst.Spec.Loki.Manual.StatusURL = r.Spec.Loki.StatusURL
dst.Spec.Loki.Manual.TenantID = r.Spec.Loki.TenantID
dst.Spec.Loki.Manual.AuthToken = r.Spec.Loki.AuthToken
if err := Convert_v1beta1_ClientTLS_To_v1beta2_ClientTLS(&r.Spec.Loki.TLS, &dst.Spec.Loki.Manual.TLS, nil); err != nil {
return fmt.Errorf("copying v1beta1.Loki.TLS into v1beta2.Loki.Manual.TLS: %w", err)
}
if err := Convert_v1beta1_ClientTLS_To_v1beta2_ClientTLS(&r.Spec.Loki.StatusTLS, &dst.Spec.Loki.Manual.StatusTLS, nil); err != nil {
return fmt.Errorf("copying v1beta1.Loki.StatusTLS into v1beta2.Loki.Manual.StatusTLS: %w", err)
}
return err
}

// Loki
dst.Spec.Loki.Mode = restored.Spec.Loki.Mode
dst.Spec.Loki.Manual = restored.Spec.Loki.Manual
if restored.Spec.Loki.Distributed != nil {
dst.Spec.Loki.Distributed = restored.Spec.Loki.Distributed
}
if restored.Spec.Loki.Monolith != nil {
dst.Spec.Loki.Monolith = restored.Spec.Loki.Monolith
}
if restored.Spec.Loki.LokiStack != nil {
dst.Spec.Loki.LokiStack = restored.Spec.Loki.LokiStack
}
dst.Spec.Loki = restored.Spec.Loki

return nil
}
Expand All @@ -83,25 +58,6 @@ func (r *FlowCollector) ConvertFrom(srcRaw conversion.Hub) error {
r.Status.Conditions = make([]v1.Condition, len(src.Status.Conditions))
copy(r.Status.Conditions, src.Status.Conditions)

r.Spec.Loki.URL = helper.LokiIngesterURL(&src.Spec.Loki)
r.Spec.Loki.QuerierURL = helper.LokiQuerierURL(&src.Spec.Loki)
r.Spec.Loki.StatusURL = helper.LokiStatusURL(&src.Spec.Loki)
r.Spec.Loki.TenantID = helper.LokiTenantID(&src.Spec.Loki)
switch src.Spec.Loki.Mode {
case v1beta2.LokiModeManual:
r.Spec.Loki.AuthToken = src.Spec.Loki.Manual.AuthToken
case v1beta2.LokiModeDistributed, v1beta2.LokiModeMonolith:
r.Spec.Loki.AuthToken = ""
case v1beta2.LokiModeLokiStack:
r.Spec.Loki.AuthToken = v1beta2.LokiAuthForwardUserToken
}
if err := Convert_v1beta2_ClientTLS_To_v1beta1_ClientTLS(helper.LokiTLS(&src.Spec.Loki), &r.Spec.Loki.TLS, nil); err != nil {
return fmt.Errorf("copying v1beta2.LokiTLS into v1beta1.LokiTLS: %w", err)
}
if err := Convert_v1beta2_ClientTLS_To_v1beta1_ClientTLS(helper.LokiStatusTLS(&src.Spec.Loki), &r.Spec.Loki.StatusTLS, nil); err != nil {
return fmt.Errorf("copying v1beta2.LokiStatusTLS into v1beta1.LokiStatusTLS: %w", err)
}

// Preserve Hub data on down-conversion except for metadata
return utilconversion.MarshalData(src, r)
}
Expand Down Expand Up @@ -134,12 +90,38 @@ func Convert_v1beta2_FLPMetrics_To_v1beta1_FLPMetrics(in *v1beta2.FLPMetrics, ou
// we have new defined fields in v1beta2 not in v1beta1
// nolint:golint,stylecheck,revive
func Convert_v1beta2_FlowCollectorLoki_To_v1beta1_FlowCollectorLoki(in *v1beta2.FlowCollectorLoki, out *FlowCollectorLoki, s apiconversion.Scope) error {
manual := helper.NewLokiConfig(in)
out.URL = manual.IngesterURL
out.QuerierURL = manual.QuerierURL
out.StatusURL = manual.StatusURL
out.TenantID = manual.TenantID
out.AuthToken = manual.AuthToken
if err := Convert_v1beta2_ClientTLS_To_v1beta1_ClientTLS(&manual.TLS, &out.TLS, nil); err != nil {
return fmt.Errorf("copying Loki v1beta2 TLS into v1beta1 TLS: %w", err)
}
if err := Convert_v1beta2_ClientTLS_To_v1beta1_ClientTLS(&manual.StatusTLS, &out.StatusTLS, nil); err != nil {
return fmt.Errorf("copying Loki v1beta2 StatusTLS into v1beta1 StatusTLS: %w", err)
}
return autoConvert_v1beta2_FlowCollectorLoki_To_v1beta1_FlowCollectorLoki(in, out, s)
}

// This function need to be manually created because conversion-gen not able to create it intentionally because
// we have new defined fields in v1beta2 not in v1beta1
// nolint:golint,stylecheck,revive
func Convert_v1beta1_FlowCollectorLoki_To_v1beta2_FlowCollectorLoki(in *FlowCollectorLoki, out *v1beta2.FlowCollectorLoki, s apiconversion.Scope) error {
out.Mode = v1beta2.LokiModeManual
out.Manual = v1beta2.LokiManualParams{
IngesterURL: in.URL,
QuerierURL: in.QuerierURL,
StatusURL: in.StatusURL,
TenantID: in.TenantID,
AuthToken: in.AuthToken,
}
if err := Convert_v1beta1_ClientTLS_To_v1beta2_ClientTLS(&in.TLS, &out.Manual.TLS, nil); err != nil {
return fmt.Errorf("copying v1beta1.Loki.TLS into v1beta2.Loki.Manual.TLS: %w", err)
}
if err := Convert_v1beta1_ClientTLS_To_v1beta2_ClientTLS(&in.StatusTLS, &out.Manual.StatusTLS, nil); err != nil {
return fmt.Errorf("copying v1beta1.Loki.StatusTLS into v1beta2.Loki.Manual.StatusTLS: %w", err)
}
return autoConvert_v1beta1_FlowCollectorLoki_To_v1beta2_FlowCollectorLoki(in, out, s)
}
4 changes: 2 additions & 2 deletions api/v1beta1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

85 changes: 53 additions & 32 deletions api/v1beta2/flowcollector_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,7 @@ type LokiManualParams struct {
// When using the Loki Operator, set it to `network`, which corresponds to a special tenant mode.
TenantID string `json:"tenantID,omitempty"`

// +kubebuilder:validation:Enum:="DISABLED";"HOST";"FORWARD"
//+kubebuilder:validation:Enum:="DISABLED";"HOST";"FORWARD"
//+kubebuilder:default:="DISABLED"
// `authToken` describes the way to get a token to authenticate to Loki.<br>
// - `DISABLED` does not send any token with the request.<br>
Expand All @@ -562,74 +562,95 @@ type LokiManualParams struct {
StatusTLS ClientTLS `json:"statusTls"`
}

// LokiDistributedParams defines the parameters to connect loki for microservices mode
type LokiDistributedParams struct {
//+kubebuilder:default:="http://loki:3100/"
// LokiMicroservicesParams is the configuration for microservices Loki (https://grafana.com/docs/loki/latest/fundamentals/architecture/deployment-modes/#microservices-mode)
type LokiMicroservicesParams struct {
//+kubebuilder:default:="http://loki-distributor:3100/"
// `ingesterUrl` is the address of an existing Loki ingester service to push the flows to.
IngesterURL string `json:"ingesterUrl,omitempty"`

//+kubebuilder:validation:optional
// `querierURL` specifies the address of the Loki querier service, in case it is different from the
// Loki ingester URL. If empty, the URL value is used (assuming that the Loki ingester
// and querier are in the same server).
//+kubebuilder:default:="http://loki-query-frontend:3100/"
// `querierURL` specifies the address of the Loki querier service.
QuerierURL string `json:"querierUrl,omitempty"`

//+kubebuilder:default:="netobserv"
// `tenantID` is the Loki `X-Scope-OrgID` header that identifies the tenant for each request.
TenantID string `json:"tenantID,omitempty"`

// TLS client configuration for Loki URL.
// +optional
TLS ClientTLS `json:"tls"`
}

// LokiMonolithParams defines the parameters to connect loki for monolithic mode
// LokiMonolithParams is the configuration for monolithic Loki (https://grafana.com/docs/loki/latest/fundamentals/architecture/deployment-modes/#monolithic-mode)
type LokiMonolithParams struct {
//+kubebuilder:default:="http://loki:3100/"
// `url` is the unique address of an existing Loki service that point both ingester and querier.
// `url` is the unique address of an existing Loki service that points to both the ingester and the querier.
URL string `json:"url,omitempty"`

//+kubebuilder:default:="netobserv"
// `tenantID` is the Loki `X-Scope-OrgID` header that identifies the tenant for each request.
TenantID string `json:"tenantID,omitempty"`

// TLS client configuration for Loki URL.
// +optional
TLS ClientTLS `json:"tls"`
}

// LokiStack defines the name and namespace of the loki-operator instance
type LokiStack struct {
// LokiStackRef defines the name and namespace of the LokiStack instance
type LokiStackRef struct {
// Name of an existing LokiStack resource to use.
//+kubebuilder:default:="loki"
//+kubebuilder:validation:Required
Name string `json:"name,omitempty"`
//+kubebuilder:default:="netobserv"

// Namespace where this `LokiStack` resource is located. If omited, it is assumed to be the same as `spec.namespace`.
// +optional
Namespace string `json:"namespace,omitempty"`
}

type LokiMode string

const (
LokiModeManual = "MANUAL"
LokiModeDistributed = "DISTRIBUTED"
LokiModeMonolith = "MONOLITH"
LokiModeLokiStack = "LOKISTACK"
LokiModeManual LokiMode = "Manual"
LokiModeLokiStack LokiMode = "LokiStack"
LokiModeMonolithic LokiMode = "Monolithic"
LokiModeMicroservices LokiMode = "Microservices"
Comment on lines +614 to +617
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why these should be camel case when we use uppercase for agent type, deployment mode, HPA status, Loki auth, SASL, Exporter type ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cf #394 (comment)
We did it wrong on the others.
Now we have a mix, we should eventually make them all consistent, but from now on, like we did on Features, we should adopt the correct convention

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so to avoid having a mix I would suggest to fix everything at once for v1beta2 since it's already an API change. WDYT ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree to do it before releasing v1beta2, but rather on a different PR to not mix it up

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I created a task: https://issues.redhat.com/browse/NETOBSERV-1374
We should add there everything that needs to be cleaned up before we do the first release, ie. any breaking change that we might still find before 1.5

)

// `FlowCollectorLoki` defines the desired state for FlowCollector's Loki client.
type FlowCollectorLoki struct {
//+kubebuilder:validation:Enum=MANUAL;DISTRIBUTED;MONOLITH;LOKISTACK;
//+kubebuilder:default:="MANUAL"
Mode string `json:"mode,omitempty"`

// Loki configuration for MANUAL mode. This is the more flexible configuration.
// It will be ignored for other modes.
// `mode` must be set according to the installation mode of Loki:<br>
// - Use "LokiStack" when Loki is managed using the Loki Operator<br>
// - Use "Monolithic" when Loki is installed as a monolithic workload<br>
// - Use "Microservices" when Loki is installed as microservices, but without Loki Operator<br>
// - Use "Manual" if none of the options above match your setup<br>
//+unionDiscriminator
//+kubebuilder:validation:Enum=Manual;LokiStack;Monolithic;Microservices
//+kubebuilder:default:="Monolithic"
//+kubebuilder:validation:Required
Mode LokiMode `json:"mode,omitempty"`

// Loki configuration for "Manual" mode. This is the most flexible configuration.
// It is ignored for other modes.
// +optional
Manual LokiManualParams `json:"manual,omitempty"`

// Loki configuration for DISTRIBUTED mode. This is usefull for an easy microservices loki config.
// It will be ignored for other mods
// Loki configuration for "Microservices" mode.
// Use this option when Loki is installed using the microservices deployment mode (https://grafana.com/docs/loki/latest/fundamentals/architecture/deployment-modes/#microservices-mode).
// It is ignored for other modes.
// +optional
Distributed *LokiDistributedParams `json:"distributed,omitempty"`
Microservices LokiMicroservicesParams `json:"microservices,omitempty"`

// Loki configuration for MONOLITH mode. This is usefull for an easy monolithic loki config.
// It will be ignored for other mods
// Loki configuration for "Monolithic" mode.
// Use this option when Loki is installed using the monolithic deployment mode (https://grafana.com/docs/loki/latest/fundamentals/architecture/deployment-modes/#monolithic-mode).
// It is ignored for other modes.
// +optional
Monolith *LokiMonolithParams `json:"monolith,omitempty"`
Monolithic LokiMonolithParams `json:"monolithic,omitempty"`

// Loki configuration for LOKISTACK mode. This is usefull for an easy loki-operator config.
// It will be ignored for other mods
// Loki configuration for "LokiStack" mode. This is useful for an easy loki-operator configuration.
// It is ignored for other modes.
// +optional
LokiStack *LokiStack `json:"lokiStack,omitempty"`
LokiStack LokiStackRef `json:"lokiStack,omitempty"`

//+kubebuilder:default:=true
// Set `enable` to `true` to store flows in Loki. It is required for the OpenShift Console plugin installation.
Expand Down
Loading