Skip to content

Commit

Permalink
feat: implement compression of config fields on resources
Browse files Browse the repository at this point in the history
Add compression support.

Signed-off-by: Utku Ozdemir <utku.ozdemir@siderolabs.com>
  • Loading branch information
utkuozdemir committed Sep 11, 2024
1 parent 4ed9049 commit e3d46f9
Show file tree
Hide file tree
Showing 86 changed files with 30,627 additions and 960 deletions.
2 changes: 1 addition & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-07-23T12:11:06Z by kres 6d3182c.
# Generated on 2024-09-07T21:30:27Z by kres 8be5fa7.

*
!frontend/src
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-08-16T15:47:13Z by kres 7be2a05.
# Generated on 2024-09-07T21:30:27Z by kres 8be5fa7.

name: default
concurrency:
Expand Down Expand Up @@ -71,7 +71,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down Expand Up @@ -253,7 +253,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down Expand Up @@ -331,7 +331,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down Expand Up @@ -409,7 +409,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down Expand Up @@ -487,7 +487,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down Expand Up @@ -565,7 +565,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down Expand Up @@ -643,7 +643,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-backups-cron.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.

name: e2e-backups-cron
concurrency:
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-scaling-cron.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.

name: e2e-scaling-cron
concurrency:
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-short-cron.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.

name: e2e-short-cron
concurrency:
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-templates-cron.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.

name: e2e-templates-cron
concurrency:
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-upgrades-cron.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.

name: e2e-upgrades-cron
concurrency:
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-workload-proxy-cron.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
#
# Generated on 2024-08-15T23:44:03Z by kres 7be2a05.
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.

name: e2e-workload-proxy-cron
concurrency:
Expand Down Expand Up @@ -54,7 +54,7 @@ jobs:
timeout-minutes: 10
- name: Mask secrets
run: |
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
- name: Set secrets for job
run: |
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
Expand Down
102 changes: 102 additions & 0 deletions client/api/omni/specs/cluster_machine_config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package specs

import (
"errors"

"gopkg.in/yaml.v3"
)

// MarshalJSON implements json.Marshaler interface.
//
// It represents compressed fields as uncompressed in the output.
func (x *ClusterMachineConfigSpec) MarshalJSON() ([]byte, error) {
obj := x.CloneVT()

buffer, err := obj.GetUncompressedData()
if err != nil {
return nil, err
}

defer buffer.Free()

obj.Data = buffer.Data()
obj.CompressedData = nil

return jsonMarshaler.Marshal(obj)
}

// UnmarshalJSON implements json.Unmarshaler interface.
func (x *ClusterMachineConfigSpec) UnmarshalJSON(data []byte) error {
return unmarshalJSON(x, data)
}

// MarshalYAML implements yaml.Marshaler interface.
//
// It represents compressed fields as uncompressed in the output.
func (x *ClusterMachineConfigSpec) MarshalYAML() (any, error) {
obj := x.CloneVT()

buffer, err := obj.GetUncompressedData()
if err != nil {
return nil, err
}

defer buffer.Free()

obj.Data = buffer.Data()
obj.CompressedData = nil

type alias *ClusterMachineConfigSpec // prevent recursion

return alias(obj), nil
}

// UnmarshalYAML implements yaml.Unmarshaler interface.
func (x *ClusterMachineConfigSpec) UnmarshalYAML(node *yaml.Node) error {
type alias ClusterMachineConfigSpec // prevent recursion

aux := (*alias)(x)

return unmarshalYAML(x, aux, node)
}

// GetUncompressedData returns the config data from the ClusterMachineConfigSpec, decompressing it if necessary.
func (x *ClusterMachineConfigSpec) GetUncompressedData(opts ...CompressionOption) (Buffer, error) {
if x == nil {
return newNoOpBuffer(nil), nil
}

if x.CompressedData == nil {
return newNoOpBuffer(x.Data), nil
}

return doDecompress(x.CompressedData, getCompressionConfig(opts))
}

// SetUncompressedData sets the config data in the ClusterMachineConfigSpec, compressing it if requested.
func (x *ClusterMachineConfigSpec) SetUncompressedData(data []byte, opts ...CompressionOption) error {
if x == nil {
return errors.New("ClusterMachineConfigSpec is nil")
}

config := getCompressionConfig(opts)
compress := config.Enabled

if !compress {
x.Data = data
x.CompressedData = nil

return nil
}

compressed := doCompress(data, config)

x.CompressedData = compressed
x.Data = nil

return nil
}
Loading

0 comments on commit e3d46f9

Please sign in to comment.