Skip to content

Commit 7a1c62b

Browse files
smirasmira
committed
feat: publish installed extensions as node labels/annotations
Extensions are posted the following way: `extensions.talos.dev/<name>=<version>` The name should be valid as a label (annotation) key. If the value is valid as a label value, use labels, otherwise use annotations. Also implements node annotations in the machine config as a side-effect. Fixes #9089 Fixes #8971 See #9070 Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
1 parent 3f2058a commit 7a1c62b

File tree

33 files changed

+1419
-277
lines changed

33 files changed

+1419
-277
lines changed

‎api/resource/definitions/k8s/k8s.proto

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,12 @@ message ManifestStatusSpec {
170170
repeated string manifests_applied = 1;
171171
}
172172

173+
// NodeAnnotationSpecSpec represents an annoation that's attached to a Talos node.
174+
message NodeAnnotationSpecSpec {
175+
string key = 1;
176+
string value = 2;
177+
}
178+
173179
// NodeIPConfigSpec holds the Node IP specification.
174180
message NodeIPConfigSpec {
175181
repeated string valid_subnets = 1;

‎hack/release.toml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,25 @@ Talos Linux on config generation now adds a label `node.kubernetes.io/exclude-fr
108108
title = "Secure Boot"
109109
description = """\
110110
Talos Linux now can optionally include well-known UEFI (Microsoft) SecureBoot keys into the auto-enrollment UEFI database.
111+
"""
112+
113+
[notes.annotations]
114+
title = "Node Annotations"
115+
description = """\
116+
Talos Linux now supports configuring Kubernetes node annotations via machine configuration (`.machine.nodeAnnotations`) in a way similar to node labels.
117+
"""
118+
119+
[notes.kubelet]
120+
title = "Extensions in Kubernetes Nodes"
121+
description = """\
122+
Talos Linux now publishes list of installed extensions as Kubernetes node labels/annotations.
123+
124+
The key format is `extensions.talos.dev/<name>` and the value is the extension version.
125+
If the extension name is not valid as a label key, it will be skipped.
126+
If the extension version is a valid label value, it will be put to the label; otherwise it will be put to the annotation.
127+
128+
For Talos machines booted of the Image Factory artifacts, this means that the schematic ID will be published as the annotation
129+
`extensions.talos.dev/schematic` (as it is longer than 63 characters).
111130
"""
112131

113132
[make_deps]

‎internal/app/machined/pkg/controllers/k8s/node_annotation_spec.go

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
package k8s
6+
7+
import (
8+
"context"
9+
"fmt"
10+
11+
"github.com/cosi-project/runtime/pkg/controller"
12+
"github.com/cosi-project/runtime/pkg/safe"
13+
"github.com/cosi-project/runtime/pkg/state"
14+
"github.com/siderolabs/gen/optional"
15+
"go.uber.org/zap"
16+
17+
"github.com/siderolabs/talos/pkg/machinery/labels"
18+
"github.com/siderolabs/talos/pkg/machinery/resources/config"
19+
"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
20+
"github.com/siderolabs/talos/pkg/machinery/resources/runtime"
21+
)
22+
23+
// NodeAnnotationSpecController manages k8s.NodeAnnotationsConfig based on configuration.
24+
type NodeAnnotationSpecController struct{}
25+
26+
// Name implements controller.Controller interface.
27+
func (ctrl *NodeAnnotationSpecController) Name() string {
28+
return "k8s.NodeAnnotationSpecController"
29+
}
30+
31+
// Inputs implements controller.Controller interface.
32+
func (ctrl *NodeAnnotationSpecController) Inputs() []controller.Input {
33+
return []controller.Input{
34+
{
35+
Namespace: config.NamespaceName,
36+
Type: config.MachineConfigType,
37+
ID: optional.Some(config.V1Alpha1ID),
38+
Kind: controller.InputWeak,
39+
},
40+
{
41+
Namespace: runtime.NamespaceName,
42+
Type: runtime.ExtensionStatusType,
43+
Kind: controller.InputWeak,
44+
},
45+
}
46+
}
47+
48+
// Outputs implements controller.Controller interface.
49+
func (ctrl *NodeAnnotationSpecController) Outputs() []controller.Output {
50+
return []controller.Output{
51+
{
52+
Type: k8s.NodeAnnotationSpecType,
53+
Kind: controller.OutputExclusive,
54+
},
55+
}
56+
}
57+
58+
// Run implements controller.Controller interface.
59+
//
60+
//nolint:gocyclo
61+
func (ctrl *NodeAnnotationSpecController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
62+
for {
63+
select {
64+
case <-ctx.Done():
65+
return nil
66+
case <-r.EventCh():
67+
}
68+
69+
cfg, err := safe.ReaderGetByID[*config.MachineConfig](ctx, r, config.V1Alpha1ID)
70+
if err != nil && !state.IsNotFoundError(err) {
71+
return fmt.Errorf("error getting config: %w", err)
72+
}
73+
74+
r.StartTrackingOutputs()
75+
76+
nodeAnnotations := map[string]string{}
77+
78+
if cfg != nil && cfg.Config().Machine() != nil {
79+
for k, v := range cfg.Config().Machine().NodeAnnotations() {
80+
nodeAnnotations[k] = v
81+
}
82+
}
83+
84+
if err = extensionsToNodeKV(
85+
ctx, r, nodeAnnotations,
86+
func(annotationValue string) bool {
87+
return labels.ValidateLabelValue(annotationValue) != nil
88+
},
89+
); err != nil {
90+
return fmt.Errorf("error converting extensions to node annotations: %w", err)
91+
}
92+
93+
for key, value := range nodeAnnotations {
94+
if err = safe.WriterModify(ctx, r, k8s.NewNodeAnnotationSpec(key), func(k *k8s.NodeAnnotationSpec) error {
95+
k.TypedSpec().Key = key
96+
k.TypedSpec().Value = value
97+
98+
return nil
99+
}); err != nil {
100+
return fmt.Errorf("error updating node label spec: %w", err)
101+
}
102+
}
103+
104+
if err = safe.CleanupOutputs[*k8s.NodeAnnotationSpec](ctx, r); err != nil {
105+
return err
106+
}
107+
}
108+
}

‎internal/app/machined/pkg/controllers/k8s/node_annotation_spec_test.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
package k8s_test
6+
7+
import (
8+
"testing"
9+
"time"
10+
11+
"github.com/cosi-project/runtime/pkg/resource/rtestutils"
12+
"github.com/cosi-project/runtime/pkg/safe"
13+
"github.com/cosi-project/runtime/pkg/state"
14+
"github.com/stretchr/testify/assert"
15+
"github.com/stretchr/testify/suite"
16+
17+
"github.com/siderolabs/talos/internal/app/machined/pkg/controllers/ctest"
18+
k8sctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/k8s"
19+
"github.com/siderolabs/talos/pkg/machinery/config/container"
20+
"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
21+
"github.com/siderolabs/talos/pkg/machinery/extensions"
22+
"github.com/siderolabs/talos/pkg/machinery/resources/config"
23+
"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
24+
"github.com/siderolabs/talos/pkg/machinery/resources/runtime"
25+
)
26+
27+
type NodeAnnotationsSuite struct {
28+
ctest.DefaultSuite
29+
}
30+
31+
func TestNodeAnnotationsSuite(t *testing.T) {
32+
t.Parallel()
33+
34+
suite.Run(t, &NodeAnnotationsSuite{
35+
DefaultSuite: ctest.DefaultSuite{
36+
Timeout: 5 * time.Second,
37+
AfterSetup: func(s *ctest.DefaultSuite) {
38+
s.Require().NoError(s.Runtime().RegisterController(&k8sctrl.NodeAnnotationSpecController{}))
39+
},
40+
},
41+
})
42+
}
43+
44+
func (suite *NodeAnnotationsSuite) updateMachineConfig(annotations map[string]string) {
45+
cfg, err := safe.StateGetByID[*config.MachineConfig](suite.Ctx(), suite.State(), config.V1Alpha1ID)
46+
if err != nil && !state.IsNotFoundError(err) {
47+
suite.Require().NoError(err)
48+
}
49+
50+
if cfg == nil {
51+
cfg = config.NewMachineConfig(container.NewV1Alpha1(&v1alpha1.Config{
52+
MachineConfig: &v1alpha1.MachineConfig{
53+
MachineType: "controlplane",
54+
MachineNodeAnnotations: annotations,
55+
},
56+
}))
57+
58+
suite.Require().NoError(suite.State().Create(suite.Ctx(), cfg))
59+
} else {
60+
cfg.Container().RawV1Alpha1().MachineConfig.MachineNodeAnnotations = annotations
61+
suite.Require().NoError(suite.State().Update(suite.Ctx(), cfg))
62+
}
63+
}
64+
65+
func (suite *NodeAnnotationsSuite) TestChangeLabel() {
66+
// given
67+
expectedAnnotation := "some/annotation"
68+
oldValue := "oldValue"
69+
expectedValue := "newValue"
70+
71+
// when
72+
suite.updateMachineConfig(map[string]string{
73+
expectedAnnotation: oldValue,
74+
})
75+
76+
rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{expectedAnnotation},
77+
func(labelSpec *k8s.NodeAnnotationSpec, asrt *assert.Assertions) {
78+
asrt.Equal(oldValue, labelSpec.TypedSpec().Value)
79+
})
80+
81+
suite.updateMachineConfig(map[string]string{
82+
expectedAnnotation: expectedValue,
83+
})
84+
85+
// then
86+
rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{expectedAnnotation},
87+
func(labelSpec *k8s.NodeAnnotationSpec, asrt *assert.Assertions) {
88+
asrt.Equal(expectedValue, labelSpec.TypedSpec().Value)
89+
})
90+
}
91+
92+
func (suite *NodeAnnotationsSuite) TestExtensionAnnotations() {
93+
ext1 := runtime.NewExtensionStatus(runtime.NamespaceName, "0")
94+
ext1.TypedSpec().Metadata = extensions.Metadata{
95+
Name: "zfs",
96+
Version: "2.2.4",
97+
}
98+
99+
ext2 := runtime.NewExtensionStatus(runtime.NamespaceName, "1")
100+
ext2.TypedSpec().Metadata = extensions.Metadata{
101+
Name: "drbd",
102+
Version: "9.2.8-v1.7.5",
103+
}
104+
105+
ext3 := runtime.NewExtensionStatus(runtime.NamespaceName, "2")
106+
ext3.TypedSpec().Metadata = extensions.Metadata{
107+
Name: "schematic",
108+
Version: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
109+
}
110+
111+
suite.Require().NoError(suite.State().Create(suite.Ctx(), ext1))
112+
suite.Require().NoError(suite.State().Create(suite.Ctx(), ext2))
113+
suite.Require().NoError(suite.State().Create(suite.Ctx(), ext3))
114+
115+
rtestutils.AssertNoResource[*k8s.NodeAnnotationSpec](suite.Ctx(), suite.T(), suite.State(), "extensions.talos.dev/zfs")
116+
rtestutils.AssertNoResource[*k8s.NodeAnnotationSpec](suite.Ctx(), suite.T(), suite.State(), "extensions.talos.dev/drbd")
117+
118+
rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []string{"extensions.talos.dev/schematic"},
119+
func(labelSpec *k8s.NodeAnnotationSpec, asrt *assert.Assertions) {
120+
asrt.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", labelSpec.TypedSpec().Value)
121+
})
122+
}

0 commit comments

Comments
 (0)