Skip to content

Commit f966d02

Browse files
fix place where the metrics service should be scaffolded
1 parent 6c3321b commit f966d02

File tree

28 files changed

+74
-157
lines changed

28 files changed

+74
-157
lines changed
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
resources:
22
- manager.yaml
3+
# If you want to expose the metrics endpoint without TLS/HTTP protection, uncomment the following line.
4+
#- metrics_service.yaml

docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml
1312
# For each CRD, "Editor" and "Viewer" roles are scaffolded by
1413
# default, aiding admins in cluster management. Those roles are
1514
# not used by the Project itself. You can comment the following lines
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
resources:
22
- manager.yaml
3+
# If you want to expose the metrics endpoint without TLS/HTTP protection, uncomment the following line.
4+
#- metrics_service.yaml

docs/book/src/getting-started/testdata/project/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml
1312
# For each CRD, "Editor" and "Viewer" roles are scaffolded by
1413
# default, aiding admins in cluster management. Those roles are
1514
# not used by the Project itself. You can comment the following lines

docs/book/src/reference/metrics.md

+8
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,14 @@ in the file `config/default/kustomization.yaml`, see:
5454
# kind: Deployment
5555
```
5656

57+
Then, you will need to enable the Metrics Service in `config/manager/kustomization.yaml`:
58+
59+
```yaml
60+
- manager.yaml
61+
# If you want to expose the metrics endpoint without TLS/HTTP protection, uncomment the following line.
62+
#- metrics_service.yaml
63+
```
64+
5765
Note that projects are scaffolded by default passing the flag `--metrics-bind-address=0`
5866
to the manager to ensure that metrics are disabled. See the [controller-runtime
5967
implementation](https://github.com/kubernetes-sigs/controller-runtime/blob/834905b07c7b5a78e86d21d764f7c2fdaa9602e0/pkg/metrics/server/server.go#L119-L122)

pkg/plugins/common/kustomize/v2/scaffolds/init.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func (s *initScaffolder) Scaffold() error {
6464

6565
templates := []machinery.Builder{
6666
&rbac.Kustomization{},
67-
&rbac.MetricsService{},
67+
&manager.MetricsService{},
6868
&rbac.RoleBinding{},
6969
// We need to create a Role because if the project
7070
// has not CRD define the controller-gen will not generate this file

pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager/kustomization.go

+2
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,6 @@ func (f *Kustomization) SetTemplateDefaults() error {
4444

4545
const kustomizeManagerTemplate = `resources:
4646
- manager.yaml
47+
# If you want to expose the metrics endpoint without TLS/HTTP protection, uncomment the following line.
48+
#- metrics_service.yaml
4749
`

pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/metrics_service.go pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/manager/metrics_service.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package rbac
17+
package manager
1818

1919
import (
2020
"path/filepath"
@@ -33,7 +33,7 @@ type MetricsService struct {
3333
// SetTemplateDefaults implements file.Template
3434
func (f *MetricsService) SetTemplateDefaults() error {
3535
if f.Path == "" {
36-
f.Path = filepath.Join("config", "rbac", "metrics_service.yaml")
36+
f.Path = filepath.Join("config", "manager", "metrics_service.yaml")
3737
}
3838

3939
f.TemplateBody = metricsServiceTemplate

pkg/plugins/common/kustomize/v2/scaffolds/internal/templates/config/rbac/kustomization.go

-1
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,4 @@ const kustomizeRBACTemplate = `resources:
5353
- role_binding.yaml
5454
- leader_election_role.yaml
5555
- leader_election_role_binding.yaml
56-
- metrics_service.yaml
5756
`

test/e2e/v4/generate_test.go

+6
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ func GenerateV4(kbc *utils.TestContext) {
6666
ExpectWithOffset(1, pluginutil.UncommentCode(
6767
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
6868
metricsTarget, "#")).To(Succeed())
69+
ExpectWithOffset(1, pluginutil.UncommentCode(
70+
filepath.Join(kbc.Dir, "config", "manager", "kustomization.yaml"),
71+
"#- metrics_service.yaml", "#")).To(Succeed())
6972

7073
ExpectWithOffset(1, pluginutil.UncommentCode(filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
7174
certManagerTarget, "#")).To(Succeed())
@@ -126,6 +129,9 @@ func GenerateV4WithoutWebhooks(kbc *utils.TestContext) {
126129
ExpectWithOffset(1, pluginutil.UncommentCode(
127130
filepath.Join(kbc.Dir, "config", "default", "kustomization.yaml"),
128131
metricsTarget, "#")).To(Succeed())
132+
ExpectWithOffset(1, pluginutil.UncommentCode(
133+
filepath.Join(kbc.Dir, "config", "manager", "kustomization.yaml"),
134+
"#- metrics_service.yaml", "#")).To(Succeed())
129135

130136
if kbc.IsRestricted {
131137
By("uncomment kustomize files to ensure that pods are restricted")

test/e2e/v4/plugin_cluster_test.go

+51-56
Original file line numberDiff line numberDiff line change
@@ -278,66 +278,47 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool)
278278

279279
// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned.
280280
func curlMetrics(kbc *utils.TestContext, hasMetrics bool) string {
281-
By("validating that the controller-manager service is available")
282-
_, err := kbc.Kubectl.Get(
283-
true,
284-
"service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix),
285-
)
286-
ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist")
287-
288-
By("validating that the controller-manager deployment is ready")
289-
verifyDeploymentReady := func() error {
290-
output, err := kbc.Kubectl.Get(
281+
var metricsOutput string
282+
if hasMetrics {
283+
By("validating that the controller-manager service is available")
284+
_, err := kbc.Kubectl.Get(
291285
true,
292-
"deployment", fmt.Sprintf("e2e-%s-controller-manager", kbc.TestSuffix),
293-
"-o", "jsonpath={.status.readyReplicas}",
286+
"service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix),
294287
)
295-
if err != nil {
296-
return err
297-
}
298-
readyReplicas, _ := strconv.Atoi(output)
299-
if readyReplicas < 1 {
300-
return fmt.Errorf("expected at least 1 ready replica, got %d", readyReplicas)
301-
}
302-
return nil
303-
}
304-
EventuallyWithOffset(2, verifyDeploymentReady, 240*time.Second, time.Second).Should(Succeed(),
305-
"Deployment is not ready")
288+
ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist")
306289

307-
By("ensuring the service endpoint is ready")
308-
eventuallyCheckServiceEndpoint := func() error {
309-
output, err := kbc.Kubectl.Get(
310-
true,
311-
"endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix),
312-
"-o", "jsonpath={.subsets[*].addresses[*].ip}",
313-
)
314-
if err != nil {
315-
return err
290+
By("ensuring the service endpoint is ready")
291+
eventuallyCheckServiceEndpoint := func() error {
292+
output, err := kbc.Kubectl.Get(
293+
true,
294+
"endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix),
295+
"-o", "jsonpath={.subsets[*].addresses[*].ip}",
296+
)
297+
if err != nil {
298+
return err
299+
}
300+
if output == "" {
301+
return fmt.Errorf("no endpoints found")
302+
}
303+
return nil
316304
}
317-
if output == "" {
318-
return fmt.Errorf("no endpoints found")
305+
EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(),
306+
"Service endpoint should be ready")
307+
308+
By("creating a curl pod to access the metrics endpoint")
309+
// nolint:lll
310+
cmdOpts := []string{
311+
"run", "curl",
312+
"--restart=Never",
313+
"--namespace", kbc.Kubectl.Namespace,
314+
"--image=curlimages/curl:7.78.0",
315+
"--",
316+
"/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics",
317+
kbc.TestSuffix, kbc.Kubectl.Namespace),
319318
}
320-
return nil
321-
}
322-
EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(),
323-
"Service endpoint should be ready")
324-
325-
By("creating a curl pod to access the metrics endpoint")
326-
// nolint:lll
327-
cmdOpts := []string{
328-
"run", "curl",
329-
"--restart=Never",
330-
"--namespace", kbc.Kubectl.Namespace,
331-
"--image=curlimages/curl:7.78.0",
332-
"--",
333-
"/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics",
334-
kbc.TestSuffix, kbc.Kubectl.Namespace),
335-
}
336-
_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
337-
ExpectWithOffset(2, err).NotTo(HaveOccurred())
319+
_, err = kbc.Kubectl.CommandInNamespace(cmdOpts...)
320+
ExpectWithOffset(2, err).NotTo(HaveOccurred())
338321

339-
var metricsOutput string
340-
if hasMetrics {
341322
By("validating that the curl pod is running as expected")
342323
verifyCurlUp := func() error {
343324
status, err := kbc.Kubectl.Get(
@@ -359,6 +340,20 @@ func curlMetrics(kbc *utils.TestContext, hasMetrics bool) string {
359340
}
360341
EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/1.1 200 OK"))
361342
} else {
343+
By("creating a curl pod to access the metrics endpoint")
344+
// nolint:lll
345+
cmdOpts := []string{
346+
"run", "curl",
347+
"--restart=Never",
348+
"--namespace", kbc.Kubectl.Namespace,
349+
"--image=curlimages/curl:7.78.0",
350+
"--",
351+
"/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics",
352+
kbc.TestSuffix, kbc.Kubectl.Namespace),
353+
}
354+
_, err := kbc.Kubectl.CommandInNamespace(cmdOpts...)
355+
ExpectWithOffset(2, err).NotTo(HaveOccurred())
356+
362357
By("validating that the curl pod fail as expected")
363358
verifyCurlUp := func() error {
364359
status, err := kbc.Kubectl.Get(
@@ -375,14 +370,14 @@ func curlMetrics(kbc *utils.TestContext, hasMetrics bool) string {
375370

376371
By("validating that the metrics endpoint is not working as expected")
377372
getCurlLogs := func() string {
378-
metricsOutput, err = kbc.Kubectl.Logs("curl")
373+
metricsOutput, err := kbc.Kubectl.Logs("curl")
379374
ExpectWithOffset(3, err).NotTo(HaveOccurred())
380375
return metricsOutput
381376
}
382377
EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("Connection refused"))
383378
}
384379
By("cleaning up the curl pod")
385-
_, err = kbc.Kubectl.Delete(true, "pods/curl")
380+
_, err := kbc.Kubectl.Delete(true, "pods/curl")
386381
ExpectWithOffset(3, err).NotTo(HaveOccurred())
387382

388383
return metricsOutput

testdata/project-v4-multigroup-with-deploy-image/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml
1312
# For each CRD, "Editor" and "Viewer" roles are scaffolded by
1413
# default, aiding admins in cluster management. Those roles are
1514
# not used by the Project itself. You can comment the following lines

testdata/project-v4-multigroup-with-deploy-image/dist/install.yaml

-18
Original file line numberDiff line numberDiff line change
@@ -1474,24 +1474,6 @@ subjects:
14741474
---
14751475
apiVersion: v1
14761476
kind: Service
1477-
metadata:
1478-
labels:
1479-
app.kubernetes.io/managed-by: kustomize
1480-
app.kubernetes.io/name: project-v4-multigroup-with-deploy-image
1481-
control-plane: controller-manager
1482-
name: project-v4-multigroup-with-deploy-image-controller-manager-metrics-service
1483-
namespace: project-v4-multigroup-with-deploy-image-system
1484-
spec:
1485-
ports:
1486-
- name: http
1487-
port: 8080
1488-
protocol: TCP
1489-
targetPort: 8080
1490-
selector:
1491-
control-plane: controller-manager
1492-
---
1493-
apiVersion: v1
1494-
kind: Service
14951477
metadata:
14961478
labels:
14971479
app.kubernetes.io/managed-by: kustomize

testdata/project-v4-multigroup/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml
1312
# For each CRD, "Editor" and "Viewer" roles are scaffolded by
1413
# default, aiding admins in cluster management. Those roles are
1514
# not used by the Project itself. You can comment the following lines

testdata/project-v4-multigroup/dist/install.yaml

-18
Original file line numberDiff line numberDiff line change
@@ -1474,24 +1474,6 @@ subjects:
14741474
---
14751475
apiVersion: v1
14761476
kind: Service
1477-
metadata:
1478-
labels:
1479-
app.kubernetes.io/managed-by: kustomize
1480-
app.kubernetes.io/name: project-v4-multigroup
1481-
control-plane: controller-manager
1482-
name: project-v4-multigroup-controller-manager-metrics-service
1483-
namespace: project-v4-multigroup-system
1484-
spec:
1485-
ports:
1486-
- name: http
1487-
port: 8080
1488-
protocol: TCP
1489-
targetPort: 8080
1490-
selector:
1491-
control-plane: controller-manager
1492-
---
1493-
apiVersion: v1
1494-
kind: Service
14951477
metadata:
14961478
labels:
14971479
app.kubernetes.io/managed-by: kustomize

testdata/project-v4-with-deploy-image/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml
1312
# For each CRD, "Editor" and "Viewer" roles are scaffolded by
1413
# default, aiding admins in cluster management. Those roles are
1514
# not used by the Project itself. You can comment the following lines

testdata/project-v4-with-deploy-image/dist/install.yaml

-18
Original file line numberDiff line numberDiff line change
@@ -550,24 +550,6 @@ subjects:
550550
---
551551
apiVersion: v1
552552
kind: Service
553-
metadata:
554-
labels:
555-
app.kubernetes.io/managed-by: kustomize
556-
app.kubernetes.io/name: project-v4-with-deploy-image
557-
control-plane: controller-manager
558-
name: project-v4-with-deploy-image-controller-manager-metrics-service
559-
namespace: project-v4-with-deploy-image-system
560-
spec:
561-
ports:
562-
- name: http
563-
port: 8080
564-
protocol: TCP
565-
targetPort: 8080
566-
selector:
567-
control-plane: controller-manager
568-
---
569-
apiVersion: v1
570-
kind: Service
571553
metadata:
572554
labels:
573555
app.kubernetes.io/managed-by: kustomize

testdata/project-v4-with-grafana/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,3 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml

testdata/project-v4-with-grafana/dist/install.yaml

-18
Original file line numberDiff line numberDiff line change
@@ -107,24 +107,6 @@ subjects:
107107
name: project-v4-with-grafana-controller-manager
108108
namespace: project-v4-with-grafana-system
109109
---
110-
apiVersion: v1
111-
kind: Service
112-
metadata:
113-
labels:
114-
app.kubernetes.io/managed-by: kustomize
115-
app.kubernetes.io/name: project-v4-with-grafana
116-
control-plane: controller-manager
117-
name: project-v4-with-grafana-controller-manager-metrics-service
118-
namespace: project-v4-with-grafana-system
119-
spec:
120-
ports:
121-
- name: http
122-
port: 8080
123-
protocol: TCP
124-
targetPort: 8080
125-
selector:
126-
control-plane: controller-manager
127-
---
128110
apiVersion: apps/v1
129111
kind: Deployment
130112
metadata:

testdata/project-v4/config/rbac/kustomization.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resources:
99
- role_binding.yaml
1010
- leader_election_role.yaml
1111
- leader_election_role_binding.yaml
12-
- metrics_service.yaml
1312
# For each CRD, "Editor" and "Viewer" roles are scaffolded by
1413
# default, aiding admins in cluster management. Those roles are
1514
# not used by the Project itself. You can comment the following lines

0 commit comments

Comments
 (0)