excluded prometheus operator based charts, extended metrics configura…

…tion to include endpoint

charts/gha-runner-scale-set-controller/templates/deployment.yaml

@@ -59,11 +59,16 @@ spec:
         {{- with .Values.flags.watchSingleNamespace }}
         - "--watch-single-namespace={{ . }}"
         {{- end }}
+        {{- with .Values.metrics }}
+        - "--listener-metrics-addr={{ .listenerAddr }}"
+        - "--listener-metrics-endpoint={{ .listenerEndpoint }}"
+        - "--metrics-addr={{ .controllerAddr }}"
+        {{- end }}
         command:
         - "/manager"
-        {{- if .Values.metrics.serviceMonitor }}
+        {{- with .Values.metrics }}
         ports:
-        - containerPort: 8080
+        - containerPort: {{regexReplaceAll ":([0-9]+)" .controllerAddr "${1}"}}
           protocol: TCP
         {{- end }}
         env:

charts/gha-runner-scale-set-controller/tests/template_test.go

@@ -1,6 +1,7 @@
 package tests
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -885,3 +886,74 @@ func TestTemplate_ManagerSingleNamespaceRoleBinding(t *testing.T) {
 	assert.Equal(t, "test-arc-gha-runner-scale-set-controller", managerSingleNamespaceWatchRoleBinding.Subjects[0].Name)
 	assert.Equal(t, namespaceName, managerSingleNamespaceWatchRoleBinding.Subjects[0].Namespace)
 }
+
+func TestControllerDeployment_MetricsPorts(t *testing.T) {
+	t.Parallel()
+
+	// Path to the helm chart we will test
+	helmChartPath, err := filepath.Abs("../../gha-runner-scale-set-controller")
+	require.NoError(t, err)
+
+	chartContent, err := os.ReadFile(filepath.Join(helmChartPath, "Chart.yaml"))
+	require.NoError(t, err)
+
+	chart := new(Chart)
+	err = yaml.Unmarshal(chartContent, chart)
+	require.NoError(t, err)
+
+	releaseName := "test-arc"
+	namespaceName := "test-" + strings.ToLower(random.UniqueId())
+
+	options := &helm.Options{
+		SetValues: map[string]string{
+			"image.tag":                "dev",
+			"metrics.controllerAddr":   ":8080",
+			"metrics.listenerAddr":     ":8081",
+			"metrics.listenerEndpoint": "/metrics",
+		},
+		KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
+	}
+
+	output := helm.RenderTemplate(t, options, helmChartPath, releaseName, []string{"templates/deployment.yaml"})
+
+	var deployment appsv1.Deployment
+	helm.UnmarshalK8SYaml(t, output, &deployment)
+
+	require.Len(t, deployment.Spec.Template.Spec.Containers, 1, "Expected one container")
+	container := deployment.Spec.Template.Spec.Containers[0]
+	assert.Len(t, container.Ports, 1)
+	port := container.Ports[0]
+	assert.Equal(t, corev1.Protocol("TCP"), port.Protocol)
+	assert.Equal(t, int32(8080), port.ContainerPort)
+
+	metricsFlags := map[string]*struct {
+		expect    string
+		frequency int
+	}{
+		"--listener-metrics-addr": {
+			expect: ":8081",
+		},
+		"--listener-metrics-endpoint": {
+			expect: "/metrics",
+		},
+		"--metrics-addr": {
+			expect: ":8080",
+		},
+	}
+	for _, cmd := range container.Args {
+		s := strings.Split(cmd, "=")
+		if len(s) != 2 {
+			continue
+		}
+		flag, ok := metricsFlags[s[0]]
+		if !ok {
+			continue
+		}
+		flag.frequency++
+		assert.Equal(t, flag.expect, s[1])
+	}
+
+	for key, value := range metricsFlags {
+		assert.Equal(t, value.frequency, 1, fmt.Sprintf("frequency of %q is not 1", key))
+	}
+}

charts/gha-runner-scale-set-controller/values.yaml

@@ -85,7 +85,7 @@ flags:
   # watchSingleNamespace: ""
 
 metrics:
-  serviceMonitor: false
-  serviceMonitorNamespace: "default" # need to match the namespace your prometheus's watching namespaces
-  serviceMonitorLabels:
-    release: "prometheus"
+  controllerAddr: ":8080"
+  listenerAddr: ":8080"
+  listenerEndpoint: "/metrics"
+

cmd/githubrunnerscalesetlistener/main.go

@@ -19,13 +19,13 @@ package main
 import (
 	"context"
 	"crypto/x509"
-	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/actions/actions-runner-controller/build"
 	"github.com/actions/actions-runner-controller/github/actions"
@@ -35,6 +35,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/net/http/httpproxy"
+	"golang.org/x/sync/errgroup"
 )
 
 type RunnerScaleSetListenerConfig struct {
@@ -50,6 +51,8 @@ type RunnerScaleSetListenerConfig struct {
 	RunnerScaleSetId            int    `split_words:"true"`
 	RunnerScaleSetName          string `split_words:"true"`
 	ServerRootCA                string `split_words:"true"`
+	MetricsAddr                 string `split_words:"true"`
+	MetricsEndpoint             string `split_words:"true"`
 }
 
 func main() {
@@ -74,79 +77,87 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer stop()
 
-	errCh := make(chan error, 2)
+	g, ctx := errgroup.WithContext(ctx)
 
-	go func() {
+	g.Go(func() error {
 		opts := runOptions{
 			serviceOptions: []func(*Service){
 				WithLogger(logger),
 			},
 		}
 		opts.serviceOptions = append(opts.serviceOptions, WithPrometheusMetrics(rc))
 
-		if err := run(ctx, rc, logger, opts); err != nil && !errors.Is(err, context.Canceled) {
-			errCh <- err
-			return
-		}
-		errCh <- nil
-	}()
-
-	go func() {
-		reg := prometheus.NewRegistry()
-		reg.MustRegister(
-			availableJobs,
-			acquiredJobs,
-			assignedJobs,
-			runningJobs,
-			registeredRunners,
-			busyRunners,
-			minRunners,
-			maxRunners,
-			desiredRunners,
-			idleRunners,
-			availableJobsTotal,
-			acquiredJobsTotal,
-			assignedJobsTotal,
-			startedJobsTotal,
-			completedJobsTotal,
-			jobQueueDurationSeconds,
-			jobStartupDurationSeconds,
-			jobExecutionDurationSeconds,
-		)
-
-		mux := http.NewServeMux()
-		mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}))
-
-		srv := http.Server{
-			Addr:    ":8888",
-			Handler: mux,
-		}
+		return run(ctx, rc, logger, opts)
+	})
+
+	if len(rc.MetricsAddr) != 0 {
+		g.Go(func() error {
+			metricsServer := metricsServer{
+				rc:     rc,
+				logger: logger,
+			}
+			g.Go(func() error {
+				<-ctx.Done()
+				return metricsServer.shutdown()
+			})
+			return metricsServer.run()
+		})
+	}
 
-		go func() {
-			<-ctx.Done()
-			srv.Shutdown(context.Background())
-		}()
+	if err := g.Wait(); err != nil {
+		logger.Error(err, "Error encountered")
+		os.Exit(1)
+	}
+}
 
-		logger.Info("Starting metrics server", "addr", srv.Addr)
-		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
-			errCh <- err
-			return
-		}
-		errCh <- nil
-	}()
+type metricsServer struct {
+	rc     RunnerScaleSetListenerConfig
+	logger logr.Logger
+	srv    *http.Server
+}
 
-	err1 := <-errCh
-	stop()
-	if err1 != nil {
-		logger.Error(err1, "First error emitted")
-	}
-	err2 := <-errCh
-	if err2 != nil {
-		logger.Error(err2, "Second error emitted")
-	}
-	if err1 != nil || err2 != nil {
-		os.Exit(1)
+func (s *metricsServer) shutdown() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	return s.srv.Shutdown(ctx)
+}
+
+func (s *metricsServer) run() error {
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(
+		availableJobs,
+		acquiredJobs,
+		assignedJobs,
+		runningJobs,
+		registeredRunners,
+		busyRunners,
+		minRunners,
+		maxRunners,
+		desiredRunners,
+		idleRunners,
+		availableJobsTotal,
+		acquiredJobsTotal,
+		assignedJobsTotal,
+		startedJobsTotal,
+		completedJobsTotal,
+		jobQueueDurationSeconds,
+		jobStartupDurationSeconds,
+		jobExecutionDurationSeconds,
+	)
+
+	mux := http.NewServeMux()
+	mux.Handle(
+		s.rc.MetricsEndpoint,
+		promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}),
+	)
+
+	s.srv = &http.Server{
+		Addr:    s.rc.MetricsAddr,
+		Handler: mux,
 	}
+
+	s.logger.Info("Starting metrics server", "address", s.srv.Addr)
+	return s.srv.ListenAndServe()
 }
 
 type runOptions struct {

cmd/githubrunnerscalesetlistener/metrics.go

@@ -23,7 +23,7 @@ const (
 	labelKeyRunnerName                    = "runner_name"
 )
 
-const githubScaleSetSubsystem = "github_runner_scale_set"
+const githubScaleSetSubsystem = "gha"
 
 // labels
 var (