diff --git a/pkg/authn/delegating.go b/pkg/authn/delegating.go
index ba077a8da..ac2594625 100644
--- a/pkg/authn/delegating.go
+++ b/pkg/authn/delegating.go
@@ -42,19 +42,20 @@ func NewDelegatingAuthenticator(client authenticationclient.TokenReviewInterface
 		p   *dynamiccertificates.DynamicFileCAContent
 		err error
 	)
+
+	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
+		Anonymous:               false, // always require authentication
+		CacheTTL:                2 * time.Minute,
+		TokenAccessReviewClient: client,
+		APIAudiences:            authenticator.Audiences(authn.Token.Audiences),
+	}
+
 	if len(authn.X509.ClientCAFile) > 0 {
 		p, err = dynamiccertificates.NewDynamicCAContentFromFile("client-ca", authn.X509.ClientCAFile)
 		if err != nil {
 			return nil, err
 		}
-	}
-
-	authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous:                          false, // always require authentication
-		CacheTTL:                           2 * time.Minute,
-		ClientCertificateCAContentProvider: p,
-		TokenAccessReviewClient:            client,
-		APIAudiences:                       authenticator.Audiences(authn.Token.Audiences),
+		authenticatorConfig.ClientCertificateCAContentProvider = p
 	}
 
 	authenticator, _, err := authenticatorConfig.New()