Backport of [NET-8412] Fix APIGW policy creation ordering for upgrade…

… path into release/1.4.x (#3927)

* backport of commit 09cc033

* Added changelog

---------

Co-authored-by: jm96441n <john.maguire@hashicorp.com>

.changelog/3918.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+api-gateway: Fix order of initialization for creating ACL role/policy to avoid error logs in consul when upgrading between versions.
+```

control-plane/api-gateway/cache/consul.go

@@ -362,6 +362,9 @@ func (c *Cache) ensurePolicy(client *api.Client, gatewayName string) (string, er
 			if err != nil {
 				return "", err
 			}
+
+			// on an upgrade the cache will be empty so we need to write the policy to the cache
+			c.gatewayNameToPolicy[gatewayName] = existing
 			return existing.ID, nil
 		}
 
@@ -389,6 +392,8 @@ func (c *Cache) ensurePolicy(client *api.Client, gatewayName string) (string, er
 		return "", err
 	}
 
+	// update cache with existing policy
+	c.gatewayNameToPolicy[gatewayName] = existing
 	return existing.ID, nil
 }
 
@@ -429,7 +434,8 @@ func (c *Cache) ensureRole(client *api.Client, gatewayName string) (string, erro
 	}
 
 	if aclRole != nil {
-		return cachedRole.Name, nil
+		c.gatewayNameToRole[gatewayName] = aclRole
+		return aclRole.Name, nil
 	}
 
 	return createRole()