Skip to content

Commit

Permalink
Merge pull request #155 from TiberiuGC/feature/CARMv2
Browse files Browse the repository at this point in the history
Introduce a new CARM config map with support for `teamIDs` and service level isolation
  • Loading branch information
a-hilaly authored Aug 12, 2024
2 parents dda2022 + 2d1b192 commit a1359c9
Show file tree
Hide file tree
Showing 10 changed files with 349 additions and 88 deletions.
9 changes: 9 additions & 0 deletions apis/core/v1alpha1/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ const (
// TODO(jaypipes): Link to documentation on cross-account resource
// management
AnnotationOwnerAccountID = AnnotationPrefix + "owner-account-id"
// AnnotationTeamID is an annotation whose value is the identifier
// for the AWS team ID to manage the resources. If this annotation
// is set on a CR, the Kubernetes user is indicating that the ACK service
// controller should create/patch/delete the resource in the specified AWS
// role for this team ID. In order for this cross-account resource management
// to succeed, the AWS IAM Role that the ACK service controller runs as needs
// to have the ability to call the AWS STS::AssumeRole API call and assume an
// IAM Role in the target AWS Account.
AnnotationTeamID = AnnotationPrefix + "team-id"
// AnnotationRegion is an annotation whose value is the identifier for the
// the AWS region in which the resources should be created. If this annotation
// is set on a CR metadata, that means the user is indicating to the ACK service
Expand Down
3 changes: 3 additions & 0 deletions apis/core/v1alpha1/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ type AWSRegion string
// AWSAccountID represents an AWS account identifier
type AWSAccountID string

// TeamID represents a team ID identifier.
type TeamID string

// AWSResourceName represents an AWS Resource Name (ARN)
type AWSResourceName string

Expand Down
7 changes: 6 additions & 1 deletion pkg/featuregate/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@
// optionally overridden.
package featuregate

const (
// CARMv2 is the name of the CARMv2 feature.
CARMv2 = "CARMv2"
)

// defaultACKFeatureGates is a map of feature names to Feature structs
// representing the default feature gates for ACK controllers.
var defaultACKFeatureGates = FeatureGates{
// Set feature gates here
// "feature1": {Stage: Alpha, Enabled: false},
CARMv2: {Stage: Alpha, Enabled: false},
}

// FeatureStage represents the development stage of a feature.
Expand Down
81 changes: 67 additions & 14 deletions pkg/runtime/adoption_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"context"
"fmt"

"github.com/aws/aws-sdk-go/aws/arn"
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
Expand All @@ -32,6 +33,7 @@ import (
ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
ackcfg "github.com/aws-controllers-k8s/runtime/pkg/config"
ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors"
"github.com/aws-controllers-k8s/runtime/pkg/featuregate"
ackmetrics "github.com/aws-controllers-k8s/runtime/pkg/metrics"
"github.com/aws-controllers-k8s/runtime/pkg/requeue"
ackrtcache "github.com/aws-controllers-k8s/runtime/pkg/runtime/cache"
Expand Down Expand Up @@ -115,20 +117,44 @@ func (r *adoptionReconciler) reconcile(ctx context.Context, req ctrlrt.Request)
// If the ConfigMap is not created, or not populated with an
// accountID to roleARN mapping, we need to properly requeue with a
// helpful message to the user.
var roleARN ackv1alpha1.AWSResourceName
acctID, needCARMLookup := r.getOwnerAccountID(res)
if needCARMLookup {
// This means that the user is specifying a namespace that is
// annotated with an owner account ID. We need to retrieve the
// roleARN from the ConfigMap and properly requeue if the roleARN
// is not available.

var roleARN ackv1alpha1.AWSResourceName
if r.cfg.FeatureGates.IsEnabled(featuregate.CARMv2) {
teamID := r.getTeamID(res)
if teamID != "" {
// The user is specifying a namespace that is annotated with a team ID.
// Requeue if the corresponding roleARN is not available in the CARMv2 configmap.
// Additionally, set the account ID to the role's account ID.
roleARN, err = r.getRoleARNv2(string(teamID))
if err != nil {
ackrtlog.InfoAdoptedResource(r.log, res, fmt.Sprintf("Unable to start adoption reconcilliation %s: %v", acctID, err))
return requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
}
parsedARN, err := arn.Parse(string(roleARN))
if err != nil {
return fmt.Errorf("parsing role ARN %q from namespace annotation: %v", roleARN, err)
}
acctID = ackv1alpha1.AWSAccountID(parsedARN.AccountID)
} else if needCARMLookup {
// The user is specifying a namespace that is annotated with an owner account ID.
// Requeue if the corresponding roleARN is not available in the CARMv2 configmap.
roleARN, err = r.getRoleARNv2(string(acctID))
if err != nil {
ackrtlog.InfoAdoptedResource(r.log, res, fmt.Sprintf("Unable to start adoption reconcilliation %s: %v", acctID, err))
return requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
}
}
} else if needCARMLookup {
// The user is specifying a namespace that is annotated with an owner account ID.
// Requeue if the corresponding roleARN is not available in the Accounts (CARMv1) configmap.
roleARN, err = r.getRoleARN(acctID)
if err != nil {
ackrtlog.InfoAdoptedResource(r.log, res, fmt.Sprintf("Unable to start adoption reconcilliation %s: %v", acctID, err))
// r.getRoleARN errors are not terminal, we should requeue.
return requeue.NeededAfter(err, roleARNNotAvailableRequeueDelay)
}
}

region := r.getRegion(res)
targetDescriptor := rmf.ResourceDescriptor()
endpointURL := r.getEndpointURL(res)
Expand Down Expand Up @@ -460,6 +486,19 @@ func (r *adoptionReconciler) getOwnerAccountID(
return ackv1alpha1.AWSAccountID(r.cfg.AccountID), false
}

// getTeamID returns the team ID that owns the supplied resource.
func (r *adoptionReconciler) getTeamID(
res *ackv1alpha1.AdoptedResource,
) ackv1alpha1.TeamID {
// look for team id in the namespace annotations
namespace := res.GetNamespace()
teamID, ok := r.cache.Namespaces.GetTeamID(namespace)
if ok {
return ackv1alpha1.TeamID(teamID)
}
return ""
}

// getEndpointURL returns the AWS account that owns the supplied resource.
// We look for the namespace associated endpoint url, if that is set we use it.
// Otherwise if none of these annotations are set we use the endpoint url specified
Expand All @@ -478,14 +517,28 @@ func (r *adoptionReconciler) getEndpointURL(
return r.cfg.EndpointURL
}

// getRoleARN return the Role ARN that should be assumed in order to manage
// the resources.
func (r *adoptionReconciler) getRoleARN(
acctID ackv1alpha1.AWSAccountID,
) (ackv1alpha1.AWSResourceName, error) {
roleARN, err := r.cache.Accounts.GetAccountRoleARN(string(acctID))
// getRoleARNv2 returns the Role ARN that should be assumed for the given account/team ID,
// from the CARMv2 configmap, in order to manage the resources.
func (r *adoptionReconciler) getRoleARNv2(id string) (ackv1alpha1.AWSResourceName, error) {
// use service level roleARN if present
serviceID := r.sc.GetMetadata().ServiceAlias + "." + id
if roleARN, err := r.cache.CARMMaps.GetValue(serviceID); err == nil {
return ackv1alpha1.AWSResourceName(roleARN), nil
}
// otherwise use account/team level roleARN
roleARN, err := r.cache.CARMMaps.GetValue(id)
if err != nil {
return "", fmt.Errorf("retrieving role ARN for account/team ID %q from %q configmap: %v", id, ackrtcache.ACKCARMMapV2, err)
}
return ackv1alpha1.AWSResourceName(roleARN), nil
}

// getRoleARN returns the Role ARN that should be assumed for the given account ID,
// from the CARMv1 configmap, in order to manage the resources.
func (r *adoptionReconciler) getRoleARN(acctID ackv1alpha1.AWSAccountID) (ackv1alpha1.AWSResourceName, error) {
roleARN, err := r.cache.Accounts.GetValue(string(acctID))
if err != nil {
return "", fmt.Errorf("unable to retrieve role ARN for account %s: %v", acctID, err)
return "", fmt.Errorf("retrieving role ARN for account ID %q from %q configMap: %v", acctID, ackrtcache.ACKRoleAccountMap, err)
}
return ackv1alpha1.AWSResourceName(roleARN), nil
}
Expand Down
67 changes: 36 additions & 31 deletions pkg/runtime/cache/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,49 +28,54 @@ var (
// ErrCARMConfigMapNotFound is an error that is returned when the CARM
// configmap is not found.
ErrCARMConfigMapNotFound = errors.New("CARM configmap not found")
// ErrAccountIDNotFound is an error that is returned when the account ID
// ErrKeyNotFound is an error that is returned when the account ID
// is not found in the CARM configmap.
ErrAccountIDNotFound = errors.New("account ID not found in CARM configmap")
// ErrEmptyRoleARN is an error that is returned when the role ARN is empty
ErrKeyNotFound = errors.New("key not found in CARM configmap")
// ErrEmptyValue is an error that is returned when the role ARN is empty
// in the CARM configmap.
ErrEmptyRoleARN = errors.New("role ARN is empty in CARM configmap")
ErrEmptyValue = errors.New("role value is empty in CARM configmap")
)

const (
// ACKRoleAccountMap is the name of the configmap map object storing
// all the AWS Account IDs associated with their AWS Role ARNs.
ACKRoleAccountMap = "ack-role-account-map"

// ACKCARMMapV2 is the name of the v2 CARM map.
// It stores the mapping for:
// - Account ID to the AWS role ARNs.
ACKCARMMapV2 = "ack-carm-map"
)

// AccountCache is responsible for caching the CARM configmap
// CARMMap is responsible for caching the CARM configmap
// data. It is listening to all the events related to the CARM map and
// make the changes accordingly.
type AccountCache struct {
type CARMMap struct {
sync.RWMutex
log logr.Logger
roleARNs map[string]string
data map[string]string
configMapCreated bool
hasSynced func() bool
}

// NewAccountCache instanciate a new AccountCache.
func NewAccountCache(log logr.Logger) *AccountCache {
return &AccountCache{
log: log.WithName("cache.account"),
roleARNs: make(map[string]string),
// NewCARMMapCache instanciate a new CARMMap.
func NewCARMMapCache(log logr.Logger) *CARMMap {
return &CARMMap{
log: log.WithName("cache.carm"),
data: make(map[string]string),
configMapCreated: false,
}
}

// resourceMatchACKRoleAccountConfigMap verifies if a resource is
// resourceMatchCARMConfigMap verifies if a resource is
// the CARM configmap. It verifies the name, namespace and object type.
func resourceMatchACKRoleAccountsConfigMap(raw interface{}) bool {
func resourceMatchCARMConfigMap(raw interface{}, name string) bool {
object, ok := raw.(*corev1.ConfigMap)
return ok && object.ObjectMeta.Name == ACKRoleAccountMap
return ok && object.ObjectMeta.Name == name
}

// Run instantiate a new SharedInformer for ConfigMaps and runs it to begin processing items.
func (c *AccountCache) Run(clientSet kubernetes.Interface, stopCh <-chan struct{}) {
func (c *CARMMap) Run(name string, clientSet kubernetes.Interface, stopCh <-chan struct{}) {
c.log.V(1).Info("Starting shared informer for accounts cache", "targetConfigMap", ACKRoleAccountMap)
informer := informersv1.NewConfigMapInformer(
clientSet,
Expand All @@ -80,33 +85,33 @@ func (c *AccountCache) Run(clientSet kubernetes.Interface, stopCh <-chan struct{
)
informer.AddEventHandler(k8scache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
if resourceMatchACKRoleAccountsConfigMap(obj) {
if resourceMatchCARMConfigMap(obj, name) {
cm := obj.(*corev1.ConfigMap)
object := cm.DeepCopy()
// To avoid multiple mutex locks, we are updating the cache
// and the configmap existence flag in the same function.
configMapCreated := true
c.updateAccountRoleData(configMapCreated, object.Data)
c.updateData(configMapCreated, object.Data)
c.log.V(1).Info("created account config map", "name", cm.ObjectMeta.Name)
}
},
UpdateFunc: func(orig, desired interface{}) {
if resourceMatchACKRoleAccountsConfigMap(desired) {
if resourceMatchCARMConfigMap(desired, name) {
cm := desired.(*corev1.ConfigMap)
object := cm.DeepCopy()
//TODO(a-hilaly): compare data checksum before updating the cache
c.updateAccountRoleData(true, object.Data)
c.updateData(true, object.Data)
c.log.V(1).Info("updated account config map", "name", cm.ObjectMeta.Name)
}
},
DeleteFunc: func(obj interface{}) {
if resourceMatchACKRoleAccountsConfigMap(obj) {
if resourceMatchCARMConfigMap(obj, name) {
cm := obj.(*corev1.ConfigMap)
newMap := make(map[string]string)
// To avoid multiple mutex locks, we are updating the cache
// and the configmap existence flag in the same function.
configMapCreated := false
c.updateAccountRoleData(configMapCreated, newMap)
c.updateData(configMapCreated, newMap)
c.log.V(1).Info("deleted account config map", "name", cm.ObjectMeta.Name)
}
},
Expand All @@ -115,33 +120,33 @@ func (c *AccountCache) Run(clientSet kubernetes.Interface, stopCh <-chan struct{
c.hasSynced = informer.HasSynced
}

// GetAccountRoleARN queries the AWS accountID associated Role ARN
// GetValue queries the value
// from the cached CARM configmap. It will return an error if the
// configmap is not found, the accountID is not found or the role ARN
// configmap is not found, the key is not found or the value
// is empty.
//
// This function is thread safe.
func (c *AccountCache) GetAccountRoleARN(accountID string) (string, error) {
func (c *CARMMap) GetValue(key string) (string, error) {
c.RLock()
defer c.RUnlock()

if !c.configMapCreated {
return "", ErrCARMConfigMapNotFound
}
roleARN, ok := c.roleARNs[accountID]
roleARN, ok := c.data[key]
if !ok {
return "", ErrAccountIDNotFound
return "", ErrKeyNotFound
}
if roleARN == "" {
return "", ErrEmptyRoleARN
return "", ErrEmptyValue
}
return roleARN, nil
}

// updateAccountRoleData updates the CARM map. This function is thread safe.
func (c *AccountCache) updateAccountRoleData(exist bool, data map[string]string) {
// updateData updates the CARM map. This function is thread safe.
func (c *CARMMap) updateData(exist bool, data map[string]string) {
c.Lock()
defer c.Unlock()
c.roleARNs = data
c.data = data
c.configMapCreated = exist
}
Loading

0 comments on commit a1359c9

Please sign in to comment.