diff --git a/deepfence_server/model/registry.go b/deepfence_server/model/registry.go index e4ecf9ccf7..b03f60efc8 100644 --- a/deepfence_server/model/registry.go +++ b/deepfence_server/model/registry.go @@ -194,7 +194,7 @@ func (ra *RegistryAddReq) CreateRegistry(ctx context.Context, rContext context.C m.container_registry_ids = REDUCE(distinctElements = [], element IN COALESCE(m.container_registry_ids, []) + $pgId | CASE WHEN NOT element in distinctElements THEN distinctElements + element ELSE distinctElements END)` _, err = tx.Run(query, map[string]interface{}{"node_id": registryID, "registry_type": ra.RegistryType, "pgId": cr.ID}) - return err + return tx.Commit() } func (ru *RegistryUpdateReq) UpdateRegistry(ctx context.Context, pgClient *postgresqlDb.Queries, r int32) error { diff --git a/deepfence_server/pkg/registry/ecr/client.go b/deepfence_server/pkg/registry/ecr/client.go new file mode 100644 index 0000000000..85d85866bf --- /dev/null +++ b/deepfence_server/pkg/registry/ecr/client.go @@ -0,0 +1,128 @@ +package ecr + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/deepfence/ThreatMapper/deepfence_server/model" +) + +func listImages(awsAccessKey, awsSecretKey, awsRegion string) ([]model.ContainerImage, error) { + // Set up AWS session with access key ID and secret access key + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(awsRegion), + Credentials: credentials.NewStaticCredentials(awsAccessKey, awsSecretKey, ""), + }) + if err != nil { + return nil, fmt.Errorf("error creating session: %v", err) + } + + // Create ECR client + svc := ecr.New(sess) + + // Call DescribeRepositories API + result, err := svc.DescribeRepositories(nil) + if err != nil { + return nil, fmt.Errorf("error describing repositories: %v", err) + } + + // Create slice of ContainerImage structs + var containerImages []model.ContainerImage + + var imageResult []*ecr.ListImagesOutput + // List images for each repository + for _, repo := range result.Repositories { + // Set up input parameters + input := &ecr.ListImagesInput{ + RepositoryName: aws.String(*repo.RepositoryName), + } + + // Call ListImages API + result, err := svc.ListImages(input) + if err != nil { + return nil, fmt.Errorf("error listing images for repository %s: %v", *repo.RepositoryName, err) + } + imageResult = append(imageResult, result) + // Add containers to ContainerImage struct + for _, image := range result.ImageIds { + if image.ImageTag != nil { + var containerImage model.ContainerImage + containerImage.Name = *repo.RepositoryUri + containerImage.ID = model.DigestToID(*image.ImageDigest) + containerImage.Tag = *image.ImageTag + containerImage.DockerImageID = *image.ImageDigest + containerImage.Metadata = model.Metadata{ + "digest": *image.ImageDigest, + "last_updated": time.Now().Unix(), + } + containerImages = append(containerImages, containerImage) + } + } + } + + return containerImages, nil +} +func listImagesCrossAccount(awsRegion, awsAccountID, targetAccountRoleARN string) ([]model.ContainerImage, error) { + // Create session with default credentials provider chain + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(awsRegion), + }) + if err != nil { + return nil, fmt.Errorf("error creating session: %v", err) + } + + // Assume role in target account + creds := stscreds.NewCredentials(sess, targetAccountRoleARN) + + // Create ECR client + svc := ecr.New(sess, &aws.Config{ + Region: aws.String(awsRegion), + Credentials: creds, + }) + + // Call DescribeRepositories API + result, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{ + RegistryId: aws.String(awsAccountID), + }) + if err != nil { + return nil, fmt.Errorf("error describing repositories: %v", err) + } + + // Create slice of ContainerImage structs + var containerImages []model.ContainerImage + + // List images for each repository + for _, repo := range result.Repositories { + // Set up input parameters + input := &ecr.ListImagesInput{ + RepositoryName: aws.String(*repo.RepositoryName), + } + + // Call ListImages API + result, err := svc.ListImages(input) + if err != nil { + return nil, fmt.Errorf("error listing images for repository %s: %v", *repo.RepositoryName, err) + } + + // Add containers to ContainerImage struct + for _, image := range result.ImageIds { + var containerImage model.ContainerImage + containerImage.Name = *repo.RepositoryUri + containerImage.Tag = *image.ImageTag + containerImage.ID = model.DigestToID(*image.ImageDigest) + containerImage.DockerImageID = *image.ImageDigest + containerImage.Metadata = model.Metadata{ + "digest": *image.ImageDigest, + "last_updated": time.Now().Unix(), + } + containerImages = append(containerImages, containerImage) + } + } + + return containerImages, nil +} diff --git a/deepfence_server/pkg/registry/ecr/ecr.go b/deepfence_server/pkg/registry/ecr/ecr.go new file mode 100644 index 0000000000..29543eddc8 --- /dev/null +++ b/deepfence_server/pkg/registry/ecr/ecr.go @@ -0,0 +1,83 @@ +package ecr + +import ( + "encoding/json" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/golang_deepfence_sdk/utils/encryption" +) + +func New(requestByte []byte) (*RegistryECR, error) { + r := RegistryECR{} + err := json.Unmarshal(requestByte, &r) + if err != nil { + return &r, err + } + return &r, nil +} + +func (e *RegistryECR) IsValidCredential() bool { + return true +} + +func (e *RegistryECR) EncryptSecret(aes encryption.AES) error { + var err error + e.Secret.AWSSecretAccessKey, err = aes.Encrypt(e.Secret.AWSSecretAccessKey) + return err +} + +func (e *RegistryECR) DecryptSecret(aes encryption.AES) error { + var err error + e.Secret.AWSSecretAccessKey, err = aes.Decrypt(e.Secret.AWSSecretAccessKey) + return err +} + +func (e *RegistryECR) EncryptExtras(aes encryption.AES) error { + return nil +} + +func (e *RegistryECR) DecryptExtras(aes encryption.AES) error { + return nil +} + +func (e *RegistryECR) FetchImagesFromRegistry() ([]model.ContainerImage, error) { + // based on iamrole we need to fetch images + if e.NonSecret.UseIAMRole == "true" { + return listImagesCrossAccount(e.NonSecret.AWSRegionName, e.NonSecret.AWSAccountID, e.NonSecret.TargetAccountRoleARN) + } + return listImages(e.NonSecret.AWSAccessKeyID, e.Secret.AWSSecretAccessKey, e.NonSecret.AWSRegionName) +} + +// getters +func (e *RegistryECR) GetSecret() map[string]interface{} { + var secret map[string]interface{} + b, _ := json.Marshal(e.Secret) + json.Unmarshal(b, &secret) + return secret +} + +func (e *RegistryECR) GetExtras() map[string]interface{} { + return map[string]interface{}{} +} + +func (e *RegistryECR) GetNamespace() string { + if e.NonSecret.IsPublic == "true" { + if e.NonSecret.UseIAMRole == "true" { + return e.NonSecret.AWSAccountID + } + return e.NonSecret.AWSAccessKeyID + } else { + if e.NonSecret.UseIAMRole == "true" { + return e.NonSecret.AWSRegionName + "_" + e.NonSecret.AWSAccountID + } + } + return e.NonSecret.AWSRegionName + "_" + e.NonSecret.AWSAccessKeyID +} + +func (e *RegistryECR) GetRegistryType() string { + return e.RegistryType +} + +func (e *RegistryECR) GetUsername() string { + return "" +} diff --git a/deepfence_server/pkg/registry/ecr/types.go b/deepfence_server/pkg/registry/ecr/types.go new file mode 100644 index 0000000000..f165733fa9 --- /dev/null +++ b/deepfence_server/pkg/registry/ecr/types.go @@ -0,0 +1,66 @@ +package ecr + +import "time" + +type RegistryECR struct { + Name string `json:"name"` + NonSecret NonSecret `json:"non_secret"` + Secret Secret `json:"secret"` + RegistryType string `json:"registry_type"` +} + +type NonSecret struct { + UseIAMRole string `json:"use_iam_role"` + IsPublic string `json:"is_public"` + AWSAccessKeyID string `json:"aws_access_key_id"` + AWSRegionName string `json:"aws_region_name"` + AWSAccountID string `json:"aws_account_id"` // legacy: registry_id + TargetAccountRoleARN string `json:"target_account_role_arn"` +} + +type Secret struct { + AWSSecretAccessKey string `json:"aws_secret_access_key"` +} + +type ImageWithTag struct { + Name string + Images Images +} + +type ImageTag struct { + Count int `json:"count,omitempty"` + Next string `json:"next,omitempty"` + Previous interface{} `json:"previous,omitempty"` + Results []Results `json:"results,omitempty"` +} +type Images struct { + Architecture string `json:"architecture,omitempty"` + Features string `json:"features,omitempty"` + Variant interface{} `json:"variant,omitempty"` + Digest string `json:"digest,omitempty"` + Os string `json:"os,omitempty"` + OsFeatures string `json:"os_features,omitempty"` + OsVersion interface{} `json:"os_version,omitempty"` + Size int `json:"size,omitempty"` + Status string `json:"status,omitempty"` + LastPulled time.Time `json:"last_pulled,omitempty"` + LastPushed time.Time `json:"last_pushed,omitempty"` +} +type Results struct { + Creator int `json:"creator,omitempty"` + ID int `json:"id,omitempty"` + Images []Images `json:"images,omitempty"` + LastUpdated time.Time `json:"last_updated,omitempty"` + LastUpdater int `json:"last_updater,omitempty"` + LastUpdaterUsername string `json:"last_updater_username,omitempty"` + Name string `json:"name,omitempty"` + Repository int `json:"repository,omitempty"` + FullSize int `json:"full_size,omitempty"` + V2 bool `json:"v2,omitempty"` + TagStatus string `json:"tag_status,omitempty"` + TagLastPulled time.Time `json:"tag_last_pulled,omitempty"` + TagLastPushed time.Time `json:"tag_last_pushed,omitempty"` + MediaType string `json:"media_type,omitempty"` + ContentType string `json:"content_type,omitempty"` + Digest string `json:"digest,omitempty"` +} diff --git a/deepfence_server/pkg/registry/registry.go b/deepfence_server/pkg/registry/registry.go index bb1bb05cde..a182fc27ff 100644 --- a/deepfence_server/pkg/registry/registry.go +++ b/deepfence_server/pkg/registry/registry.go @@ -8,6 +8,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/acr" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/dockerhub" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/dockerprivate" + "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/ecr" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/gcr" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/harbor" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/jfrog" @@ -37,7 +38,10 @@ func GetRegistry(rType string, requestByte []byte) (Registry, error) { r, err = harbor.New(requestByte) case constants.JFROG: r, err = jfrog.New(requestByte) + case constants.ECR: + r, err = ecr.New(requestByte) } + return r, err } @@ -218,6 +222,34 @@ func GetRegistryWithRegistryRow(row postgresql_db.GetContainerRegistriesRow) (Re }, } return r, nil + + case constants.ECR: + var nonSecret map[string]string + var secret map[string]string + err := json.Unmarshal(row.NonSecret, &nonSecret) + if err != nil { + return nil, err + } + err = json.Unmarshal(row.EncryptedSecret, &secret) + if err != nil { + return nil, err + } + r = &ecr.RegistryECR{ + RegistryType: row.RegistryType, + Name: row.Name, + NonSecret: ecr.NonSecret{ + UseIAMRole: nonSecret["use_iam_role"], + IsPublic: nonSecret["is_public"], + AWSAccessKeyID: nonSecret["aws_access_key_id"], + AWSRegionName: nonSecret["aws_region_name"], + AWSAccountID: nonSecret["aws_account_id"], + TargetAccountRoleARN: nonSecret["target_account_role_arn"], + }, + Secret: ecr.Secret{ + AWSSecretAccessKey: secret["aws_secret_access_key"], + }, + } + return r, err } return r, err } @@ -334,6 +366,25 @@ func GetRegistryWithRegistrySafeRow(row postgresql_db.GetContainerRegistriesSafe }, } return r, nil + case constants.ECR: + var nonSecret map[string]string + err := json.Unmarshal(row.NonSecret, &nonSecret) + if err != nil { + return nil, err + } + r = &ecr.RegistryECR{ + RegistryType: row.RegistryType, + Name: row.Name, + NonSecret: ecr.NonSecret{ + UseIAMRole: nonSecret["use_iam_role"], + IsPublic: nonSecret["is_public"], + AWSAccessKeyID: nonSecret["aws_access_key_id"], + AWSRegionName: nonSecret["aws_region_name"], + AWSAccountID: nonSecret["aws_account_id"], + TargetAccountRoleARN: nonSecret["target_account_role_arn"], + }, + } + return r, nil } return r, err } diff --git a/deepfence_worker/utils/registry.go b/deepfence_worker/utils/registry.go index a7b6593e1f..4d3f4ba23e 100644 --- a/deepfence_worker/utils/registry.go +++ b/deepfence_worker/utils/registry.go @@ -15,6 +15,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/acr" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/dockerhub" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/dockerprivate" + registryECR "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/ecr" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/gcr" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/harbor" "github.com/deepfence/ThreatMapper/deepfence_server/pkg/registry/jfrog" @@ -105,11 +106,83 @@ func GetCredentialsFromRegistry(ctx context.Context, registryId string) (regCred return dockerprivateCreds(reg, aes) case constants.JFROG: return jfrogCreds(reg, aes) + case constants.ECR: + return ecrCreds(reg, aes) default: return regCreds{}, nil } } +func ecrCreds(reg postgresql_db.GetContainerRegistryRow, aes encryption.AES) (regCreds, error) { + var ( + err error + hub registryECR.RegistryECR + nonsecret registryECR.NonSecret + secret registryECR.Secret + ) + err = json.Unmarshal(reg.NonSecret, &nonsecret) + if err != nil { + log.Error().Msg(err.Error()) + } + err = json.Unmarshal(reg.EncryptedSecret, &secret) + if err != nil { + log.Error().Msg(err.Error()) + } + + hub = registryECR.RegistryECR{ + Name: reg.Name, + Secret: secret, + NonSecret: nonsecret, + } + + err = hub.DecryptSecret(aes) + if err != nil { + log.Error().Msg(err.Error()) + } + + var awsConfig aws.Config + var svc *ecr.ECR + var creds *credentials.Credentials + + if hub.NonSecret.UseIAMRole == "false" { + awsConfig.WithCredentials(credentials.NewStaticCredentials(hub.NonSecret.AWSAccessKeyID, hub.Secret.AWSSecretAccessKey, "")) + } + mySession := session.Must(session.NewSession(&awsConfig)) + + if hub.NonSecret.UseIAMRole == "true" { + creds = stscreds.NewCredentials(mySession, hub.NonSecret.TargetAccountRoleARN) + svc = ecr.New(mySession, &aws.Config{ + Credentials: creds, + Region: aws.String(hub.NonSecret.AWSRegionName), + }) + } else { + svc = ecr.New(mySession, aws.NewConfig().WithRegion(hub.NonSecret.AWSRegionName)) + } + + var authorizationTokenRequestInput ecr.GetAuthorizationTokenInput + if hub.NonSecret.AWSAccountID != "" { + authorizationTokenRequestInput.SetRegistryIds([]*string{aws.String(hub.NonSecret.AWSAccountID)}) + } + authorizationTokenResponse, err := svc.GetAuthorizationToken(&authorizationTokenRequestInput) + if err != nil { + return regCreds{}, err + } + authorizationData := authorizationTokenResponse.AuthorizationData + if len(authorizationData) == 0 { + return regCreds{}, errors.New("no authorization data found") + } + authData := *authorizationData[0] + return regCreds{ + URL: *authData.ProxyEndpoint, + UserName: *authData.AuthorizationToken, + Password: "", + NameSpace: "", + ImagePrefix: "", + SkipTLSVerify: false, + UseHttp: useHttp(*authData.ProxyEndpoint), + }, nil +} + func dockerHubCreds(reg postgresql_db.GetContainerRegistryRow, aes encryption.AES) (regCreds, error) { var ( err error @@ -379,6 +452,7 @@ func jfrogCreds(reg postgresql_db.GetContainerRegistryRow, aes encryption.AES) ( }, nil } +// todo: unused func GetDockerCredentials(registryData map[string]interface{}) (string, string, string) { var registryType string registryType, ok := registryData["registry_type"].(string) @@ -458,6 +532,7 @@ func GetDockerCredentials(registryData map[string]interface{}) (string, string, } } +// todo: remove this!!! func getDefaultDockerCredentials(registryData map[string]interface{}, registryUrlKey, registryUsernameKey, registryPasswordKey string) (string, string, string) { var dockerUsername, dockerPassword, dockerRegistryUrl string var ok bool