Skip to content

Commit

Permalink
Fix Shared ADF Lambda Layer builds and add move to ARM-64 Lambdas
Browse files Browse the repository at this point in the history
**Why?**

The Lambda Layer that ADF builds included the `bootstrap/adf-build/shared`
folder as its source.

Unfortunately, this resulted in 388 MB of unzipped dependencies.
While the maximum limit for AWS Lambda Layers is set at 250 MB.

**What?**

After going through each of the lambda functions that use this Lambda Layer,
I realized that they only require the files located in
`bootstrap/adf-build/shared/python`.

Since this implied no longer requiring AWS CDK in the Lambda Layer, I also
used this opportunity to move to the ARM-64 Lambdas in ADF.

The new Lambda Layer is 26 MB.

To build ARM-64 on CodeBuild, I've added the Amazon managed container that
enables cross-architecture builds on Docker. That will launch and
configure Qemu correctly, so ARM-64 Docker images can execute correctly.
I've also fixed the build metadata and updated the `sam build` process to
use a container instead. As without it, it could not resolve the dependencies
required for PyYaml and others specifically for ARM-64.

Lastly, I've updated the shared lambda layer permissions, so we have
control in the stack who can access and build on top of it.
  • Loading branch information
sbkok committed Jan 18, 2024
1 parent f219e30 commit 1f0bd78
Show file tree
Hide file tree
Showing 39 changed files with 257 additions and 143 deletions.
1 change: 1 addition & 0 deletions linters/custom-adf-dict.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ adf
adfconfig
awscli
backoff
binfmt
bitnami
boto
boto3
Expand Down
10 changes: 6 additions & 4 deletions src/lambda_codebase/account_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,22 @@
"""

import os
import boto3

import boto3
from botocore.exceptions import ClientError
from logger import configure_logger

# ADF imports
from errors import (
AccountCreationNotFinishedError,
GenericAccountConfigureError,
ParameterNotFoundError,
)
from parameter_store import ParameterStore
from cloudformation import CloudFormation
from logger import configure_logger
from parameter_store import ParameterStore
from partition import get_partition
from s3 import S3
from sts import STS
from partition import get_partition

# Globals taken from the lambda environment variables
S3_BUCKET = os.environ["S3_BUCKET_NAME"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@
"""

import os
from sts import STS

from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger
from sts import STS

patch_all()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
"""
Moves an account to the specified OU.
"""
from organizations import Organizations
import boto3
from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger
from organizations import Organizations


patch_all()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
"""
from ast import literal_eval


import boto3
from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger

patch_all()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
"""
import boto3
from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger

patch_all()
Expand Down
2 changes: 2 additions & 0 deletions src/lambda_codebase/account_processing/create_account.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import os
from aws_xray_sdk.core import patch_all
import boto3

# ADF imports
from logger import configure_logger

patch_all()
Expand Down
4 changes: 3 additions & 1 deletion src/lambda_codebase/account_processing/delete_default_vpc.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
Deletes the default VPC in a particular region
"""
import os
from sts import STS
from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger
from sts import STS

patch_all()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
"""

import os
from sts import STS
from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger
from sts import STS

patch_all()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,17 @@
import logging
from typing import Any, TypedDict
import re
import yaml

import yaml
from yaml.error import YAMLError

import boto3
from botocore.exceptions import ClientError
from aws_xray_sdk.core import patch_all
from organizations import Organizations

# ADF imports
from logger import configure_logger
from organizations import Organizations


patch_all()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@
import boto3
from botocore.exceptions import ClientError, BotoCoreError
from botocore.config import Config
from logger import configure_logger
from aws_xray_sdk.core import patch_all

# ADF imports
from logger import configure_logger


LOGGER = configure_logger(__name__)
patch_all()
Expand Down
1 change: 1 addition & 0 deletions src/lambda_codebase/cross_region_bucket/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
delete,
)

# ADF imports
from partition import get_partition

# Type aliases:
Expand Down
1 change: 1 addition & 0 deletions src/lambda_codebase/deployment_account_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""

import os

import boto3

from cloudformation import CloudFormation
Expand Down
4 changes: 3 additions & 1 deletion src/lambda_codebase/determine_event.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@
"""

import os

import boto3

from parameter_store import ParameterStore
# ADF imports
from cache import Cache
from event import Event
from organizations import Organizations
from parameter_store import ParameterStore

REGION_DEFAULT = os.environ["AWS_REGION"]

Expand Down
8 changes: 5 additions & 3 deletions src/lambda_codebase/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,16 @@

import ast
import os

# ADF imports
from errors import ParameterNotFoundError, RootOUIDError

DEPLOYMENT_ACCOUNT_OU_NAME = 'deployment'
DEPLOYMENT_ACCOUNT_S3_BUCKET = os.environ["DEPLOYMENT_ACCOUNT_BUCKET"]
ADF_VERSION = os.environ["ADF_VERSION"]
ADF_LOG_LEVEL = os.environ["ADF_LOG_LEVEL"]


class Event:
"""
Class for structuring the Event in Step Functions
Expand Down Expand Up @@ -71,7 +74,6 @@ def __init__(self, event, parameter_store, organizations, account_id):
)
self.set_destination_ou_name()


def _determine_if_deployment_account(self):
"""
Sets property based on if the account that has been moved
Expand Down Expand Up @@ -126,7 +128,7 @@ def create_output_object(self, account_path):
'full_path': "ROOT" if self.moved_to_root else account_path,
'destination_ou_id': self.destination_ou_id,
'source_ou_id': self.source_ou_id,
'deployment_account_parameters' : {
'deployment_account_parameters': {
'organization_id': organization_information.get(
"organization_id"
),
Expand All @@ -139,5 +141,5 @@ def create_output_object(self, account_path):
'deployment_account_bucket': DEPLOYMENT_ACCOUNT_S3_BUCKET,
'adf_version': ADF_VERSION,
'adf_log_level': ADF_LOG_LEVEL
}
},
}
5 changes: 3 additions & 2 deletions src/lambda_codebase/generic_account_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@

import os

# ADF imports
from logger import configure_logger
from sts import STS
from stepfunctions import StepFunctions
from partition import get_partition
from stepfunctions import StepFunctions
from sts import STS

LOGGER = configure_logger(__name__)
REGION_DEFAULT = os.getenv('AWS_REGION')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,26 +72,22 @@ Conditions:

Globals:
Function:
Architectures:
- arm64
CodeUri: lambda_codebase
Runtime: python3.12

Resources:
LambdaLayerVersion:
ADFSharedPythonLambdaLayerVersion:
Type: "AWS::Serverless::LayerVersion"
Properties:
ContentUri: "../../adf-build/shared/"
ContentUri: "../../adf-build/shared/python"
CompatibleRuntimes:
- python3.12
Description: "Shared Lambda Layer between master and deployment account"
LayerName: shared_layer

LambdaLayerVersionPermission:
Type: "AWS::Lambda::LayerVersionPermission"
Properties:
Action: lambda:GetLayerVersion
LayerVersionArn: !Ref LambdaLayerVersion
OrganizationId: !Ref OrganizationId
Principal: "*"
LayerName: adf_shared_layer
Metadata:
BuildMethod: python3.12

KMSKey:
Type: AWS::KMS::Key
Expand Down Expand Up @@ -185,7 +181,7 @@ Resources:
Properties:
Location: pipeline_management.yml
Parameters:
LambdaLayer: !Ref LambdaLayerVersion
LambdaLayer: !Ref ADFSharedPythonLambdaLayerVersion
ADFVersion: !Ref ADFVersion
OrganizationId: !Ref OrganizationId
CrossAccountAccessRole: !Ref CrossAccountAccessRole
Expand Down Expand Up @@ -1014,7 +1010,7 @@ Resources:
Type: "AWS::Serverless::Function"
Properties:
Layers:
- !Ref LambdaLayerVersion
- !Ref ADFSharedPythonLambdaLayerVersion
Description: "ADF Lambda Function - Send Slack Notification"
FunctionName: SendSlackNotification
Handler: slack.lambda_handler
Expand All @@ -1024,12 +1020,14 @@ Resources:
ADF_PIPELINE_PREFIX: !Ref PipelinePrefix
ADF_LOG_LEVEL: !Ref ADFLogLevel
Timeout: 10
Metadata:
BuildMethod: python3.12

EnableCrossAccountAccess:
Type: "AWS::Serverless::Function"
Properties:
Layers:
- !Ref LambdaLayerVersion
- !Ref ADFSharedPythonLambdaLayerVersion
Description: "ADF Lambda Function - EnableCrossAccountAccess"
MemorySize: 1024
Environment:
Expand All @@ -1042,12 +1040,14 @@ Resources:
Handler: enable_cross_account_access.lambda_handler
Role: !GetAtt EnableCrossAccountAccessLambdaRole.Arn
Timeout: 900
Metadata:
BuildMethod: python3.12

CheckPipelineStatus:
Type: "AWS::Serverless::Function"
Properties:
Layers:
- !Ref LambdaLayerVersion
- !Ref ADFSharedPythonLambdaLayerVersion
Description: "ADF Lambda Function - CheckPipelineStatus"
Environment:
Variables:
Expand All @@ -1058,6 +1058,8 @@ Resources:
Handler: update_pipelines.lambda_handler
Role: !GetAtt CheckPipelineStatusLambdaRole.Arn
Timeout: 120
Metadata:
BuildMethod: python3.12

SendSlackNotificationLambdaRole:
Type: "AWS::IAM::Role"
Expand Down Expand Up @@ -1174,7 +1176,7 @@ Resources:
Action:
- "lambda:GetLayerVersion"
Resource:
- !Ref LambdaLayerVersion
- !Ref ADFSharedPythonLambdaLayerVersion
- Effect: Allow
Action:
- "logs:CreateLogGroup"
Expand Down Expand Up @@ -1383,6 +1385,8 @@ Resources:
- codecommit:GetRepository
Resource: !GetAtt CodeCommitRepository.Arn
FunctionName: ADFPipelinesDetermineDefaultBranchName
Metadata:
BuildMethod: python3.12

InitialCommit:
Type: Custom::InitialCommit
Expand Down Expand Up @@ -1415,6 +1419,8 @@ Resources:
Resource: !GetAtt CodeCommitRepository.Arn
FunctionName: PipelinesCreateInitialCommitFunction
Timeout: 300
Metadata:
BuildMethod: python3.12

KmsKeyArnParameter:
Type: "AWS::SSM::Parameter"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@
"""

import os
import boto3

import boto3
from botocore.exceptions import ClientError

# ADF imports
from iam_cfn_deploy_role_policy import IAMCfnDeployRolePolicy
from logger import configure_logger
from parameter_store import ParameterStore
from sts import STS
from partition import get_partition
from iam_cfn_deploy_role_policy import IAMCfnDeployRolePolicy
from sts import STS


KEY_ID = os.environ["KMS_KEY_ID"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
PR_DESCRIPTION = """ADF Version {0}
You can find the changelog at:
https://github.com/awslabs/aws-deployment-framework/releases/tag/v{0}
https://github.com/awslabs/aws-deployment-framework/releases/tag/{0}
This PR was automatically created when you deployed version {0} of the
AWS Deployment Framework.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import urllib
import boto3

# ADF imports
from parameter_store import ParameterStore


Expand Down
Loading

0 comments on commit 1f0bd78

Please sign in to comment.