Skip to content

Commit

Permalink
SAM INIT support for Advanced Logging (aws#556)
Browse files Browse the repository at this point in the history
* SAM Init support for Advanced Logging

* feature links added

* SAM Init support for Advanced Logging

* change class named from LoggingConfigTemplateModifier to LoggingConfigTemplateModifier

* update file name to match structured_logging_template_modifier
  • Loading branch information
jonife authored Nov 15, 2023
1 parent e32d800 commit c2fd98d
Show file tree
Hide file tree
Showing 11 changed files with 319 additions and 16 deletions.
10 changes: 10 additions & 0 deletions samcli/commands/init/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,11 @@ def wrapped(*args, **kwargs):
default=None,
help="Enable CloudWatch Application Insights monitoring for application.",
)
@click.option(
"--structured-logging/--no-structured-logging",
default=None,
help="Enable Structured Logging for application.",
)
@common_options
@save_params_option
@non_interactive_validation
Expand All @@ -249,6 +254,7 @@ def cli(
extra_context,
tracing,
application_insights,
structured_logging,
save_params,
config_file,
config_env,
Expand All @@ -273,6 +279,7 @@ def cli(
extra_context,
tracing,
application_insights,
structured_logging,
) # pragma: no cover


Expand All @@ -294,6 +301,7 @@ def do_cli(
extra_context,
tracing,
application_insights,
structured_logging,
):
"""
Implementation of the ``cli`` method
Expand Down Expand Up @@ -346,6 +354,7 @@ def do_cli(
extra_context,
tracing,
application_insights,
structured_logging,
)
else:
if not (pt_explicit or runtime or dependency_manager or base_image or architecture):
Expand All @@ -366,6 +375,7 @@ def do_cli(
no_input,
tracing,
application_insights,
structured_logging,
)


Expand Down
5 changes: 1 addition & 4 deletions samcli/commands/init/core/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,7 @@

CONFIGURATION_OPTION_NAMES: List[str] = ["config_env", "config_file"] + SAVE_PARAMS_OPTIONS

ADDITIONAL_OPTIONS: List[str] = [
"tracing",
"application_insights",
]
ADDITIONAL_OPTIONS: List[str] = ["tracing", "application_insights", "structured_logging"]

ALL_OPTIONS: List[str] = (
APPLICATION_OPTIONS + NON_INTERACTIVE_OPTIONS + CONFIGURATION_OPTION_NAMES + ADDITIONAL_OPTIONS + ALL_COMMON_OPTIONS
Expand Down
2 changes: 2 additions & 0 deletions samcli/commands/init/init_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ def do_generate(
extra_context,
tracing,
application_insights,
structured_logging,
):
try:
generate_project(
Expand All @@ -31,6 +32,7 @@ def do_generate(
extra_context,
tracing,
application_insights,
structured_logging,
)
except InitErrorException as e:
raise UserException(str(e), wrapped_from=e.__class__.__name__) from e
41 changes: 40 additions & 1 deletion samcli/commands/init/interactive_init_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def do_interactive(
no_input,
tracing,
application_insights,
structured_logging,
):
"""
Implementation of the ``cli`` method when --interactive is provided.
Expand Down Expand Up @@ -80,6 +81,7 @@ def do_interactive(
location_opt_choice,
tracing,
application_insights,
structured_logging,
)


Expand All @@ -98,6 +100,7 @@ def generate_application(
location_opt_choice,
tracing,
application_insights,
structured_logging,
): # pylint: disable=too-many-arguments
"""
The method holds the decision logic for generating an application
Expand Down Expand Up @@ -132,6 +135,8 @@ def generate_application(
boolen value to determine if X-Ray tracing show be activated or not
application_insights : bool
boolean value to determine if AppInsights monitoring should be enabled or not
structured_logging: bool
boolean value to determine if Json structured logging should be enabled or not
"""
if location_opt_choice == "1":
_generate_from_use_case(
Expand All @@ -147,6 +152,7 @@ def generate_application(
architecture,
tracing,
application_insights,
structured_logging,
)

else:
Expand All @@ -160,12 +166,22 @@ def generate_application(
no_input,
tracing,
application_insights,
structured_logging,
)


# pylint: disable=too-many-statements
def _generate_from_location(
location, package_type, runtime, dependency_manager, output_dir, name, no_input, tracing, application_insights
location,
package_type,
runtime,
dependency_manager,
output_dir,
name,
no_input,
tracing,
application_insights,
structured_logging,
):
location = click.prompt("\nTemplate location (git, mercurial, http(s), zip, path)", type=str)
summary_msg = """
Expand All @@ -189,6 +205,7 @@ def _generate_from_location(
None,
tracing,
application_insights,
structured_logging,
)


Expand All @@ -206,6 +223,7 @@ def _generate_from_use_case(
architecture: Optional[str],
tracing: Optional[bool],
application_insights: Optional[bool],
structured_logging: Optional[bool],
) -> None:
templates = InitTemplates()
runtime_or_base_image = runtime if runtime else base_image
Expand Down Expand Up @@ -236,6 +254,9 @@ def _generate_from_use_case(
if application_insights is None:
application_insights = prompt_user_to_enable_application_insights()

if structured_logging is None:
structured_logging = prompt_user_to_enable_structured_logging()

app_template = template_chosen["appTemplate"]
base_image = (
LAMBDA_IMAGES_RUNTIMES_MAP.get(str(runtime)) if not base_image and package_type == IMAGE else base_image
Expand Down Expand Up @@ -292,6 +313,7 @@ def _generate_from_use_case(
extra_context,
tracing,
application_insights,
structured_logging,
)
# executing event_bridge logic if call is for Schema dynamic template
if is_dynamic_schemas_template:
Expand Down Expand Up @@ -426,6 +448,23 @@ def prompt_user_to_enable_application_insights():
return False


def prompt_user_to_enable_structured_logging():
"""
Prompt user to choose if structured loggingConfig should activated
for their functions in the SAM template and vice versa
"""
if click.confirm("\nWould you like to set Structured Logging in JSON format on your Lambda functions? "):
doc_link = (
"https://docs.aws.amazon.com/lambda/latest/dg/"
"monitoring-cloudwatchlogs.html#monitoring-cloudwatchlogs-pricing"
)
click.echo(
f"Structured Logging in JSON format might incur an additional cost. View {doc_link} for more details"
)
return True
return False


def _get_choice_from_options(chosen, options, question, msg):
if chosen:
return chosen
Expand Down
13 changes: 13 additions & 0 deletions samcli/lib/init/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from samcli.lib.init.template_modifiers.application_insights_template_modifier import (
ApplicationInsightsTemplateModifier,
)
from samcli.lib.init.template_modifiers.structured_logging_template_modifier import StructuredLoggingTemplateModifier
from samcli.lib.init.template_modifiers.xray_tracing_template_modifier import XRayTracingTemplateModifier
from samcli.lib.telemetry.event import EventName, EventTracker, UsedFeature
from samcli.lib.utils import osutils
Expand All @@ -38,6 +39,7 @@ def generate_project(
extra_context=None,
tracing=False,
application_insights=False,
structured_logging=False,
):
"""Generates project using cookiecutter and options given
Expand Down Expand Up @@ -70,6 +72,8 @@ def generate_project(
Enable or disable X-Ray Tracing
application_insights: Optional[str]
Enable or disable AppInsights Monitoring
structured_logging: Optional[bool]
boolean value to determine if Json structured logging should be enabled or not
Raises
------
Expand Down Expand Up @@ -132,6 +136,8 @@ def generate_project(

_enable_application_insights(application_insights, output_dir, name)

_enable_structured_logging(structured_logging, output_dir, name)

_create_default_samconfig(package_type, output_dir, name)


Expand All @@ -150,6 +156,13 @@ def _enable_application_insights(application_insights: bool, output_dir: str, na
EventTracker.track_event(EventName.USED_FEATURE.value, UsedFeature.INIT_WITH_APPLICATION_INSIGHTS.value)


def _enable_structured_logging(structured_logging, output_dir, name):
if structured_logging:
template_file_path = f"{output_dir}/{name}/template.yaml"
template_modifier = StructuredLoggingTemplateModifier(template_file_path)
template_modifier.modify_template()


def _create_default_samconfig(package_type: str, output_dir: str, name: str) -> None:
"""
Init post-processing function to create a default samconfig.toml
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
"""
Class used to parse and update template when structured logging is enabled
"""
import logging
from typing import Any

from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from ruamel.yaml.representer import RoundTripRepresenter

from samcli.lib.init.template_modifiers.cli_template_modifier import TemplateModifier

LOG = logging.getLogger(__name__)


class StructuredLoggingTemplateModifier(TemplateModifier):
GLOBALS = "Globals"
RESOURCE = "Resources"
FUNCTION = "Function"
LOGGING_CONFIG = "LoggingConfig"
JSON_LOGFORMAT = {"LogFormat": "JSON"}
DOC_LINK = (
"https://docs.aws.amazon.com/serverless-application-model/latest/"
"developerguide/sam-resource-function.html#sam-function-loggingconfig"
)

COMMENT = (
"More info about Globals: "
"https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst\n"
)

MESSAGE = (
"You can add LoggingConfig parameters such as the Logformat, "
"Log Group, and SystemLogLevel or ApplicationLogLevel. "
f"Learn more here {DOC_LINK}.\n"
)

# set ignore aliases to true. This configuration avoids usage yaml aliases which is not parsed by CloudFormation.
class NonAliasingRTRepresenter(RoundTripRepresenter):
def ignore_aliases(self, data):
return True

def __init__(self, location):
self.yaml = YAML()
self.yaml.Representer = StructuredLoggingTemplateModifier.NonAliasingRTRepresenter
super().__init__(location)

def _get_template(self) -> Any:
with open(self.template_location) as file:
return self.yaml.load(file)

def _update_template_fields(self):
"""
Add new field to SAM template
"""
if self.template.get(self.GLOBALS):
template_globals = self.template.get(self.GLOBALS)

function_globals = template_globals.get(self.FUNCTION, {})
if not function_globals:
template_globals[self.FUNCTION] = {}
template_globals[self.FUNCTION][self.LOGGING_CONFIG] = CommentedMap(self.JSON_LOGFORMAT)
template_globals[self.FUNCTION].yaml_set_comment_before_after_key(
self.LOGGING_CONFIG, before=self.MESSAGE, indent=4
)

else:
self._add_logging_config_with_globals()

def _add_logging_config_with_globals(self):
"""Adds Globals and LoggingConfig fields"""
global_section = {
self.FUNCTION: {self.LOGGING_CONFIG: self.JSON_LOGFORMAT},
}

self.template = CommentedMap(self.template)
self.template[self.GLOBALS] = CommentedMap(global_section)
self.template[self.GLOBALS].yaml_set_comment_before_after_key(self.LOGGING_CONFIG, before=self.MESSAGE)
self.template.yaml_set_comment_before_after_key(self.GLOBALS, before=self.COMMENT)

def _print_sanity_check_error(self):
message = (
"Warning: Unable to add LoggingConfig to the project. "
"To learn more about LoggingConfig visit {self.DOC_LINK}"
)
LOG.warning(message)

def _write(self, template: list):
"""
write generated template into SAM template
Parameters
----------
template : list
array with updated template data
"""
with open(self.template_location, "w") as file:
self.yaml.dump(self.template, file)
7 changes: 6 additions & 1 deletion schema/samcli.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"properties": {
"parameters": {
"title": "Parameters for the init command",
"description": "Available parameters for the init command:\n* no_interactive:\nDisable interactive prompting for init parameters. (fail if any required values are missing)\n* architecture:\nArchitectures for Lambda functions.\n\nArchitectures: ['arm64', 'x86_64']\n* location:\nTemplate location (git, mercurial, http(s), zip, path).\n* runtime:\nLambda runtime for application.\n\nRuntimes: dotnet6, go1.x, java17, java11, java8.al2, java8, nodejs18.x, nodejs16.x, nodejs14.x, nodejs12.x, provided, provided.al2, python3.9, python3.8, python3.7, python3.11, python3.10, ruby3.2, ruby2.7\n* package_type:\nLambda deployment package type.\n\nPackage Types: Zip, Image\n* base_image:\nLambda base image for deploying IMAGE based package type.\n\nBase images: amazon/dotnet6-base, amazon/go-provided.al2-base, amazon/go1.x-base, amazon/java11-base, amazon/java17-base, amazon/java8-base, amazon/java8.al2-base, amazon/nodejs12.x-base, amazon/nodejs14.x-base, amazon/nodejs16.x-base, amazon/nodejs18.x-base, amazon/python3.10-base, amazon/python3.11-base, amazon/python3.7-base, amazon/python3.8-base, amazon/python3.9-base, amazon/ruby2.7-base, amazon/ruby3.2-base\n* dependency_manager:\nDependency manager for Lambda runtime.\n\nDependency managers: bundler, cli-package, gradle, maven, mod, npm, pip\n* output_dir:\nDirectory to initialize AWS SAM application.\n* name:\nName of AWS SAM Application.\n* app_template:\nIdentifier of the managed application template to be used. Alternatively, run '$sam init' without options for an interactive workflow.\n* no_input:\nDisable Cookiecutter prompting and accept default values defined in the cookiecutter config.\n* extra_context:\nOverride custom parameters in the template's cookiecutter.json configuration e.g. {\"customParam1\": \"customValue1\", \"customParam2\":\"customValue2\"}\n* tracing:\nEnable AWS X-Ray tracing for application.\n* application_insights:\nEnable CloudWatch Application Insights monitoring for application.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* save_params:\nSave the parameters provided via the command line to the configuration file.",
"description": "Available parameters for the init command:\n* no_interactive:\nDisable interactive prompting for init parameters. (fail if any required values are missing)\n* architecture:\nArchitectures for Lambda functions.\n\nArchitectures: ['arm64', 'x86_64']\n* location:\nTemplate location (git, mercurial, http(s), zip, path).\n* runtime:\nLambda runtime for application.\n\nRuntimes: dotnet6, go1.x, java17, java11, java8.al2, java8, nodejs18.x, nodejs16.x, nodejs14.x, nodejs12.x, provided, provided.al2, python3.9, python3.8, python3.7, python3.11, python3.10, ruby3.2, ruby2.7\n* package_type:\nLambda deployment package type.\n\nPackage Types: Zip, Image\n* base_image:\nLambda base image for deploying IMAGE based package type.\n\nBase images: amazon/dotnet6-base, amazon/go-provided.al2-base, amazon/go1.x-base, amazon/java11-base, amazon/java17-base, amazon/java8-base, amazon/java8.al2-base, amazon/nodejs12.x-base, amazon/nodejs14.x-base, amazon/nodejs16.x-base, amazon/nodejs18.x-base, amazon/python3.10-base, amazon/python3.11-base, amazon/python3.7-base, amazon/python3.8-base, amazon/python3.9-base, amazon/ruby2.7-base, amazon/ruby3.2-base\n* dependency_manager:\nDependency manager for Lambda runtime.\n\nDependency managers: bundler, cli-package, gradle, maven, mod, npm, pip\n* output_dir:\nDirectory to initialize AWS SAM application.\n* name:\nName of AWS SAM Application.\n* app_template:\nIdentifier of the managed application template to be used. Alternatively, run '$sam init' without options for an interactive workflow.\n* no_input:\nDisable Cookiecutter prompting and accept default values defined in the cookiecutter config.\n* extra_context:\nOverride custom parameters in the template's cookiecutter.json configuration e.g. {\"customParam1\": \"customValue1\", \"customParam2\":\"customValue2\"}\n* tracing:\nEnable AWS X-Ray tracing for application.\n* application_insights:\nEnable CloudWatch Application Insights monitoring for application.\n* structured_logging:\nEnable Structured Logging for application.\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* save_params:\nSave the parameters provided via the command line to the configuration file.",
"type": "object",
"properties": {
"no_interactive": {
Expand Down Expand Up @@ -155,6 +155,11 @@
"type": "boolean",
"description": "Enable CloudWatch Application Insights monitoring for application."
},
"structured_logging": {
"title": "structured_logging",
"type": "boolean",
"description": "Enable Structured Logging for application."
},
"beta_features": {
"title": "beta_features",
"type": "boolean",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def test_unknown_runtime(self, git_repo_mock, requests_mock):
no_input=False,
tracing=False,
application_insights=False,
structured_logging=False,
)
output_files = list(self.output_dir.rglob("*"))
self.assertEqual(len(output_files), 9)
Expand Down
Loading

0 comments on commit c2fd98d

Please sign in to comment.