diff --git a/lib/pipeline-watcher/index.ts b/lib/pipeline-watcher/index.ts new file mode 100644 index 00000000..d38a6f09 --- /dev/null +++ b/lib/pipeline-watcher/index.ts @@ -0,0 +1 @@ +export * from './watcher'; \ No newline at end of file diff --git a/lib/pipeline-watcher/watcher-handler.ts b/lib/pipeline-watcher/watcher-handler.ts new file mode 100644 index 00000000..a52174ac --- /dev/null +++ b/lib/pipeline-watcher/watcher-handler.ts @@ -0,0 +1,34 @@ +import AWS = require('aws-sdk'); + +// export for tests +export const codePipeline = new AWS.CodePipeline(); +export const logger = { + log: (line: string) => process.stdout.write(line) +}; + +/** + * Lambda function for checking the stages of a CodePipeline and emitting log + * entries with { failedCount = } for async metric + * aggregation via metric filters. + * + * It requires the pipeline's name be set as the 'PIPELINE_NAME' environment variable. + */ +export async function handler() { + const pipelineName = process.env.PIPELINE_NAME; + if (!pipelineName) { + throw new Error("Pipeline name expects environment variable: 'PIPELINE_NAME'"); + } + const state = await codePipeline.getPipelineState({ + name: pipelineName + }).promise(); + + let failedCount = 0; + if (state.stageStates) { + failedCount = state.stageStates + .filter(stage => stage.latestExecution !== undefined && stage.latestExecution.status === 'Failed') + .length; + } + logger.log(JSON.stringify({ + failedCount + })); +} diff --git a/lib/pipeline-watcher/watcher.ts b/lib/pipeline-watcher/watcher.ts new file mode 100644 index 00000000..01871455 --- /dev/null +++ b/lib/pipeline-watcher/watcher.ts @@ -0,0 +1,96 @@ +import cloudwatch = require('@aws-cdk/aws-cloudwatch'); +import cpipeline = require('@aws-cdk/aws-codepipeline'); +import events = require('@aws-cdk/aws-events'); +import iam = require('@aws-cdk/aws-iam'); +import lambda = require('@aws-cdk/aws-lambda'); +import logs = require('@aws-cdk/aws-logs'); +import cdk = require('@aws-cdk/cdk'); +import fs = require('fs'); +import path = require('path'); + +export interface PipelineWatcherProps { + /** + * Code Pipeline to monitor for failed stages + */ + pipeline: cpipeline.Pipeline; + + /** + * Set the pipelineName of the alarm description. + * + * Description is set to 'Pipeline has failed stages' + * + * @default pipeline's name + */ + title?: string; +} + +/** + * Construct which watches a Code Pipeline for failed stages and raises an alarm + * if there are any failed stages. + * + * A function runs every minute and calls GetPipelineState for the provided pipeline's + * name, counts the number of failed stages and emits a JSON log { failedCount: <number> }. + * A metric filter is then configured to track this value as a CloudWatch metric, and + * a corresponding alarm is set to fire when the maximim value of a single 5-minute interval + * is >= 1. + */ +export class PipelineWatcher extends cdk.Construct { + public readonly alarm: cloudwatch.Alarm; + + constructor(parent: cdk.Construct, name: string, props: PipelineWatcherProps) { + super(parent, name); + + const pipelineWatcher = new lambda.Function(this, 'Poller', { + handler: 'index.handler', + runtime: lambda.Runtime.NodeJS810, + code: lambda.Code.inline(fs.readFileSync(path.join(__dirname, 'watcher-handler.js')).toString('utf8')), + environment: { + PIPELINE_NAME: props.pipeline.pipelineName + } + }); + + // See https://github.com/awslabs/aws-cdk/issues/1340 for exposing grants on the pipeline. + pipelineWatcher.addToRolePolicy(new iam.PolicyStatement() + .addResource(props.pipeline.pipelineArn) + .addAction('codepipeline:GetPipelineState')); + + // ex: arn:aws:logs:us-east-1:123456789012:log-group:my-log-group + const logGroup = new logs.LogGroup(this, 'Logs', { + logGroupName: `/aws/lambda/${pipelineWatcher.functionName}`, + retentionDays: 731 + }); + + const trigger = new events.EventRule(this, 'Trigger', { + scheduleExpression: 'rate(1 minute)', + targets: [pipelineWatcher] + }); + + const logGroupResource = logGroup.findChild('Resource') as cdk.Resource; + const triggerResource = trigger.findChild('Resource') as cdk.Resource; + triggerResource.addDependency(logGroupResource); + + const metricNamespace = `CDK/Delivlib`; + const metricName = `${props.pipeline.pipelineName}_FailedStages`; + + new logs.MetricFilter(this, 'MetricFilter', { + filterPattern: logs.FilterPattern.exists('$.failedCount'), + metricNamespace, + metricName, + metricValue: '$.failedCount', + logGroup + }); + + this.alarm = new cloudwatch.Alarm(this, 'Alarm', { + alarmDescription: `Pipeline ${props.title || props.pipeline.pipelineName} has failed stages`, + metric: new cloudwatch.Metric({ + metricName, + namespace: metricNamespace, + statistic: cloudwatch.Statistic.Maximum + }), + threshold: 1, + comparisonOperator: cloudwatch.ComparisonOperator.GreaterThanOrEqualToThreshold, + evaluationPeriods: 1, + treatMissingData: cloudwatch.TreatMissingData.Ignore, // We expect a steady stream of data points + }); + } +} \ No newline at end of file diff --git a/lib/pipeline.ts b/lib/pipeline.ts index c2a8f7b0..7e0eff39 100644 --- a/lib/pipeline.ts +++ b/lib/pipeline.ts @@ -6,6 +6,7 @@ import iam = require('@aws-cdk/aws-iam'); import sns = require('@aws-cdk/aws-sns'); import cdk = require('@aws-cdk/cdk'); import path = require('path'); +import { PipelineWatcher } from './pipeline-watcher'; import publishing = require('./publishing'); import { IRepo } from './repo'; import { Testable, TestableProps } from './testable'; @@ -255,26 +256,11 @@ export class Pipeline extends cdk.Construct { })); } - private addFailureAlarm(title?: string) { - const pipelineFailureTopic = new sns.Topic(this, 'PipelineFailureTopic'); - - this.pipeline.onStateChange('PipelineFailureEvent', pipelineFailureTopic, { - eventPattern: { detail: { state: [ 'FAILED' ] } } - }); - - new cloudwatch.Alarm(this, 'PipelineFailureAlarm', { - alarmDescription: `Pipeline ${title || ''} Failed`, - metric: new cloudwatch.Metric({ - metricName: 'NumberOfMessagesPublished', - namespace: 'SNS', - statistic: cloudwatch.Statistic.Sum, - dimensions: { TopicName: pipelineFailureTopic.topicName } - }), - threshold: 1, - comparisonOperator: cloudwatch.ComparisonOperator.GreaterThanOrEqualToThreshold, - evaluationPeriods: 1, - treatMissingData: cloudwatch.TreatMissingData.NotBreaching, - }); + private addFailureAlarm(title?: string): cloudwatch.Alarm { + return new PipelineWatcher(this, 'PipelineWatcher', { + pipeline: this.pipeline, + title + }).alarm; } } diff --git a/package.json b/package.json index 4b665206..efe1bf79 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "build": "tsc && tslint --fix --project .", "package": "/bin/bash ./package.sh", "watch": "tsc -w", - "test": "/bin/bash ./test/run-test.sh", + "test": "/bin/bash ./test/run-test.sh && jest", "cdk": "cdk", "pipeline-update": "npm run build && cdk -a pipeline/delivlib.js deploy", "pipeline-diff": "npm run build && cdk -a pipeline/delivlib.js diff" diff --git a/test/expected.json b/test/expected.json index 04320558..905a6e1d 100644 --- a/test/expected.json +++ b/test/expected.json @@ -290,35 +290,6 @@ Resources: RegisterWithThirdParty: true Metadata: aws:cdk:path: delivlib-test/CodeCommitPipeline/BuildPipeline/Source/Pull/WebhookResource - CodeCommitPipelineBuildPipelinePipelineFailureEventA9CF6CC6: - Type: AWS::Events::Rule - Properties: - EventPattern: - detail: - state: - - FAILED - detail-type: - - CodePipeline Pipeline Execution State Change - source: - - aws.codepipeline - resources: - - Fn::Join: - - "" - - - "arn:" - - Ref: AWS::Partition - - ":codepipeline:" - - Ref: AWS::Region - - ":" - - Ref: AWS::AccountId - - ":" - - Ref: CodeCommitPipelineBuildPipeline656B8CCB - State: ENABLED - Targets: - - Arn: - Ref: CodeCommitPipelinePipelineFailureTopicCF6E97E9 - Id: PipelineFailureTopic - Metadata: - aws:cdk:path: delivlib-test/CodeCommitPipeline/BuildPipeline/PipelineFailureEvent/Resource CodeCommitPipelinesuperchainAdoptRepository67DBB814: Type: Custom::ECRAdoptedRepository Properties: @@ -480,47 +451,185 @@ Resources: Type: CODEPIPELINE Metadata: aws:cdk:path: delivlib-test/CodeCommitPipeline/BuildProject/Resource - CodeCommitPipelinePipelineFailureTopicCF6E97E9: - Type: AWS::SNS::Topic + CodeCommitPipelinePipelineWatcherPollerServiceRole0A1D8005: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: lambda.amazonaws.com + Version: "2012-10-17" + ManagedPolicyArns: + - Fn::Join: + - "" + - - "arn:" + - Ref: AWS::Partition + - :iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Metadata: - aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineFailureTopic/Resource - CodeCommitPipelinePipelineFailureTopicPolicy921957D6: - Type: AWS::SNS::TopicPolicy + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Poller/ServiceRole/Resource + CodeCommitPipelinePipelineWatcherPollerServiceRoleDefaultPolicyE2104AD1: + Type: AWS::IAM::Policy Properties: PolicyDocument: Statement: - - Action: sns:Publish + - Action: codepipeline:GetPipelineState Effect: Allow - Principal: - Service: events.amazonaws.com Resource: - Ref: CodeCommitPipelinePipelineFailureTopicCF6E97E9 - Sid: "0" + Fn::Join: + - "" + - - "arn:" + - Ref: AWS::Partition + - ":codepipeline:" + - Ref: AWS::Region + - ":" + - Ref: AWS::AccountId + - ":" + - Ref: CodeCommitPipelineBuildPipeline656B8CCB Version: "2012-10-17" - Topics: - - Ref: CodeCommitPipelinePipelineFailureTopicCF6E97E9 + PolicyName: CodeCommitPipelinePipelineWatcherPollerServiceRoleDefaultPolicyE2104AD1 + Roles: + - Ref: CodeCommitPipelinePipelineWatcherPollerServiceRole0A1D8005 Metadata: - aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineFailureTopic/Policy/Resource - CodeCommitPipelinePipelineFailureAlarm9B90F586: + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Poller/ServiceRole/DefaultPolicy/Resource + CodeCommitPipelinePipelineWatcherPoller5C65ACDE: + Type: AWS::Lambda::Function + Properties: + Code: + ZipFile: >- + "use strict"; + + Object.defineProperty(exports, "__esModule", { value: true }); + + const AWS = require("aws-sdk"); + + // export for tests + + exports.codePipeline = new AWS.CodePipeline(); + + exports.logger = { + log: (line) => process.stdout.write(line) + }; + + /** + * Lambda function for checking the stages of a CodePipeline and emitting log + * entries with { failedCount = <no. of failed stages> } for async metric + * aggregation via metric filters. + * + * It requires the pipeline's name be set as the 'PIPELINE_NAME' environment variable. + */ + async function handler() { + const pipelineName = process.env.PIPELINE_NAME; + if (!pipelineName) { + throw new Error("Pipeline name expects environment variable: 'PIPELINE_NAME'"); + } + const state = await exports.codePipeline.getPipelineState({ + name: pipelineName + }).promise(); + let failedCount = 0; + if (state.stageStates) { + failedCount = state.stageStates + .filter(stage => stage.latestExecution !== undefined && stage.latestExecution.status === 'Failed') + .length; + } + exports.logger.log(JSON.stringify({ + failedCount + })); + } + + exports.handler = handler; + + //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoid2F0Y2hlci1oYW5kbGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsid2F0Y2hlci1oYW5kbGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7O0FBQUEsK0JBQWdDO0FBRWhDLG1CQUFtQjtBQUNOLFFBQUEsWUFBWSxHQUFHLElBQUksR0FBRyxDQUFDLFlBQVksRUFBRSxDQUFDO0FBQ3RDLFFBQUEsTUFBTSxHQUFHO0lBQ3BCLEdBQUcsRUFBRSxDQUFDLElBQVksRUFBRSxFQUFFLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDO0NBQ2xELENBQUM7QUFFRjs7Ozs7O0dBTUc7QUFDSSxLQUFLLFVBQVUsT0FBTztJQUMzQixNQUFNLFlBQVksR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLGFBQWEsQ0FBQztJQUMvQyxJQUFJLENBQUMsWUFBWSxFQUFFO1FBQ2pCLE1BQU0sSUFBSSxLQUFLLENBQUMsNkRBQTZELENBQUMsQ0FBQztLQUNoRjtJQUNELE1BQU0sS0FBSyxHQUFHLE1BQU0sb0JBQVksQ0FBQyxnQkFBZ0IsQ0FBQztRQUNoRCxJQUFJLEVBQUUsWUFBWTtLQUNuQixDQUFDLENBQUMsT0FBTyxFQUFFLENBQUM7SUFFYixJQUFJLFdBQVcsR0FBRyxDQUFDLENBQUM7SUFDcEIsSUFBSSxLQUFLLENBQUMsV0FBVyxFQUFFO1FBQ3JCLFdBQVcsR0FBRyxLQUFLLENBQUMsV0FBVzthQUM1QixNQUFNLENBQUMsS0FBSyxDQUFDLEVBQUUsQ0FBQyxLQUFLLENBQUMsZUFBZSxLQUFLLFNBQVMsSUFBSSxLQUFLLENBQUMsZUFBZSxDQUFDLE1BQU0sS0FBSyxRQUFRLENBQUM7YUFDakcsTUFBTSxDQUFDO0tBQ1g7SUFDRCxjQUFNLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUM7UUFDeEIsV0FBVztLQUNaLENBQUMsQ0FBQyxDQUFDO0FBQ04sQ0FBQztBQWxCRCwwQkFrQkMifQ== + Handler: index.handler + Role: + Fn::GetAtt: + - CodeCommitPipelinePipelineWatcherPollerServiceRole0A1D8005 + - Arn + Runtime: nodejs8.10 + Environment: + Variables: + PIPELINE_NAME: + Ref: CodeCommitPipelineBuildPipeline656B8CCB + DependsOn: + - CodeCommitPipelinePipelineWatcherPollerServiceRole0A1D8005 + - CodeCommitPipelinePipelineWatcherPollerServiceRoleDefaultPolicyE2104AD1 + Metadata: + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Poller/Resource + CodeCommitPipelinePipelineWatcherPollerAllowEventRuledelivlibtestCodeCommitPipelinePipelineWatcherTrigger0F55C26402F871FD: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: CodeCommitPipelinePipelineWatcherPoller5C65ACDE + Principal: events.amazonaws.com + SourceArn: + Fn::GetAtt: + - CodeCommitPipelinePipelineWatcherTriggerA38A4AD0 + - Arn + Metadata: + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Poller/AllowEventRuledelivlibtestCodeCommitPipelinePipelineWatcherTrigger0F55C264 + CodeCommitPipelinePipelineWatcherLogs5DE54482: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: + Fn::Join: + - "" + - - /aws/lambda/ + - Ref: CodeCommitPipelinePipelineWatcherPoller5C65ACDE + RetentionInDays: 731 + DeletionPolicy: Retain + Metadata: + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Logs/Resource + CodeCommitPipelinePipelineWatcherTriggerA38A4AD0: + Type: AWS::Events::Rule + Properties: + ScheduleExpression: rate(1 minute) + State: ENABLED + Targets: + - Arn: + Fn::GetAtt: + - CodeCommitPipelinePipelineWatcherPoller5C65ACDE + - Arn + Id: Poller + DependsOn: + - CodeCommitPipelinePipelineWatcherLogs5DE54482 + Metadata: + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Trigger/Resource + CodeCommitPipelinePipelineWatcherMetricFilter1D1A0C4D: + Type: AWS::Logs::MetricFilter + Properties: + FilterPattern: '{ $.failedCount = "*" }' + LogGroupName: + Ref: CodeCommitPipelinePipelineWatcherLogs5DE54482 + MetricTransformations: + - MetricName: + Fn::Join: + - "" + - - Ref: CodeCommitPipelineBuildPipeline656B8CCB + - _FailedStages + MetricNamespace: CDK/Delivlib + MetricValue: $.failedCount + Metadata: + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/MetricFilter/Resource + CodeCommitPipelinePipelineWatcherAlarm73779F48: Type: AWS::CloudWatch::Alarm Properties: ComparisonOperator: GreaterThanOrEqualToThreshold EvaluationPeriods: 1 Threshold: 1 - AlarmDescription: Pipeline aws-delivlib test pipeline Failed - Dimensions: - - Name: TopicName - Value: - Fn::GetAtt: - - CodeCommitPipelinePipelineFailureTopicCF6E97E9 - - TopicName - MetricName: NumberOfMessagesPublished - Namespace: SNS + AlarmDescription: Pipeline aws-delivlib test pipeline has failed stages + MetricName: + Fn::Join: + - "" + - - Ref: CodeCommitPipelineBuildPipeline656B8CCB + - _FailedStages + Namespace: CDK/Delivlib Period: 300 - Statistic: Sum - TreatMissingData: notBreaching + Statistic: Maximum + TreatMissingData: ignore Metadata: - aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineFailureAlarm/Resource + aws:cdk:path: delivlib-test/CodeCommitPipeline/PipelineWatcher/Alarm/Resource CodeCommitPipelineDashboard6FF06F29: Type: AWS::CloudWatch::Dashboard Properties: @@ -2371,10 +2480,10 @@ Resources: =0.19.0,@aws-cdk/aws-codecommit=0.19.0,@aws-cdk/aws-codepipeline=0.19.0\ ,@aws-cdk/aws-codepipeline-api=0.19.0,@aws-cdk/aws-ec2=0.19.0,@aws-cdk/\ aws-ecr=0.19.0,@aws-cdk/aws-events=0.19.0,@aws-cdk/aws-iam=0.19.0,@aws-\ - cdk/aws-kms=0.19.0,@aws-cdk/aws-lambda=0.19.0,@aws-cdk/aws-s3=0.19.0,@a\ - ws-cdk/aws-s3-notifications=0.19.0,@aws-cdk/aws-sns=0.19.0,@aws-cdk/aws\ - -sqs=0.19.0,@aws-cdk/aws-ssm=0.19.0,@aws-cdk/cdk=0.19.0,@aws-cdk/cx-api\ - =0.19.0,aws-delivlib=0.1.1" + cdk/aws-kms=0.19.0,@aws-cdk/aws-lambda=0.19.0,@aws-cdk/aws-logs=0.19.0,\ + @aws-cdk/aws-s3=0.19.0,@aws-cdk/aws-s3-notifications=0.19.0,@aws-cdk/aw\ + s-sns=0.19.0,@aws-cdk/aws-sqs=0.19.0,@aws-cdk/aws-ssm=0.19.0,@aws-cdk/c\ + dk=0.19.0,@aws-cdk/cx-api=0.19.0,aws-delivlib=0.1.2" Parameters: CodeCommitPipelineBuildPipelineGitHubTokenParameter9FBEDC6B: Type: AWS::SSM::Parameter::Value<String> diff --git a/test/watcher-handler.test.ts b/test/watcher-handler.test.ts new file mode 100644 index 00000000..8380b2ca --- /dev/null +++ b/test/watcher-handler.test.ts @@ -0,0 +1,106 @@ +import { codePipeline, handler, logger } from '../lib/pipeline-watcher/watcher-handler'; +codePipeline.getPipelineState = jest.fn(); + +test('handler should propagate error if GetPipelineState fails', async () => { + process.env.PIPELINE_NAME = 'name'; + expect.assertions(2); + codePipeline.getPipelineState = jest.fn(request => { + expect(request).toEqual({ name: 'name' }); + return { + promise: () => new Promise((_, reject) => reject(new Error('fail'))) + }; + }); + try { + await handler(); + } catch (err) { + expect(err.message).toEqual('fail'); + } +}); + +test('handler should throw error if process.env.PIPELINE_NAME is undefined', async () => { + delete process.env.PIPELINE_NAME; + expect.assertions(1); + try { + await handler(); + } catch (err) { + expect(err.message).toEqual("Pipeline name expects environment variable: 'PIPELINE_NAME'"); + } +}); + +// prepare log with a new mock, set the name env variable and mock the getPipelineState fn. +function mock(response: any) { + logger.log = jest.fn(); + process.env.PIPELINE_NAME = 'name'; + codePipeline.getPipelineState = jest.fn(request => { + expect(request.name).toEqual(process.env.PIPELINE_NAME); + return { + promise: () => new Promise((resolve) => resolve(response)) + }; + }); +} + +test('handler should log {failCount: 0} if pipeline.stageStates is undefined', async () => { + mock({}); + await handler(); + expect(logger.log).toBeCalledTimes(1); + expect(logger.log).toBeCalledWith(JSON.stringify({failedCount: 0})); +}); + +test('handler should log {failCount: 0} if pipeline.stageStates is empty', async () => { + mock({ stageStates: [] }); + await handler(); + expect(logger.log).toBeCalledTimes(1); + expect(logger.log).toBeCalledWith(JSON.stringify({failedCount: 0})); +}); + +test('handler should log {failCount: 0} if pipeline.stageStates[:0].latestExecution are undefined', async () => { + mock({ stageStates: [{ + latestExecution: undefined + }] }); + await handler(); + expect(logger.log).toBeCalledTimes(1); + expect(logger.log).toBeCalledWith(JSON.stringify({failedCount: 0})); +}); + +test('handler should log {failCount: 0} if none of pipeline.stageStates[:0].latestExecution.status are Failed', async () => { + mock({ stageStates: [{ + latestExecution: { + status: 'Success' + } + }] }); + await handler(); + expect(logger.log).toBeCalledTimes(1); + expect(logger.log).toBeCalledWith(JSON.stringify({failedCount: 0})); +}); + +test('handler should log {failCount: 1} if one of pipeline.stageStates[:0].latestExecution.status is Failed', async () => { + mock({ stageStates: [{ + latestExecution: { + status: 'Failed' + } + }] }); + await handler(); + expect(logger.log).toBeCalledTimes(1); + expect(logger.log).toBeCalledWith(JSON.stringify({failedCount: 1})); +}); + +test('handler should log {failCount: 2} for 2 "Failed" and 1 "Success" pipeline.stageStates[:0].latestExecution.status values', async () => { + mock({ + stageStates: [{ + latestExecution: { + status: 'Failed' + } + }, { + latestExecution: { + status: 'Sucess' + } + }, { + latestExecution: { + status: 'Failed' + } + }] + }); + await handler(); + expect(logger.log).toBeCalledTimes(1); + expect(logger.log).toBeCalledWith(JSON.stringify({failedCount: 2})); +});