Skip to content

Commit

Permalink
Display CloudFormation StackEvent in the CLI (elastic#9409)
Browse files Browse the repository at this point in the history 
* Display CloudFormation StackEvent in the CLI

When creating, updating or deleting a stack on AWS a lot of different
things can go wrong, to name a few:

- Permissions
- Missing resources
- Missing files

Before when we were deploying a new cloudformation template we only
displayed information when running in debug mode internal
log statements and the final status from cloudformation.
This was actually hidding all the work that was
happening behind the scene and when needed to debug something we
were required to log into the UI and inspect manually the stack event.

This commit changes the behavior and will instead starts a goroutine that
will listen to events coming from the DescribeStackEvent's API.

The API is not the most flexible to work with, since there are no way to
specify at what point in time we are interested in receiving events and
we cannot say give me all the events from a specific Stack Change.
So instead we retrieve the list of events and skip through it until we
position ourself to when a new stack event is created. Functionbeat
stack events are small so it doesn't take too much time to position
ourself.

After we periodically poll the events and keep track of pagination, any
already seen events are ignored and everything else is printed to the
console in Info mode.

When you are using one of the deploy, update or delete subcommand and if the
stack fail you will get a stack failed error but you will be able to
inspect the log and find where and why it fails.

Example output in verbose:

```
2018-12-06T14:49:14.855-0500    INFO    instance/beat.go:597    Home path: [/Users/ph/go/src/github.com/elastic/beats/x-pack/functionbeat] Config path: [/Users/ph/go/src/github.com/elastic/beats/x-pack/functionbeat] Data path: [/Users/ph/go/src/github.com/elastic/beats/x-pack/functionbeat/data] Logs path: [/Users/ph/go/src/github.com/elastic/beats/x-pack/functionbeat/logs]
2018-12-06T14:49:14.855-0500    INFO    instance/beat.go:604    Beat UUID: 693706c5-688b-4673-a8af-e0d931894fdc
2018-12-06T14:49:28.292-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::CloudFormation::Stack, LogicalResourceId: fnb-mylogs-stack, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: User Initiated
2018-12-06T14:49:32.405-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::IAM::Role, LogicalResourceId: IAMRoleLambdaExecution, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:32.405-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::LogGroup, LogicalResourceId: fnbmylogsLogGroup, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:32.405-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::IAM::Role, LogicalResourceId: IAMRoleLambdaExecution, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:34.465-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::LogGroup, LogicalResourceId: fnbmylogsLogGroup, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:34.465-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::LogGroup, LogicalResourceId: fnbmylogsLogGroup, ResourceStatus: CREATE_COMPLETE
2018-12-06T14:49:44.804-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::IAM::Role, LogicalResourceId: IAMRoleLambdaExecution, ResourceStatus: CREATE_COMPLETE
2018-12-06T14:49:46.861-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Function, LogicalResourceId: fnbmylogs, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:48.927-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Function, LogicalResourceId: fnbmylogs, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:50.982-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Function, LogicalResourceId: fnbmylogs, ResourceStatus: CREATE_COMPLETE
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::SubscriptionFilter, LogicalResourceId: fnbmylogsSubscriptionFilterawslambdaall, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Permission, LogicalResourceId: fnbmylogsPermission0, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Permission, LogicalResourceId: fnbmylogsPermission1, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::SubscriptionFilter, LogicalResourceId: fnbmylogsSubscriptionFilterawslambdafilter, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::SubscriptionFilter, LogicalResourceId: fnbmylogsSubscriptionFilterawslambdafilter, ResourceStatus: CREATE_COMPLETE
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Permission, LogicalResourceId: fnbmylogsPermission1, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::SubscriptionFilter, LogicalResourceId: fnbmylogsSubscriptionFilterawslambdafilter, ResourceStatus: CREATE_IN_PROGRESS
2018-12-06T14:49:53.054-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Permission, LogicalResourceId: fnbmylogsPermission0, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:57.197-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::SubscriptionFilter, LogicalResourceId: fnbmylogsSubscriptionFilterawslambdaall, ResourceStatus: CREATE_IN_PROGRESS, ResourceStatusReason: Resource creation Initiated
2018-12-06T14:49:57.197-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Logs::SubscriptionFilter, LogicalResourceId: fnbmylogsSubscriptionFilterawslambdaall, ResourceStatus: CREATE_COMPLETE
2018-12-06T14:50:03.403-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Permission, LogicalResourceId: fnbmylogsPermission1, ResourceStatus: CREATE_COMPLETE
2018-12-06T14:50:03.403-0500    INFO    [aws]   aws/op_cloudformation.go:106    Stack event received, ResourceType: AWS::Lambda::Permission, LogicalResourceId: fnbmylogsPermission0, ResourceStatus: CREATE_COMPLETE
Function: mylogs, deploy successful
```

Fixes: elastic#8912
(cherry picked from commit 54314dd)
  • Loading branch information
@ph
ph committed Dec 13, 2018
1 parent 961bc1f commit d0b0e3a
Show file tree
Hide file tree
Showing 20 changed files with 1,256 additions and 231 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ https://github.com/elastic/beats/compare/v6.5.3...6.5[Check the HEAD diff]

*Functionbeat*

- The CLI will now log CloudFormation Stack events. {issue}8912[8912]

==== Bugfixes

*Affecting all Beats*
Expand Down
164 changes: 164 additions & 0 deletions vendor/github.com/aws/aws-sdk-go-v2/service/cloudformation/cloudformationiface/interface.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions vendor/vendor.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,14 @@
"version": "v2.0.0-preview.5",
"versionExact": "v2.0.0-preview.5"
},
{
"checksumSHA1": "TsYe4THK6mTAH52LC3xVkq9VE0Q=",
"path": "github.com/aws/aws-sdk-go-v2/service/cloudformation/cloudformationiface/",
"revision": "d52522b5f4b95591ff6528d7c54923951aadf099",
"revisionTime": "2018-09-27T22:51:20Z",
"version": "v2.0.0-preview.5",
"versionExact": "v2.0.0-preview.5"
},
{
"checksumSHA1": "wJrdYP7TyFYBwNPprzcA+ZdAjyo=",
"path": "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs",
Expand Down
63 changes: 39 additions & 24 deletions x-pack/functionbeat/provider/aws/cli_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/external"
cf "github.com/aws/aws-sdk-go-v2/service/cloudformation"
"github.com/awslabs/goformation/cloudformation"
merrors "github.com/pkg/errors"

Expand Down Expand Up @@ -66,7 +67,7 @@ func (c *CLIManager) findFunction(name string) (installer, error) {
return function, nil
}

func (c *CLIManager) template(function installer, name, templateLoc string) *cloudformation.Template {
func (c *CLIManager) template(function installer, name, codeLoc string) *cloudformation.Template {
lambdaConfig := function.LambdaConfig()

prefix := func(s string) string {
Expand Down Expand Up @@ -136,7 +137,7 @@ func (c *CLIManager) template(function installer, name, templateLoc string) *clo
AWSLambdaFunction: &cloudformation.AWSLambdaFunction{
Code: &cloudformation.AWSLambdaFunction_Code{
S3Bucket: c.bucket(),
S3Key: templateLoc,
S3Key: codeLoc,
},
Description: lambdaConfig.Description,
Environment: &cloudformation.AWSLambdaFunction_Environment{
Expand Down Expand Up @@ -186,9 +187,10 @@ func (c *CLIManager) deployTemplate(update bool, name string) error {

fnTemplate := function.Template()

codeLoc := codeKey(name, content)
zipChecksum := checksum(content)
codeKey := "functionbeat-deployment/" + name + "/" + zipChecksum + "/functionbeat.zip"

to := c.template(function, name, codeLoc)
to := c.template(function, name, codeKey)
if err := mergeTemplate(to, fnTemplate); err != nil {
return err
}
Expand All @@ -198,39 +200,51 @@ func (c *CLIManager) deployTemplate(update bool, name string) error {
return err
}

templateChecksum := checksum(json)
templateKey := "functionbeat-deployment/" + name + "/" + templateChecksum + "/cloudformation-template-create.json"
templateURL := "https://s3.amazonaws.com/" + c.bucket() + "/" + templateKey

c.log.Debugf("Using cloudformation template:\n%s", json)
svcCF := cf.New(c.awsCfg)

executer := newExecutor(c.log)
executer.Add(newOpEnsureBucket(c.log, c.awsCfg, c.bucket()))
executer.Add(newOpUploadToBucket(c.log, c.awsCfg, c.bucket(), codeLoc, content))
executer.Add(newOpUploadToBucket(
c.log,
c.awsCfg,
c.bucket(),
"functionbeat-deployment/"+name+"/cloudformation-template-create.json",
codeKey,
content,
))
executer.Add(newOpUploadToBucket(
c.log,
c.awsCfg,
c.bucket(),
templateKey,
json,
))
if update {
executer.Add(newOpUpdateCloudFormation(
c.log,
c.awsCfg,
"https://s3.amazonaws.com/"+c.bucket()+"/functionbeat-deployment/"+name+"/cloudformation-template-create.json",
svcCF,
templateURL,
c.stackName(name),
))
} else {
executer.Add(newOpCreateCloudFormation(
c.log,
c.awsCfg,
"https://s3.amazonaws.com/"+c.bucket()+"/functionbeat-deployment/"+name+"/cloudformation-template-create.json",
svcCF,
templateURL,
c.stackName(name),
))
}

executer.Add(newOpWaitCloudFormation(c.log, c.awsCfg, c.stackName(name)))
executer.Add(newOpDeleteFileBucket(c.log, c.awsCfg, c.bucket(), codeLoc))
executer.Add(newOpWaitCloudFormation(c.log, cf.New(c.awsCfg)))
executer.Add(newOpDeleteFileBucket(c.log, c.awsCfg, c.bucket(), codeKey))

if err := executer.Execute(); err != nil {
if rollbackErr := executer.Rollback(); rollbackErr != nil {
ctx := newStackContext()
if err := executer.Execute(ctx); err != nil {
if rollbackErr := executer.Rollback(ctx); rollbackErr != nil {
return merrors.Wrapf(err, "could not rollback, error: %s", rollbackErr)
}
return err
Expand Down Expand Up @@ -259,7 +273,7 @@ func (c *CLIManager) Update(name string) error {
return err
}

c.log.Debugf("Successfully updated function: '%s'")
c.log.Debugf("Successfully updated function: '%s'", name)
return nil
}

Expand All @@ -268,12 +282,14 @@ func (c *CLIManager) Remove(name string) error {
c.log.Debugf("Removing function: %s", name)
defer c.log.Debugf("Removal of function '%s' complete", name)

svc := cf.New(c.awsCfg)
executer := newExecutor(c.log)
executer.Add(newOpDeleteCloudFormation(c.log, c.awsCfg, c.stackName(name)))
executer.Add(newWaitDeleteCloudFormation(c.log, c.awsCfg, c.stackName(name)))
executer.Add(newOpDeleteCloudFormation(c.log, svc, c.stackName(name)))
executer.Add(newWaitDeleteCloudFormation(c.log, c.awsCfg))

if err := executer.Execute(); err != nil {
if rollbackErr := executer.Rollback(); rollbackErr != nil {
ctx := newStackContext()
if err := executer.Execute(ctx); err != nil {
if rollbackErr := executer.Rollback(ctx); rollbackErr != nil {
return merrors.Wrapf(err, "could not rollback, error: %s", rollbackErr)
}
return err
Expand Down Expand Up @@ -305,7 +321,7 @@ func NewCLI(
config: config,
provider: provider,
awsCfg: awsCfg,
log: logp.NewLogger("aws lambda cli"),
log: logp.NewLogger("aws"),
}, nil
}

Expand Down Expand Up @@ -353,8 +369,7 @@ func normalizeResourceName(s string) string {
return common.RemoveChars(s, invalidChars)
}

func codeKey(name string, content []byte) string {
sha := sha256.Sum256(content)
checksum := base64.RawURLEncoding.EncodeToString(sha[:])
return "functionbeat-deployment/" + name + "-" + checksum + "/functionbeat.zip"
func checksum(data []byte) string {
sha := sha256.Sum256(data)
return base64.RawURLEncoding.EncodeToString(sha[:])
}
8 changes: 3 additions & 5 deletions x-pack/functionbeat/provider/aws/cli_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,17 @@ import (
"github.com/elastic/beats/libbeat/common"
)

func TestCodeKey(t *testing.T) {
func TestChecksum(t *testing.T) {
t.Run("same bytes content return the same key", func(t *testing.T) {
name := "hello"
content, err := common.RandomBytes(100)
if !assert.NoError(t, err) {
return
}

assert.Equal(t, codeKey(name, content), codeKey(name, content))
assert.Equal(t, checksum(content), checksum(content))
})

t.Run("different bytes return a different key", func(t *testing.T) {
name := "hello"
content, err := common.RandomBytes(100)
if !assert.NoError(t, err) {
return
Expand All @@ -35,7 +33,7 @@ func TestCodeKey(t *testing.T) {
return
}

assert.NotEqual(t, codeKey(name, content), codeKey(name, other))
assert.NotEqual(t, checksum(content), checksum(other))
})
}

Expand Down
Loading

0 comments on commit d0b0e3a

Please sign in to comment.