Skip to content

Commit

Permalink
All tests complete
Browse files Browse the repository at this point in the history 
Tracked by #25 (comment)
  • Loading branch information
@rivernews
rivernews committed Sep 29, 2022
1 parent 233ce60 commit d2f752c
Show file tree
Hide file tree
Showing 9 changed files with 58 additions and 26 deletions.
11 changes: 11 additions & 0 deletions cloud_module/pipeline/s3_triggers.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ EOF

attach_policy_statements = true
policy_statements = {
allow_db_put = {
effect = "Allow",
actions = [
"dynamodb:UpdateItem",
],
resources = [
local.media_table_arn,
]
}
s3_archive_bucket = {
effect = "Allow",
actions = [
Expand All @@ -84,8 +93,10 @@ EOF
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
LOGLEVEL = "DEBUG"
ENV = local.environment
DEBUG = "true"

S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
DYNAMODB_TABLE_ID = local.media_table_id
SFN_ARN = module.batch_stories_sfn.state_machine_arn
}

Expand Down
11 changes: 11 additions & 0 deletions cloud_module/pipeline/stories_sfn.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,15 @@ module fetch_story_lambda {

attach_policy_statements = true
policy_statements = {
allow_db_put = {
effect = "Allow",
actions = [
"dynamodb:PutItem",
],
resources = [
local.media_table_arn,
]
}
s3_archive_bucket = {
effect = "Allow",
actions = [
Expand Down Expand Up @@ -77,6 +86,8 @@ module fetch_story_lambda {
LOG_LEVEL = "DEBUG"
DEBUG = "true"
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id

DYNAMODB_TABLE_ID = local.media_table_id
}

tags = {
Expand Down
2 changes: 2 additions & 0 deletions lambda_golang/cmd/landing_metadata_cronjob/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ func HandleRequest(ctx context.Context, cronjobEvent events.CloudWatchEvent) (La
landingPageMetadata := newssite.GetStoriesFromEconomy(landingPageHtmlText)
landingPageMetadata.LandingPageS3Key = landingPageS3Key
landingPageMetadata.LandingPageUuid = landingItem.Uuid
landingPageMetadata.LandingPageCreatedAt = landingItem.CreatedAt
metadataJSONString := GoTools.AsJson(landingPageMetadata)

cloud.Archive(cloud.ArchiveArgs{
Expand All @@ -62,6 +63,7 @@ func HandleRequest(ctx context.Context, cronjobEvent events.CloudWatchEvent) (La
newssite.DynamoDBUpdateItemMarkAsMetadataComplete(
ctx,
landingItem.Uuid,
landingItem.CreatedAt,
newssite.GetEventLandingMetadataDone(metadataS3Key, landingPageS3Key),
)

Expand Down
10 changes: 8 additions & 2 deletions lambda_golang/cmd/stories/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,14 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse,
GoTools.Logger("INFO", fmt.Sprintf("Test first story: %d:%d", len(metadata.Stories), len(metadata.UntitledStories)))

// fire step function, input = {stories: [{}, {}, {}...], newsSiteAlias:"", landingPageTimeStamp:""}
var processStories []newssite.Topic
if GoTools.Debug {
processStories = metadata.Stories[:2]
} else {
processStories = metadata.Stories
}
sfnInput := GoTools.AsJson(&newssite.StepFunctionInput{
Stories: metadata.Stories,
Stories: processStories,
NewsSiteAlias: newsSiteAlias,
LandingPageUuid: metadata.LandingPageUuid,
LandingPageS3Key: metadata.LandingPageS3Key,
Expand All @@ -69,7 +75,7 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse,
StateMachineArn: aws.String(sfnArn),
},
)
newssite.DynamoDBUpdateItemAddEvent(ctx, metadata.LandingPageUuid, newssite.GetEventLandingStoriesRequested(metadataS3Key))
newssite.DynamoDBUpdateItemAddEvent(ctx, metadata.LandingPageUuid, metadata.LandingPageCreatedAt, newssite.GetEventLandingStoriesRequested(metadataS3Key, len(processStories)))

GoTools.Logger("INFO", fmt.Sprintf("Sfn output ``` %s ```\n", GoTools.AsJson(sfnOutput)))

Expand Down
10 changes: 6 additions & 4 deletions lambda_golang/cmd/stories_finalizer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,19 @@ type LambdaResponse struct {
func HandleRequest(ctx context.Context, stepFunctionInput newssite.StepFunctionInput) (LambdaResponse, error) {
GoTools.Logger("INFO", "Stories finalizer launched")

items := newssite.DynamoDBQueryByS3Key(ctx, stepFunctionInput.LandingPageS3Key)
result := newssite.DynamoDBQueryByS3Key(ctx, stepFunctionInput.LandingPageS3Key)

if len(*items) != 1 {
if len(*result) != 1 {
GoTools.Logger("ERROR", fmt.Sprintf(
"Stories finalizer expect exactly one landing page `%s`, but query resulted in `%d` items",
stepFunctionInput.LandingPageS3Key,
len(*items),
len(*result),
))
}

newssite.DynamoDBUpdateItemAddEvent(ctx, stepFunctionInput.LandingPageUuid, newssite.GetEventLandingStoriesFetched(stepFunctionInput.LandingPageS3Key))
landingPageItem := (*result)[0]

newssite.DynamoDBUpdateItemAddEvent(ctx, landingPageItem.Uuid, landingPageItem.CreatedAt, newssite.GetEventLandingStoriesFetched(stepFunctionInput.LandingPageS3Key))

return LambdaResponse{
OK: true,
Expand Down
2 changes: 1 addition & 1 deletion lambda_golang/cmd/story/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func HandleRequest(ctx context.Context, stepFunctionMapIterationInput newssite.S
stepFunctionMapIterationInput.Story.URL,
),
},
IsDocTypeWaitingForMetadata: newssite.DOCTYPE_LANDING,
IsDocTypeWaitingForMetadata: newssite.DOCTYPE_STORY,
},
)

Expand Down
6 changes: 0 additions & 6 deletions lambda_golang/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,22 @@ github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxB
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2 v1.16.15 h1:2sInOWGE4HV54R90Pj8QgqBBw3Qf1I0husqbqjPZzys=
github.com/aws/aws-sdk-go-v2 v1.16.15/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk=
github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
github.com/aws/aws-sdk-go-v2/config v1.7.0 h1:J2cZ7qe+3IpqBEXnHUrFrOjoB9BlsXg7j53vxcl5IVg=
github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
github.com/aws/aws-sdk-go-v2/credentials v1.4.0 h1:kmvesfjY861FzlCU9mvAfe01D9aeXcG2ZuC+k9F2YLM=
github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.19 h1:WlbMnYXjKlntGC7LDjnzj/33u+9fRHJkPIFCrdmF/JQ=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.9.19/go.mod h1:+CBJZMhsb1pTUcB/NTdS505bDX10xS4xnPMqDZj2Ptw=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 h1:bKbdstt7+PzIRSIXZ11Yo8Qh8t0AHn6jEYUfsbVcLjE=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0/go.mod h1:+CBJZMhsb1pTUcB/NTdS505bDX10xS4xnPMqDZj2Ptw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 h1:OxTAgH8Y4BXHD6PGCJ8DHx2kaZPCQfSTqmDsdRZFezE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0/go.mod h1:CpNzHK9VEFUCknu50kkB8z58AH2B5DvPP7ea1LHve/Y=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0 h1:zzH1xd1/PX7bFO4/BQwVQP+UcBfYieI1sMH9DA68xZY=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0/go.mod h1:fPU0eFGnS47RyKHHs8BNcCKOm5oOA5xm0BlrZWsQT/A=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22 h1:pE27/u2A7JlwICjOvONQDob8PToShRTkuiUE74ymVWg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22/go.mod h1:/vNv5Al0bpiF8YdX2Ov6Xy05VTiXsql94yUqJMYaj0w=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16 h1:L5LKGHHXOl4t7+5QZMTl38GIzSAq07XUTRtEquiHGMA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16/go.mod h1:62dsXI0BqTIGomDl8Hpm33dv0OntGaVblri3ZRParVQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
Expand All @@ -36,7 +31,6 @@ github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 h1:1QpTkQIAaZpR387it1L+erj
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1/go.mod h1:BZhn/C3z13ULTSstVi2Kymc62bgjFh/JwLO9Tm2OFYI=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 h1:V9q4A0qnUfDsfivspY1LQRQTOG3Y9FLHvXIaTbcU7XM=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20/go.mod h1:7qWU48SMzlrfOlNhHpazW3psFWlOIWrq4SmOr2/ESmk=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw=
Expand Down
9 changes: 5 additions & 4 deletions lambda_golang/pkg/newssite/media.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@ type Topic struct {
}

type LandingPageMetadata struct {
LandingPageS3Key string `json:"landingPageS3Key"`
LandingPageUuid string `json:"landingPageUuid"`
Stories []Topic `json:"stories"`
UntitledStories []Topic `json:"untitledstories"`
LandingPageS3Key string `json:"landingPageS3Key"`
LandingPageUuid string `json:"landingPageUuid"`
LandingPageCreatedAt string `json:"landingPageCreatedAt"`
Stories []Topic `json:"stories"`
UntitledStories []Topic `json:"untitledstories"`
}

type StepFunctionInput struct {
Expand Down
23 changes: 14 additions & 9 deletions lambda_golang/pkg/newssite/media_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,10 @@ func GetEventLandingMetadataDone(metadataS3Key string, landingPageS3Key string)
}
}

func GetEventLandingStoriesRequested(metadataS3Key string) MediaTableItemEvent {
func GetEventLandingStoriesRequested(metadataS3Key string, storiesCount int) MediaTableItemEvent {
return MediaTableItemEvent{
EventName: EVENT_LANDING_STORIES_REQUESTED,
Detail: fmt.Sprintf("Stories requested for landing page based on metadata %s", metadataS3Key),
Detail: fmt.Sprintf("Stories(%d) requested for landing page based on metadata %s", storiesCount, metadataS3Key),
EventTime: common.Now(),
}
}
Expand Down Expand Up @@ -170,21 +170,26 @@ func DynamoDBQueryWaitingMetadata(ctx context.Context, docType DocType) *[]Media
return &results
}

func DynamoDBUpdateItem(ctx context.Context, uuid string, event MediaTableItemEvent, isMarkMetadataComplete bool) *dynamodb.UpdateItemOutput {
func DynamoDBUpdateItem(ctx context.Context, uuid string, createdAt string, event MediaTableItemEvent, isMarkMetadataComplete bool) *dynamodb.UpdateItemOutput {
dynamoDBItemEvent, err := attributevalue.MarshalMap(event)
if err != nil {
GoTools.Logger("ERROR", err.Error())
}
updateItemInput := dynamodb.UpdateItemInput{
TableName: aws.String(cloud.GetTableName()),
Key: map[string]types.AttributeValue{
"uuid": &types.AttributeValueMemberS{Value: uuid},
"uuid": &types.AttributeValueMemberS{Value: uuid},
"createdAt": &types.AttributeValueMemberS{Value: createdAt},
},
// manual
// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.UpdateExpressions.html#Expressions.UpdateExpressions.ADD
UpdateExpression: aws.String(`SET events = list_append(events, :e)`),
ExpressionAttributeValues: map[string]types.AttributeValue{
":e": &types.AttributeValueMemberM{Value: dynamoDBItemEvent},
":e": &types.AttributeValueMemberL{
Value: []types.AttributeValue{
&types.AttributeValueMemberM{Value: dynamoDBItemEvent},
},
},
},
}
if isMarkMetadataComplete {
Expand All @@ -200,10 +205,10 @@ func DynamoDBUpdateItem(ctx context.Context, uuid string, event MediaTableItemEv
return out
}

func DynamoDBUpdateItemAddEvent(ctx context.Context, uuid string, event MediaTableItemEvent) *dynamodb.UpdateItemOutput {
return DynamoDBUpdateItem(ctx, uuid, event, false)
func DynamoDBUpdateItemAddEvent(ctx context.Context, uuid string, createdAt string, event MediaTableItemEvent) *dynamodb.UpdateItemOutput {
return DynamoDBUpdateItem(ctx, uuid, createdAt, event, false)
}

func DynamoDBUpdateItemMarkAsMetadataComplete(ctx context.Context, uuid string, event MediaTableItemEvent) *dynamodb.UpdateItemOutput {
return DynamoDBUpdateItem(ctx, uuid, event, true)
func DynamoDBUpdateItemMarkAsMetadataComplete(ctx context.Context, uuid string, createdAt string, event MediaTableItemEvent) *dynamodb.UpdateItemOutput {
return DynamoDBUpdateItem(ctx, uuid, createdAt, event, true)
}

0 comments on commit d2f752c

Please sign in to comment.