Draining mechanism draft - identify all TODOs

#25 (comment)

lambda_golang/cmd/landing_metadata/main.go → lambda_golang/cmd/landing_metadata_cronjob/main.go

@@ -36,6 +36,12 @@ type LambdaResponse struct {
 }
 
 func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) {
+
+	// TODO: create a cronjob rule in tf
+	// TODO: point cronjob to this func
+
+	// TODO: change from s3Event to fetch from dynamoDB (query)
+
 	GoTools.Logger("INFO", "Landing page metadata.json generator launched")
 
 	for _, record := range s3Event.Records {
@@ -58,6 +64,10 @@ func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse,
 
 		bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
 		GoTools.Logger("INFO", fmt.Sprintf("Saved landing page metadata to s3://%s/%s", bucket, metadataS3Key))
+
+		// TODO: update `isDocTypeWaitingForMetadata` on landing page entry in db
+
+		// (optional) TODO: add event "metadataGenFinish" on landing page entry in db
 	}
 
 	return LambdaResponse{

lambda_golang/cmd/landing_s3_trigger/main.go

@@ -0,0 +1,135 @@
+// You must use `main` package for lambda
+// https://stackoverflow.com/a/50701572/9814131
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	//"math"
+
+	"github.com/aws/aws-lambda-go/events" // https://github.com/aws/aws-lambda-go/blob/main/events/README_S3.md
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/rivernews/GoTools"
+
+	//"github.com/aws/aws-sdk-go-v2/aws"
+	//"github.com/aws/aws-sdk-go-v2/config"
+	//"github.com/aws/aws-sdk-go-v2/service/sqs"
+
+	// local packages
+	"github.com/rivernews/media-literacy/pkg/cloud"
+	"github.com/rivernews/media-literacy/pkg/newssite"
+)
+
+func main() {
+	lambda.Start(HandleRequest)
+}
+
+type StepFunctionInput struct {
+	LandingS3Key string `json:"landingS3Key"`
+}
+
+type LambdaResponse struct {
+	OK      bool   `json:"OK:"`
+	Message string `json:"message:"`
+}
+
+func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) {
+	GoTools.Logger("INFO", "Landing page metadata.json generator launched")
+
+	for _, record := range s3Event.Records {
+		landingPageS3Key := record.S3.Object.URLDecodedKey
+		GoTools.Logger("INFO", fmt.Sprintf("Captured landing page at %s", landingPageS3Key))
+
+		// TODO: push into dynamoDB instead
+
+		// TODO: let landing s3 trigger switch to point to this func
+
+		// landingPageHtmlText := cloud.Pull(landingPageS3Key)
+		// landingPageS3KeyTokens := strings.Split(landingPageS3Key, "/")
+		// metadataS3DirKeyTokens := landingPageS3KeyTokens[:len(landingPageS3KeyTokens)-1]
+		// metadataS3Key := fmt.Sprintf("%s/metadata.json", strings.Join(metadataS3DirKeyTokens, "/"))
+
+		// result := newssite.GetStoriesFromEconomy(landingPageHtmlText)
+		// metadataJSONString := GoTools.AsJson(result)
+
+		// cloud.Archive(cloud.ArchiveArgs{
+		// 	BodyText:          metadataJSONString,
+		// 	Key:               metadataS3Key,
+		// 	FileTypeExtension: "json",
+		// })
+
+		// bucket := GoTools.GetEnvVarHelper("S3_ARCHIVE_BUCKET")
+		// GoTools.Logger("INFO", fmt.Sprintf("Saved landing page metadata to s3://%s/%s", bucket, metadataS3Key))
+	}
+
+	return LambdaResponse{
+		OK:      true,
+		Message: "Done",
+	}, nil
+
+	/*
+
+		// e.g. 90 links in total
+		// chunk size := 30
+		// chunk count = 3
+		chunkSize := 30
+		chunkCount := int(math.Ceil(float64(len(stories) / chunkSize)))
+		linkChunks := make([][]string, chunkCount)
+
+		for i := 0; i < chunkCount; i++ {
+			linkChunk := make([]string, chunkSize)
+			for j := 0; j < chunkSize; j++ {
+				linkChunk[j] = stories[i*chunkSize+j].URL
+			}
+			linkChunks[i] = linkChunk
+		}
+
+		GoTools.Logger("INFO", fmt.Sprintf("Pulled landing page content:\n ``` %s ``` \n ", landingPageHtmlText[:500]))
+
+		for _, linkChunk := range linkChunks {
+			// send SQS
+			// refer to
+			// https://aws.github.io/aws-sdk-go-v2/docs/code-examples/sqs/sendmessage/
+			queueName := GoTools.GetEnvVarHelper("STORIES_QUEUE_NAME")
+			awsConfig, configErr := config.LoadDefaultConfig(context.TODO())
+			if configErr != nil {
+				GoTools.Logger("ERROR", "AWS shared configuration failed", configErr.Error())
+			}
+			sqsClient := sqs.NewFromConfig(awsConfig)
+			getQueueResponse, getQueueError := sqsClient.GetQueueUrl(context.TODO(), &sqs.GetQueueUrlInput{QueueName: aws.String(queueName)})
+			if getQueueError != nil {
+				GoTools.Logger("ERROR", fmt.Sprintf("Error getting queue URL: %s", getQueueError.Error()))
+			}
+			queueURL := getQueueResponse.QueueUrl
+
+			res, err := sqsClient.SendMessage(context.TODO(), &sqs.SendMessageInput{
+				// AWS required attributes
+				// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
+				// Golang API Reference
+				// https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/sqs#SendMessageInput
+
+				QueueUrl:    queueURL,
+				MessageBody: aws.String(strings.Join(linkChunk, " ")),
+
+				// Only FIFO queue can use `MessageGroupId`
+				// MessageGroupId: aws.String(fmt.Sprintf("%s-00", queueName)),
+
+				// TODO: add randomized delay
+				DelaySeconds: 0,
+			})
+
+			if err != nil {
+				GoTools.Logger("ERROR", fmt.Sprintf("Error sending message: %s", err))
+			}
+			GoTools.SimpleLogger("INFO", fmt.Sprintf("Message sent %s", *res.MessageId))
+		}
+
+		return LambdaResponse{
+			OK:      true,
+			Message: fmt.Sprintf("Sent %d messages OK", len(linkChunks)),
+		}, nil
+
+	*/
+}

lambda_golang/cmd/stories/main.go

@@ -70,6 +70,8 @@ func HandleRequest(ctx context.Context, S3Event events.S3Event) (LambdaResponse,
 
 		GoTools.Logger("INFO", fmt.Sprintf("Sfn output ``` %s ```\n", GoTools.AsJson(sfnOutput)))
 
+		// TODO: add event "storiesFetchStart" on landing page entry in db
+
 		/*
 			storyChunk := message.Body
 			GoTools.Logger("INFO", fmt.Sprintf("Story consumer! story chunk: %s", storyChunk))

lambda_golang/cmd/stories_finalizer/main.go

@@ -0,0 +1 @@
+// TODO: Sfn add a step after map - log event "FinishStoriesFetchingAll" into DDB landing page object pipelineEvents.