diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2933522..a7b5080 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -67,4 +67,5 @@ jobs: IMAGE_TAG: ${{ github.sha }} GIPHY_API_KEY: ${{ secrets.GIPHY_API_KEY }} UNSPLASH_API_KEY: ${{ secrets.UNSPLASH_API_KEY }} - run: terraform apply --var="tag=$IMAGE_TAG" --var="giphy_api_key=$GIPHY_API_KEY" --var="unsplash_api_key=$UNSPLASH_API_KEY" -auto-approve + OPENSEARCH_ENABLED: false + run: terraform apply --var="tag=$IMAGE_TAG" --var="giphy_api_key=$GIPHY_API_KEY" --var="unsplash_api_key=$UNSPLASH_API_KEY" --var="opensearch_enabled=$OPENSEARCH_ENABLED" -auto-approve diff --git a/lambda-jobs/app.rb b/lambda-jobs/app.rb index b99d33d..ec2da00 100644 --- a/lambda-jobs/app.rb +++ b/lambda-jobs/app.rb @@ -26,12 +26,9 @@ def self.process(event:, context:) # URL: https://api.giphy.com/v1/gifs/search?api_key=&q=lgtm&limit=50&offset=0&rating=g&lang=en&bundle=messaging_non_clips URI("https://api.giphy.com/v1/gifs/random?api_key=#{ENV["GIPHY_API_KEY"]}&tag=#{event['keyword']}") else - puts "[#process] Error: Invalid source" return { "success": false } end - puts "[#process] Sending request to URL: #{url}" - # Create an HTTP client http = Net::HTTP.new(url.host, url.port) http.use_ssl = (url.scheme == 'https') @@ -42,16 +39,12 @@ def self.process(event:, context:) # Send the request and get the response response = http.request(request) - puts "[#process] Received response from API: #{response}" - # Check if the response is successful (HTTP status 200) if !response.is_a?(Net::HTTPSuccess) - puts "Error: #{response.code} - #{response.message}" + puts "[#process] Error: #{response.code} - #{response.message}" return end - puts "[#process] Parsing response from API" - formatted_results = case event['source'] when 'giphy' # Parse the JSON response @@ -82,8 +75,6 @@ def self.process(event:, context:) return { "success": false } end - puts "[#process] Formatted results: #{formatted_results}" - formatted_results.each do |result| id = result["id"] url = result["url"] @@ -103,8 +94,6 @@ def self.process(event:, context:) next end - puts "[#process] Reading image from URL: #{url}" - image_type = FastImage.type(url) img, original_img_first_frame, unedited_image_type = case image_type when :gif @@ -197,6 +186,7 @@ def self.process(event:, context:) # Upload the images (original and processed) to S3 puts "[#process] Uploading image to S3" + s3.put_object({ bucket: "lgtm-tonystrawberry-codes", key: "lgtm/#{id}.#{image_type}", @@ -211,8 +201,6 @@ def self.process(event:, context:) content_type: "image/#{unedited_image_type}" }) - puts "[#process] Image #{id} uploaded to S3" - # Analyze the image using Rekognition puts "[#process] Analyzing image using Rekognition" @@ -225,8 +213,6 @@ def self.process(event:, context:) }, }) - puts "[#process] Labels: #{response.labels}" - # Get the labels from the response (whose confidence is greater than 80%) labels = response.labels.map do |label| if label.confidence > 80 @@ -234,8 +220,9 @@ def self.process(event:, context:) end end.compact - puts "[#process] Saving image info to DynamoDB" # Save the image info to DynamoDB + puts "[#process] Saving image info to DynamoDB" + dynamodb.put_item({ table_name: "lgtm-tonystrawberry-codes", item: { diff --git a/lambda-jobs/initialize_images_data.rb b/lambda-jobs/initialize_images_data.rb index 9020b2e..3528883 100644 --- a/lambda-jobs/initialize_images_data.rb +++ b/lambda-jobs/initialize_images_data.rb @@ -37,11 +37,9 @@ # Send the request and get the response response = http.request(request) - puts "[#process] Received response from API: #{response}" - # Check if the response is successful (HTTP status 200) if !response.is_a?(Net::HTTPSuccess) - puts "Error: #{response.code} - #{response.message}" + puts "[#process] Error: #{response.code} - #{response.message}" return end diff --git a/terraform/lambda-api.zip b/terraform/lambda-api.zip index 916ddce..71360e5 100644 Binary files a/terraform/lambda-api.zip and b/terraform/lambda-api.zip differ diff --git a/terraform/lambda-api/lgtm_fetch.rb b/terraform/lambda-api/lgtm_fetch.rb index a11b16c..7c49c6d 100644 --- a/terraform/lambda-api/lgtm_fetch.rb +++ b/terraform/lambda-api/lgtm_fetch.rb @@ -62,12 +62,12 @@ def lambda_handler(event:, context:) # Fetch the images by keywords with ElasticSearch url = "#{opensearch_host}/#{opensearch_index_name}/_search" uri = URI(url) - request = Net::HTTP::POST.new(uri) + request = Net::HTTP::Post.new(uri) body = { "query": { "fuzzy": { - "keywords": { + "labels.L.S": { "value": keywords, "fuzziness": 2 } @@ -80,8 +80,8 @@ def lambda_handler(event:, context:) service: 'es', region: 'ap-northeast-1', access_key_id: ENV['AWS_ACCESS_KEY_ID'], - secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'] - # session_token: ENV['AWS_SESSION_TOKEN'] + secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'], + session_token: ENV['AWS_SESSION_TOKEN'] ).sign_request( http_method: 'POST', url: url, @@ -89,6 +89,12 @@ def lambda_handler(event:, context:) ) request.body = body + request['Host'] = signature.headers['host'] + request['X-Amz-Date'] = signature.headers['x-amz-date'] + request['X-Amz-Security-Token'] = signature.headers['x-amz-security-token'] + request['X-Amz-Content-Sha256']= signature.headers['x-amz-content-sha256'] + request['Authorization'] = signature.headers['authorization'] + request['Content-Type'] = 'application/json' response = Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http| http.request(request) @@ -96,26 +102,29 @@ def lambda_handler(event:, context:) hits_ids = JSON.parse(response.body)['hits']['hits'].map { |hit| hit['_id'] } - # Fetch the images by IDs - request_items = { - table_name => { - keys: hits_ids.map do |id| - { - id: id, - source: 'giphy' - } - end + items = [] + unless hits_ids.empty? + # Fetch the images by IDs + request_items = { + table_name => { + keys: hits_ids.map do |id| + { + id: id, + source: 'giphy' + } + end + } } - } - response = dynamodb.batch_get_item(request_items: request_items) + response = dynamodb.batch_get_item(request_items: request_items) - items = response.responses[table_name].map do |item| - { - id: item['id'], - keyword: item['keyword'], - image_url: "https://#{ENV["CLOUDFRONT_DISTRIBUTION_URL"]}/#{item['s3_key']}" - } + items = response.responses[table_name].map do |item| + { + id: item['id'], + keyword: item['keyword'], + image_url: "https://#{ENV["CLOUDFRONT_DISTRIBUTION_URL"]}/#{item['s3_key']}" + } + end end return { diff --git a/terraform/lambda-opensearch/lgtm_opensearch.rb b/terraform/lambda-opensearch/lgtm_opensearch.rb index 4cf5bbf..be36a81 100644 --- a/terraform/lambda-opensearch/lgtm_opensearch.rb +++ b/terraform/lambda-opensearch/lgtm_opensearch.rb @@ -24,11 +24,6 @@ def self.request(http_method, url, body) request = Net::HTTP::Post.new(uri) end - puts "[#OpenSearchClient.request] Sending request to URL: #{url}" - puts "[#OpenSearchClient.request] ENV['AWS_ACCESS_KEY_ID']: #{ENV['AWS_ACCESS_KEY_ID']}" - puts "[#OpenSearchClient.request] ENV['AWS_SECRET_ACCESS_KEY']: #{ENV['AWS_SECRET_ACCESS_KEY']}" - puts "[#OpenSearchClient.request] ENV['AWS_SESSION_TOKEN']: #{ENV['AWS_SESSION_TOKEN']}" - # Signature Version 4 signature = Aws::Sigv4::Signer.new( service: 'es', @@ -62,7 +57,6 @@ def self.request(http_method, url, body) end def lambda_handler(event:, context:) - puts "[#lambda_handler] Received event: #{event}" count = 0 opensearch_host = "https://search-lgtm-tonystrawberry-codes-7wakghxg7b6vvsfwyfkarv7htm.ap-northeast-1.es.amazonaws.com" index_name = "lgtm-images" diff --git a/terraform/main.tf b/terraform/main.tf index 560eb98..33ec010 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -36,10 +36,13 @@ module "storage" { source = "./modules/storage" aws_cloudfront_distribution_arn = module.cloudfront.aws_cloudfront_distribution_arn + opensearch_enabled = var.opensearch_enabled } module "opensearch" { source = "./modules/opensearch" + count = var.opensearch_enabled ? 1 : 0 + dynamo_db_stream_arn = module.storage.dynamo_db_stream_arn } diff --git a/terraform/modules/lambda/main.tf b/terraform/modules/lambda/main.tf index 4725b0e..e138a02 100644 --- a/terraform/modules/lambda/main.tf +++ b/terraform/modules/lambda/main.tf @@ -69,6 +69,13 @@ resource "aws_iam_policy" "lgtm_job_aws_iam_policy" { "dynamodb:*" ], "Resource": "arn:aws:dynamodb:*:*:*" + }, + { + "Effect": "Allow", + "Action": [ + "rekognition:*" + ], + "Resource": "*" } ] } @@ -159,6 +166,14 @@ resource "aws_iam_policy" "lgtm_api_aws_iam_policy" { "dynamodb:*" ] Resource = "arn:aws:dynamodb:*:*:*" + }, + { + Effect = "Allow", + Action = [ + "es:ESHttpPost", + "es:ESHttpPut" + ], + Resource = "*" } ] }) diff --git a/terraform/modules/opensearch/main.tf b/terraform/modules/opensearch/main.tf index 4e71efc..b20bdc5 100644 --- a/terraform/modules/opensearch/main.tf +++ b/terraform/modules/opensearch/main.tf @@ -9,7 +9,7 @@ resource "aws_opensearch_domain" "lgtm_opensearch_aws_opensearch_domain" { advanced_security_options { enabled = true - anonymous_auth_enabled = true + anonymous_auth_enabled = false internal_user_database_enabled = true master_user_options { diff --git a/terraform/modules/storage/main.tf b/terraform/modules/storage/main.tf index c3d191f..f68af13 100644 --- a/terraform/modules/storage/main.tf +++ b/terraform/modules/storage/main.tf @@ -6,7 +6,7 @@ resource "aws_dynamodb_table" "aws_dynamodb_table" { range_key = "source" stream_view_type = "NEW_AND_OLD_IMAGES" - stream_enabled = true + stream_enabled = var.opensearch_enabled ? true : false attribute { name = "id" diff --git a/terraform/modules/storage/variables.tf b/terraform/modules/storage/variables.tf index 5ff1095..041e43c 100644 --- a/terraform/modules/storage/variables.tf +++ b/terraform/modules/storage/variables.tf @@ -2,3 +2,8 @@ variable "aws_cloudfront_distribution_arn" { description = "ARN of the CloudFront distribution" type = string } + +variable "opensearch_enabled" { + description = "Whether to enable OpenSearch" + type = bool +} diff --git a/terraform/variables.tf b/terraform/variables.tf index 229a788..ebb3240 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -30,3 +30,9 @@ variable "unsplash_api_key" { description = "The Unsplash API key" type = string } + +variable "opensearch_enabled" { + description = "Whether to enable OpenSearch" + type = bool + default = false +}