From c2ac4f7065f0584b75fe6eb5bff8aec5ccd282e7 Mon Sep 17 00:00:00 2001 From: Dieg0Code Date: Sun, 11 Aug 2024 03:47:59 -0400 Subject: [PATCH] lambda --- .github/workflows/cicd.yaml | 83 +++++++++ Makefile | 9 +- .../product_controller_impl_test.go | 135 ++++++++++++++ api/repository/product_repository_impl.go | 5 +- .../product_repository_impl_test.go | 163 +++++++++++++++++ api/service/product_service_impl_test.go | 172 ++++++++++++++++++ api/utils/scraper.go | 2 +- api/utils/scraper_impl.go | 26 ++- api/utils/scraper_impl_test.go | 23 +++ go.mod | 9 +- go.sum | 3 +- main.go | 22 ++- terraform/.terraform.lock.hcl | 1 + terraform/api_gateway.tf | 71 +++----- terraform/dynamodb.tf | 24 +-- terraform/iam.tf | 32 ++-- terraform/lambda.tf | 20 +- terraform/variables.tf | 10 + 18 files changed, 695 insertions(+), 115 deletions(-) create mode 100644 .github/workflows/cicd.yaml create mode 100644 api/controller/product_controller_impl_test.go create mode 100644 api/repository/product_repository_impl_test.go create mode 100644 api/service/product_service_impl_test.go create mode 100644 api/utils/scraper_impl_test.go create mode 100644 terraform/variables.tf diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml new file mode 100644 index 0000000..e1491de --- /dev/null +++ b/.github/workflows/cicd.yaml @@ -0,0 +1,83 @@ +name: CI/CD Pipeline + +on: + push: + branches: + - main + +jobs: + test-and-build: + name: Test and Build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: '1.22.4' + + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install dependencies + run: go mod download + + - name: Run tests + run: go test -coverprofile=coverage.out ./... + + - name: Upload coverage to codecov + uses: codecov/codecov-action@v2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Build binary + run: GOOS=linux GOARCH=amd64 go build -o api_scraper_lambda + + - name: Zip binary + run: zip api_scraper_lambda.zip api_scraper_lambda + + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: api_scraper_lambda + path: api_scraper_lambda.zip + + deploy: + name: Deploy + runs-on: ubuntu-latest + needs: test-and-build + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifact + uses: actions/download-artifact@v2 + with: + name: api_scraper_lambda + path: . + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v1 + + - name: Initialize Terraform + working-directory: ./terraform + run: terraform init + + - name: Apply Terraform + working-directory: ./terraform + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: terraform apply -auto-approve -var="aws_region=${{ secrets.AWS_REGION }}" -var="aws_account_id=${{ secrets.AWS_ACCOUNT_ID }}" + + - name: Get API Gateway URL + working-directory: ./terraform + run: echo "API Gateway URL => https://$(terraform output -json | jq -r '.api_gateway_invoke_url')" + \ No newline at end of file diff --git a/Makefile b/Makefile index 9a0a0b4..4e0ec91 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,14 @@ compile_lambda: - set GOOS=linux&& set GOARCH=amd64&& set CGO_ENABLED=0&& go build -o main main.go + set GOOS=linux&& set GOARCH=amd64&& set CGO_ENABLED=0&& go build -o api_scraper_lambda main.go + +zip_lambda: + zip api_scraper_lambda.zip api_scraper_lambda + + start_db: docker run -d --name dynamodb -p 8000:8000 amazon/dynamodb-local + + create_table: aws dynamodb create-table \ --table-name products \ diff --git a/api/controller/product_controller_impl_test.go b/api/controller/product_controller_impl_test.go new file mode 100644 index 0000000..0586c79 --- /dev/null +++ b/api/controller/product_controller_impl_test.go @@ -0,0 +1,135 @@ +package controller + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/dieg0code/serverles-api-scraper/api/data/request" + "github.com/dieg0code/serverles-api-scraper/api/data/response" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockProductService struct { + mock.Mock +} + +func (m *MockProductService) GetAll() ([]response.ProductResponse, error) { + args := m.Called() + return args.Get(0).([]response.ProductResponse), args.Error(1) +} +func (m *MockProductService) GetByID(productID string) (response.ProductResponse, error) { + args := m.Called(productID) + return args.Get(0).(response.ProductResponse), args.Error(1) +} +func (m *MockProductService) UpdateData(updateData request.UpdateDataRequest) (bool, error) { + args := m.Called(updateData) + return args.Bool(0), args.Error(1) +} + +func TestProductController_GetAll(t *testing.T) { + gin.SetMode(gin.TestMode) + mockService := new(MockProductService) + productController := NewProductControllerImpl(mockService) + + router := gin.Default() + router.GET("/products", productController.GetAll) + + mockService.On("GetAll").Return([]response.ProductResponse{ + { + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + }, + { + ProductID: "test-id-2", + Name: "Test Product 2", + Category: "Test Category 2", + OriginalPrice: 200, + DiscountedPrice: 180, + }, + }, nil) + + req, err := http.NewRequest(http.MethodGet, "/products", nil) + assert.NoError(t, err, "Expected no error creating request") + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code, "Expected status code 200") + + var response response.BaseResponse + err = json.Unmarshal(rec.Body.Bytes(), &response) + assert.NoError(t, err, "Expected no error unmarshalling response") + assert.Equal(t, 200, response.Code, "Response code should be 200") + assert.Equal(t, "OK", response.Status, "Response status should be Success") +} + +func TestProductController_GetByID(t *testing.T) { + gin.SetMode(gin.TestMode) + mockService := new(MockProductService) + productController := NewProductControllerImpl(mockService) + + router := gin.Default() + router.GET("/products/:productId", productController.GetByID) + + mockService.On("GetByID", "test-id").Return(response.ProductResponse{ + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + }, nil) + + req, err := http.NewRequest(http.MethodGet, "/products/test-id", nil) + assert.NoError(t, err, "Expected no error creating request") + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code, "Expected status code 200") + + var response response.BaseResponse + err = json.Unmarshal(rec.Body.Bytes(), &response) + assert.NoError(t, err, "Expected no error unmarshalling response") + assert.Equal(t, 200, response.Code, "Response code should be 200") + assert.Equal(t, "OK", response.Status, "Response status should be Success") +} + +func TestProductController_UpdateData(t *testing.T) { + gin.SetMode(gin.TestMode) + mockService := new(MockProductService) + productController := NewProductControllerImpl(mockService) + + router := gin.Default() + router.PUT("/products", productController.UpdateData) + + mockService.On("UpdateData", request.UpdateDataRequest{ + UpdateData: true, + }).Return(true, nil) + + reqBody, err := json.Marshal(request.UpdateDataRequest{ + UpdateData: true, + }) + assert.NoError(t, err, "Expected no error marshalling request") + + req, err := http.NewRequest(http.MethodPut, "/products", bytes.NewBuffer(reqBody)) + assert.NoError(t, err, "Expected no error creating request") + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code, "Expected status code 200") + + var response response.BaseResponse + err = json.Unmarshal(rec.Body.Bytes(), &response) + assert.NoError(t, err, "Expected no error unmarshalling response") + assert.Equal(t, 200, response.Code, "Response code should be 200") + assert.Equal(t, "OK", response.Status, "Response status should be Success") +} diff --git a/api/repository/product_repository_impl.go b/api/repository/product_repository_impl.go index 38c0fa4..d8a08de 100644 --- a/api/repository/product_repository_impl.go +++ b/api/repository/product_repository_impl.go @@ -7,12 +7,13 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" "github.com/dieg0code/serverles-api-scraper/api/models" "github.com/sirupsen/logrus" ) type ProductRepositoryImpl struct { - db *dynamodb.DynamoDB + db dynamodbiface.DynamoDBAPI tableName string } @@ -138,7 +139,7 @@ func (p *ProductRepositoryImpl) GetByID(id string) (models.Product, error) { return product, nil } -func NewProductRepositoryImpl(db *dynamodb.DynamoDB, tableName string) ProductRepository { +func NewProductRepositoryImpl(db dynamodbiface.DynamoDBAPI, tableName string) ProductRepository { return &ProductRepositoryImpl{ db: db, tableName: tableName, diff --git a/api/repository/product_repository_impl_test.go b/api/repository/product_repository_impl_test.go new file mode 100644 index 0000000..fbe932b --- /dev/null +++ b/api/repository/product_repository_impl_test.go @@ -0,0 +1,163 @@ +package repository + +import ( + "testing" + + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" + "github.com/dieg0code/serverles-api-scraper/api/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// MockDynamoDB is a mock implementation of the DynamoDBAPI interface +type MockDynamoDB struct { + dynamodbiface.DynamoDBAPI + mock.Mock +} + +func (m *MockDynamoDB) PutItem(input *dynamodb.PutItemInput) (*dynamodb.PutItemOutput, error) { + args := m.Called(input) + return args.Get(0).(*dynamodb.PutItemOutput), args.Error(1) +} + +func (m *MockDynamoDB) GetItem(input *dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) { + args := m.Called(input) + return args.Get(0).(*dynamodb.GetItemOutput), args.Error(1) +} + +func (m *MockDynamoDB) Scan(input *dynamodb.ScanInput) (*dynamodb.ScanOutput, error) { + args := m.Called(input) + return args.Get(0).(*dynamodb.ScanOutput), args.Error(1) +} + +func (m *MockDynamoDB) DeleteItem(input *dynamodb.DeleteItemInput) (*dynamodb.DeleteItemOutput, error) { + args := m.Called(input) + return args.Get(0).(*dynamodb.DeleteItemOutput), args.Error(1) +} + +// NewProductRepositoryImpl creates a new ProductRepositoryImpl with the given DynamoDBAPI and table name +func TestProductRepositoryImpl_Create(t *testing.T) { + mockDB := new(MockDynamoDB) + repo := NewProductRepositoryImpl(mockDB, "test-table") + + product := models.Product{ + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + } + + mockDB.On("PutItem", mock.Anything).Return(&dynamodb.PutItemOutput{}, nil) + + createdProduct, err := repo.Create(product) + + assert.NoError(t, err, "Expected no error, Create() returned an error") + assert.Equal(t, product, createdProduct, "Expected created product to be equal to the input product") + mockDB.AssertExpectations(t) +} + +// NewProductRepositoryImpl creates a new ProductRepositoryImpl with the given DynamoDBAPI and table name +func TestProductRepositoryImpl_GetByID(t *testing.T) { + mockDB := new(MockDynamoDB) + repo := NewProductRepositoryImpl(mockDB, "test-table") + + expectedProduct := models.Product{ + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + } + + mockDB.On("GetItem", mock.Anything).Return(&dynamodb.GetItemOutput{ + Item: map[string]*dynamodb.AttributeValue{ + "ProductID": {S: &expectedProduct.ProductID}, + "Name": {S: &expectedProduct.Name}, + "Category": {S: &expectedProduct.Category}, + "OriginalPrice": {N: stringPtr("100")}, + "DiscountedPrice": {N: stringPtr("90")}, + }, + }, nil) + + product, err := repo.GetByID("test-id") + + assert.NoError(t, err, "Expected no error, GetByID() returned an error") + assert.Equal(t, expectedProduct, product, "Expected product to be equal to the expected product") + mockDB.AssertExpectations(t) +} + +// NewProductRepositoryImpl creates a new ProductRepositoryImpl with the given DynamoDBAPI and table name +func TestProductRepositoryImpl_GetAll(t *testing.T) { + mockDB := new(MockDynamoDB) + repo := NewProductRepositoryImpl(mockDB, "test-table") + + expectedProducts := []models.Product{ + { + ProductID: "test-id-1", + Name: "Test Product 1", + Category: "Test Category 1", + OriginalPrice: 100, + DiscountedPrice: 90, + }, + { + ProductID: "test-id-2", + Name: "Test Product 2", + Category: "Test Category 2", + OriginalPrice: 200, + DiscountedPrice: 180, + }, + } + + mockDB.On("Scan", mock.Anything).Return(&dynamodb.ScanOutput{ + Items: []map[string]*dynamodb.AttributeValue{ + { + "ProductID": {S: &expectedProducts[0].ProductID}, + "Name": {S: &expectedProducts[0].Name}, + "Category": {S: &expectedProducts[0].Category}, + "OriginalPrice": {N: stringPtr("100")}, + "DiscountedPrice": {N: stringPtr("90")}, + }, + { + "ProductID": {S: &expectedProducts[1].ProductID}, + "Name": {S: &expectedProducts[1].Name}, + "Category": {S: &expectedProducts[1].Category}, + "OriginalPrice": {N: stringPtr("200")}, + "DiscountedPrice": {N: stringPtr("180")}, + }, + }, + }, nil) + + products, err := repo.GetAll() + + assert.NoError(t, err, "Expected no error, GetAll() returned an error") + assert.Equal(t, expectedProducts, products, "Expected products to be equal to the expected products") + mockDB.AssertExpectations(t) +} + +// NewProductRepositoryImpl creates a new ProductRepositoryImpl with the given DynamoDBAPI and table name +func TestProductRepositoryImpl_DeleteAll(t *testing.T) { + mockDB := new(MockDynamoDB) + repo := NewProductRepositoryImpl(mockDB, "test-table") + + // Mock Scan response + mockDB.On("Scan", mock.Anything).Return(&dynamodb.ScanOutput{ + Items: []map[string]*dynamodb.AttributeValue{ + { + "ProductID": {S: stringPtr("test-id-1")}, + }, + }, + }, nil) + + // Mock DeleteItem response + mockDB.On("DeleteItem", mock.Anything).Return(&dynamodb.DeleteItemOutput{}, nil) + + err := repo.DeleteAll() + assert.NoError(t, err, "Expected no error, DeleteAll() returned an error") + mockDB.AssertExpectations(t) +} + +func stringPtr(s string) *string { + return &s +} diff --git a/api/service/product_service_impl_test.go b/api/service/product_service_impl_test.go new file mode 100644 index 0000000..8da55b7 --- /dev/null +++ b/api/service/product_service_impl_test.go @@ -0,0 +1,172 @@ +package service + +import ( + "testing" + + "github.com/dieg0code/serverles-api-scraper/api/data/request" + "github.com/dieg0code/serverles-api-scraper/api/data/response" + "github.com/dieg0code/serverles-api-scraper/api/models" + "github.com/dieg0code/serverles-api-scraper/api/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockProductRepository struct { + mock.Mock +} + +type MockScraper struct { + mock.Mock +} + +func (m *MockProductRepository) GetAll() ([]models.Product, error) { + args := m.Called() + return args.Get(0).([]models.Product), args.Error(1) +} +func (m *MockProductRepository) GetByID(id string) (models.Product, error) { + args := m.Called(id) + return args.Get(0).(models.Product), args.Error(1) +} +func (m *MockProductRepository) Create(product models.Product) (models.Product, error) { + args := m.Called(product) + return args.Get(0).(models.Product), args.Error(1) +} +func (m *MockProductRepository) DeleteAll() error { + args := m.Called() + return args.Error(0) +} + +func (m *MockScraper) ScrapeData(baseURL string, maxPage int, category string) ([]models.Product, error) { + args := m.Called(baseURL, maxPage, category) + return args.Get(0).([]models.Product), args.Error(1) +} + +func (m *MockScraper) CleanPrice(price string) (int, error) { + args := m.Called(price) + return args.Int(0), args.Error(1) +} + +func TestPoductService_GetAll(t *testing.T) { + mockRepo := new(MockProductRepository) + mockScraper := new(MockScraper) + productService := NewProductServiceImpl(mockRepo, mockScraper) + + expectedProducts := []response.ProductResponse{ + { + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + }, + { + ProductID: "test-id-2", + Name: "Test Product 2", + Category: "Test Category 2", + OriginalPrice: 200, + DiscountedPrice: 190, + }, + } + + mockRepo.On("GetAll").Return([]models.Product{ + { + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + }, + { + ProductID: "test-id-2", + Name: "Test Product 2", + Category: "Test Category 2", + OriginalPrice: 200, + DiscountedPrice: 190, + }, + }, nil) + + products, err := productService.GetAll() + + assert.NoError(t, err, "Expected no error, GetAll() returned an error") + assert.Equal(t, expectedProducts, products, "Expected products to be equal to the expected products") + + mockRepo.AssertExpectations(t) + +} + +func TestPoductService_GetByID(t *testing.T) { + mockRepo := new(MockProductRepository) + mockScraper := new(MockScraper) + productService := NewProductServiceImpl(mockRepo, mockScraper) + + expectedProduct := response.ProductResponse{ + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + } + + mockRepo.On("GetByID", "test-id").Return(models.Product{ + ProductID: "test-id", + Name: "Test Product", + Category: "Test Category", + OriginalPrice: 100, + DiscountedPrice: 90, + }, nil) + + product, err := productService.GetByID("test-id") + + assert.NoError(t, err, "Expected no error, GetByID() returned an error") + assert.Equal(t, expectedProduct, product, "Expected product to be equal to the expected product") + + mockRepo.AssertExpectations(t) +} + +func TestProductService_UpdateData(t *testing.T) { + mockRepo := new(MockProductRepository) + mockScraper := new(MockScraper) + productService := NewProductServiceImpl(mockRepo, mockScraper) + + mockRepo.On("DeleteAll").Return(nil) + + utils.Categories = []utils.CategoryInfo{ + {MaxPage: 10, Category: "bebidas-alcoholicas"}, + {MaxPage: 5, Category: "alimentos-basicos"}, + } + + mockScraper.On("ScrapeData", "cugat.cl/categoria-producto", 10, "bebidas-alcoholicas").Return([]models.Product{ + { + Name: "Product 1", + Category: "bebidas-alcoholicas", + OriginalPrice: 100, + DiscountedPrice: 90, + }, + }, nil) + + mockScraper.On("ScrapeData", "cugat.cl/categoria-producto", 5, "alimentos-basicos").Return([]models.Product{ + { + Name: "Product 2", + Category: "alimentos-basicos", + OriginalPrice: 200, + DiscountedPrice: 180, + }, + }, nil) + + mockRepo.On("Create", mock.MatchedBy(func(product models.Product) bool { + return product.Name == "Product 1" && product.Category == "bebidas-alcoholicas" || + product.Name == "Product 2" && product.Category == "alimentos-basicos" + })).Return(models.Product{}, nil) + + updateDataRequest := request.UpdateDataRequest{ + UpdateData: true, + } + + success, err := productService.UpdateData(updateDataRequest) + + assert.NoError(t, err, "Expected no error, UpdateData() returned an error") + assert.True(t, success, "Expected success to be true") + + mockRepo.AssertExpectations(t) + mockScraper.AssertExpectations(t) +} diff --git a/api/utils/scraper.go b/api/utils/scraper.go index c51eb5c..2f81bdb 100644 --- a/api/utils/scraper.go +++ b/api/utils/scraper.go @@ -4,5 +4,5 @@ import "github.com/dieg0code/serverles-api-scraper/api/models" type Scraper interface { ScrapeData(baseURL string, maxPage int, category string) ([]models.Product, error) - cleanPrice(price string) (int, error) + CleanPrice(price string) (int, error) } diff --git a/api/utils/scraper_impl.go b/api/utils/scraper_impl.go index a660de3..f6f98e8 100644 --- a/api/utils/scraper_impl.go +++ b/api/utils/scraper_impl.go @@ -10,10 +10,12 @@ import ( "github.com/sirupsen/logrus" ) -type ScraperImpl struct{} +type ScraperImpl struct { + Collector *colly.Collector +} // cleanPrice implements Scraper. -func (s *ScraperImpl) cleanPrice(price string) (int, error) { +func (s *ScraperImpl) CleanPrice(price string) (int, error) { cleaned := strings.ReplaceAll(price, "$", "") cleaned = strings.ReplaceAll(cleaned, ".", "") return strconv.Atoi(cleaned) @@ -21,11 +23,9 @@ func (s *ScraperImpl) cleanPrice(price string) (int, error) { // scrapeData implements Scraper. func (s *ScraperImpl) ScrapeData(baseURL string, maxPage int, category string) ([]models.Product, error) { - collector := colly.NewCollector() - var products []models.Product - collector.OnHTML(".product-small.box", func(e *colly.HTMLElement) { + s.Collector.OnHTML(".product-small.box", func(e *colly.HTMLElement) { name := e.ChildText(".name.product-title a") category := e.ChildText(".category") originalPriceStr := e.ChildText(".price del .woocommerce-Price-amount.amount") @@ -35,12 +35,12 @@ func (s *ScraperImpl) ScrapeData(baseURL string, maxPage int, category string) ( originalPriceStr = e.ChildText(".price .woocommerce-Price-amount.amount") } - originalPrice, err := s.cleanPrice(originalPriceStr) + originalPrice, err := s.CleanPrice(originalPriceStr) if err != nil { originalPrice = 0 } - discountPrice, err := s.cleanPrice(discountPriceStr) + discountPrice, err := s.CleanPrice(discountPriceStr) if err != nil { discountPrice = 0 } @@ -55,12 +55,18 @@ func (s *ScraperImpl) ScrapeData(baseURL string, maxPage int, category string) ( for i := 1; i <= maxPage; i++ { logrus.Infof("Scraping page %d", i) - collector.Visit(fmt.Sprintf("https://%s/%s/page/%d/", baseURL, category, i)) + err := s.Collector.Visit(fmt.Sprintf("https://%s/%s/page/%d/", baseURL, category, i)) + if err != nil { + logrus.WithError(err).Errorf("Failed to visit page %d", i) + return nil, err + } } return products, nil } -func NewScraperImpl() Scraper { - return &ScraperImpl{} +func NewScraperImpl(collector *colly.Collector) *ScraperImpl { + return &ScraperImpl{ + Collector: collector, + } } diff --git a/api/utils/scraper_impl_test.go b/api/utils/scraper_impl_test.go new file mode 100644 index 0000000..fcc3140 --- /dev/null +++ b/api/utils/scraper_impl_test.go @@ -0,0 +1,23 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCleanPrice(t *testing.T) { + scraper := NewScraperImpl(nil) + + // Test para un precio limpio + priceStr := "$1.234" + expectedPrice := 1234 + price, err := scraper.CleanPrice(priceStr) + assert.NoError(t, err) + assert.Equal(t, expectedPrice, price) + + // Test para un string no numérico + priceStr = "invalid" + _, err = scraper.CleanPrice(priceStr) + assert.Error(t, err) +} diff --git a/go.mod b/go.mod index 898d57b..b69fb5f 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/dieg0code/serverles-api-scraper go 1.22.1 -require github.com/aws/aws-sdk-go v1.55.5 +require ( + github.com/aws/aws-sdk-go v1.55.5 + github.com/stretchr/testify v1.9.0 +) require ( github.com/PuerkitoBio/goquery v1.5.1 // indirect @@ -14,6 +17,7 @@ require ( github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -31,7 +35,9 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/temoto/robotstxt v1.1.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect @@ -52,6 +58,5 @@ require ( github.com/gocolly/colly/v2 v2.1.0 github.com/google/uuid v1.6.0 github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/joho/godotenv v1.5.1 github.com/sirupsen/logrus v1.9.3 ) diff --git a/go.sum b/go.sum index 95ecab8..27847c6 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= @@ -125,6 +123,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/main.go b/main.go index 3ae63c5..7af0a9f 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "github.com/dieg0code/serverles-api-scraper/api/router" "github.com/dieg0code/serverles-api-scraper/api/service" "github.com/dieg0code/serverles-api-scraper/api/utils" + "github.com/gocolly/colly/v2" "github.com/sirupsen/logrus" ) @@ -19,11 +20,26 @@ var r *router.Router func init() { logrus.Info("Initializing serverless API scraper") - db := db.NewDynamoDB("sa-east-1") - productRepo := repository.NewProductRepositoryImpl(db, "products") - scraper := utils.NewScraperImpl() + region := "sa-east-1" + tableName := "products" + + // Instance DynamoDB + db := db.NewDynamoDB(region) + + // Instance repository + productRepo := repository.NewProductRepositoryImpl(db, tableName) + + // Instance colly and scraper + collector := colly.NewCollector() + scraper := utils.NewScraperImpl(collector) + + // Instance service productService := service.NewProductServiceImpl(productRepo, scraper) + + // Instance controller productController := controller.NewProductControllerImpl(productService) + + // Instance router r = router.NewRouter(productController) r.InitRoutes() diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index f832a57..8329f86 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/hashicorp/aws" { constraints = "5.61.0" hashes = [ "h1:VE5N7OZPW6/SRMTWX5JZ9XDMcwvs9GhUtSzhVG7DLIg=", + "h1:ur8qjVbMUI0u3wZjaxyNfnHE7xEMzxALGPKVMwsCuGk=", "zh:1a0a150b6adaeacc8f56763182e76c6219ac67de1217b269d24b770067b7bab0", "zh:1d9c3a8ac3934a147569254d6e2e6ea5293974d0595c02c9e1aa31499a8f0042", "zh:1f4d1d5e2e02fd5cccafa28dade8735a3059ed1ca3284fb40116cdb67d0e7ee4", diff --git a/terraform/api_gateway.tf b/terraform/api_gateway.tf index 85d7f33..a048f5e 100644 --- a/terraform/api_gateway.tf +++ b/terraform/api_gateway.tf @@ -3,35 +3,21 @@ resource "aws_api_gateway_rest_api" "api" { description = "API Scraper" } -# Resource for API Gateway /products endpoint +# Resource for API Gateway /api/v1/products endpoint resource "aws_api_gateway_resource" "products" { rest_api_id = aws_api_gateway_rest_api.api.id parent_id = aws_api_gateway_rest_api.api.root_resource_id - path_part = "products" + path_part = "api/v1/products" } -# Resource for API Gateway /products/{id} endpoint +# Resource for API Gateway /api/v1/products/{productId} endpoint resource "aws_api_gateway_resource" "product" { rest_api_id = aws_api_gateway_rest_api.api.id parent_id = aws_api_gateway_resource.products.id - path_part = "{id}" + path_part = "{productId}" } -# Resource for API Gateway /Users endpoint -resource "aws_api_gateway_resource" "users" { - rest_api_id = aws_api_gateway_rest_api.api.id - parent_id = aws_api_gateway_rest_api.api.root_resource_id - path_part = "Users" -} - -# Resource for API Gateway /Scraper endpoint -resource "aws_api_gateway_resource" "scraper" { - rest_api_id = aws_api_gateway_rest_api.api.id - parent_id = aws_api_gateway_rest_api.api.root_resource_id - path_part = "Scraper" -} - -# Method for GET /products endpoint +# Method for GET /api/v1/products endpoint resource "aws_api_gateway_method" "get_products" { rest_api_id = aws_api_gateway_rest_api.api.id resource_id = aws_api_gateway_resource.products.id @@ -39,7 +25,7 @@ resource "aws_api_gateway_method" "get_products" { authorization = "NONE" } -# Method for GET /products/{id} endpoint +# Method for GET /api/v1/products/{productId} endpoint resource "aws_api_gateway_method" "get_product" { rest_api_id = aws_api_gateway_rest_api.api.id resource_id = aws_api_gateway_resource.product.id @@ -47,24 +33,15 @@ resource "aws_api_gateway_method" "get_product" { authorization = "NONE" } -# Method for POST /Users endpoint -resource "aws_api_gateway_method" "post_users" { +# Method for POST /api/v1/products endpoint +resource "aws_api_gateway_method" "post_products" { rest_api_id = aws_api_gateway_rest_api.api.id - resource_id = aws_api_gateway_resource.users.id + resource_id = aws_api_gateway_resource.products.id http_method = "POST" authorization = "NONE" } -# Method for POST /Scraper endpoint (update data) -resource "aws_api_gateway_method" "post_scraper" { - rest_api_id = aws_api_gateway_rest_api.api.id - resource_id = aws_api_gateway_resource.scraper.id - http_method = "POST" - authorization = "NONE" -} - - -# Integration for GET /products endpoint +# Integration for GET /api/v1/products endpoint resource "aws_api_gateway_integration" "products_lambda_integration" { rest_api_id = aws_api_gateway_rest_api.api.id resource_id = aws_api_gateway_resource.products.id @@ -75,7 +52,7 @@ resource "aws_api_gateway_integration" "products_lambda_integration" { uri = aws_lambda_function.api_scraper.invoke_arn } -# Integration for GET /products/{id} endpoint +# Integration for GET /api/v1/products/{productId} endpoint resource "aws_api_gateway_integration" "product_lambda_integration" { rest_api_id = aws_api_gateway_rest_api.api.id resource_id = aws_api_gateway_resource.product.id @@ -86,25 +63,25 @@ resource "aws_api_gateway_integration" "product_lambda_integration" { uri = aws_lambda_function.api_scraper.invoke_arn } -# Integration for POST /Users endpoint -resource "aws_api_gateway_integration" "users_lambda_integration" { +# Integration for POST /api/v1/products endpoint +resource "aws_api_gateway_integration" "post_products_lambda_integration" { rest_api_id = aws_api_gateway_rest_api.api.id - resource_id = aws_api_gateway_resource.users.id - http_method = aws_api_gateway_method.post_users.http_method + resource_id = aws_api_gateway_resource.products.id + http_method = aws_api_gateway_method.post_products.http_method integration_http_method = "POST" type = "AWS_PROXY" uri = aws_lambda_function.api_scraper.invoke_arn } -# Integration for POST /Scraper endpoint -resource "aws_api_gateway_integration" "scraper_lambda_integration" { - rest_api_id = aws_api_gateway_rest_api.api.id - resource_id = aws_api_gateway_resource.scraper.id - http_method = aws_api_gateway_method.post_scraper.http_method - - integration_http_method = "POST" - type = "AWS_PROXY" - uri = aws_lambda_function.api_scraper.invoke_arn +resource "aws_lambda_permission" "api_gateway" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.api_scraper.function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_api_gateway_rest_api.api.execution_arn}/*/*" } +output "api_gateway_invoke_url" { + value = aws_api_gateway_rest_api.api.invoke_url +} \ No newline at end of file diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf index 03756d7..e340f3d 100644 --- a/terraform/dynamodb.tf +++ b/terraform/dynamodb.tf @@ -1,34 +1,14 @@ # DynamoDB table for products resource "aws_dynamodb_table" "products_table" { - name = "Products" + name = "Products" billing_mode = "PAY_PER_REQUEST" - hash_key = "ProductID" + hash_key = "ProductID" attribute { name = "ProductID" type = "S" } - - attribute { - name = "Name" - type = "S" - } - - attribute { - name = "Category" - type = "S" - } - - attribute { - name = "OriginalPrice" - type = "N" - } - - attribute { - name = "DiscountedPrice" - type = "N" - } } diff --git a/terraform/iam.tf b/terraform/iam.tf index 21cc49f..5f5d037 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -1,6 +1,5 @@ -# IAM role for lambda function resource "aws_iam_role" "lambda_role" { - name = "api_scraper_role" + name = "lambda_role" assume_role_policy = jsonencode({ Version = "2012-10-17" @@ -16,34 +15,29 @@ resource "aws_iam_role" "lambda_role" { }) } -# Policy to allow lambda function to write to DynamoDB and CloudWatch logs -resource "aws_iam_role_policy_attachment" "lambda_policy" { - role = aws_iam_role.lambda_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" -} - - -resource "aws_iam_role_policy" "dynamodb_policy" { - role = aws_iam_role.lambda_role.id - - policy = jsonencode({ +resource "aws_iam_policy" "lambda_policy" { + name = "lambda_policy" + description = "IAM policy for Lambda to access DynamoDB" + policy = jsonencode({ Version = "2012-10-17" Statement = [ { - Effect = "Allow" Action = [ - "dynamodb:GetItem", "dynamodb:PutItem", + "dynamodb:GetItem", "dynamodb:UpdateItem", "dynamodb:DeleteItem", "dynamodb:Scan", "dynamodb:Query" ] - Resource = [ - aws_dynamodb_table.users_table.arn, - aws_dynamodb_table.products_table.arn - ] + Effect = "Allow" + Resource = "arn:aws:dynamodb:${var.aws_region}:${var.aws_account_id}:table/${aws_dynamodb_table.products_table.name}" } ] }) } + +resource "aws_iam_role_policy_attachment" "lambda_policy_attachment" { + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.lambda_policy.arn +} \ No newline at end of file diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bd0ab36..63667db 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,9 +1,17 @@ resource "aws_lambda_function" "api_scraper" { - filename = "api_scraper.zip" + filename = "api_scraper.zip" function_name = "api_scraper" - role = aws_iam_role.lambda_role.arn - handler = "api_scraper_lambda" - runtime = "go1.x" + role = aws_iam_role.lambda_role.arn + handler = "api_scraper_lambda" + runtime = "go1.x" + memory_size = 128 + timeout = 150 - source_code_hash = filebase64sha256("api_scraper.zip") -} \ No newline at end of file + source_code_hash = filebase64sha256("../api_scraper_lambda.zip") + + environment { + variables = { + TABLE_NAME = aws_dynamodb_table.products_table.name + } + } +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..e29bf67 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,10 @@ +variable "aws_region" { + description = "The AWS region to deploy resources" + type = string + default = "sa-east-1" +} + +variable "aws_account_id" { + description = "The AWS account ID" + type = string +} \ No newline at end of file