Merge pull request #13 from mentorpal/replace_w2v

replacing w2v

.github/workflows/cicd.yml

@@ -13,7 +13,7 @@ jobs:
       - name: python env setup
         uses: actions/setup-python@v2
         with:
-          python-version: 3.10.9
+          python-version: 3.10.7
       - uses: actions/setup-node@v1
         with:
           node-version: "16.15"

.gitignore

@@ -19,3 +19,7 @@ build
 dist
 htmlcov
 shared/installed/*
+shared/sentence-transformer
+transformer.pkl
+*_saved_keyed_vectors
+*.vectors.npy

.python-version

@@ -1 +1 @@
-3.10.9
+3.10.7

Makefile

@@ -12,7 +12,7 @@ install: poetry-ensure-installed
 	poetry install
 
 .PHONY docker-build:
-docker-build: sentence-transformers transformer.pkl
+docker-build: sentence-transformers transformer.pkl word2vec.bin
 # squash reduces final image size by merging layers `--squash`
 # but its not supported in github actions
 	docker build -t $(DOCKER_IMAGE) .
@@ -27,6 +27,10 @@ sentence-transformers: shared/installed
 transformer.pkl: $(VENV) shared/installed sentence-transformers
 	poetry run python ./server/transformer/embeddings.py ./shared/installed
 
+word2vec.bin: shared/installed
+	cd shared && poetry run python word2vec_download.py
+	cd shared && poetry run python word2vec_slim_download.py
+
 build/deploy:
 	# put everything we want in our beanstalk deploy.zip file
 	# into a build/deploy folder.
@@ -75,7 +79,7 @@ poetry-ensure-installed:
 	./tools/poetry_ensure_installed.sh
 
 .PHONY: test
-test: $(VENV) install poetry-ensure-installed sentence-transformers transformer.pkl
+test: $(VENV) install poetry-ensure-installed sentence-transformers transformer.pkl word2vec.bin
 	cd ./shared/ && $(MAKE) installed/sentence-transformer
 	SHARED_ROOT=./shared/installed poetry run python -m pytest -vv
 

entrypoint.sh

@@ -7,6 +7,6 @@
 ##
 export FLASK_APP=/app/server
 
-cd /app && gunicorn --config python:server.gunicorn_conf server.manage:app
+cd /app && gunicorn --config python:server.gunicorn_conf server.manage:app --timeout 300 --preload --log-level debug
 
 exit 0

infrastructure/aws/terraform/mentorpal/dev/env.hcl

@@ -0,0 +1,6 @@
+# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to
+# feed forward to the child modules.
+locals {
+  environment = "dev"
+  namespace   = "mentorpal"
+}

infrastructure/aws/terraform/mentorpal/dev/us-east-1/region.hcl

@@ -0,0 +1,5 @@
+# Set common variables for the region. This is automatically pulled in in the root terragrunt.hcl configuration to
+# configure the remote state bucket and pass forward to the child modules as inputs.
+locals {
+  aws_region = "us-east-1"
+}

infrastructure/aws/terraform/mentorpal/dev/us-east-1/sbert-service/secret.hcl.template

@@ -0,0 +1,6 @@
+locals {
+  cloudwatch_slack_webhook = "https://hooks.slack.com/services/<redacted>"
+  sentry_dsn_sbert         = "https://<redacted>@oingest.sentry.io/<redacted>"
+  api_secret_key           = "<api secret key redacted>"
+  jwt_secret_key           = "<secret jwt key redacted>"
+}

infrastructure/aws/terraform/mentorpal/dev/us-east-1/sbert-service/terragrunt.hcl

@@ -0,0 +1,84 @@
+locals {
+  # Automatically load environment-level variables
+  environment_vars = read_terragrunt_config(find_in_parent_folders("env.hcl"))
+  region_vars      = read_terragrunt_config(find_in_parent_folders("region.hcl"))
+  account_vars     = read_terragrunt_config(find_in_parent_folders("account.hcl"))
+  secret_vars      = read_terragrunt_config("./secret.hcl")
+  # Extract out common variables for reuse
+  env                      = local.environment_vars.locals.environment
+  aws_account_id           = local.account_vars.locals.aws_account_id
+  aws_region               = local.region_vars.locals.aws_region
+  cloudwatch_slack_webhook = local.secret_vars.locals.cloudwatch_slack_webhook
+  sentry_dsn_sbert         = local.secret_vars.locals.sentry_dsn_sbert
+  api_secret_key           = local.secret_vars.locals.api_secret_key
+  jwt_secret_key           = local.secret_vars.locals.jwt_secret_key
+  secret_header_name       = local.secret_vars.locals.secret_header_name
+  secret_header_value      = local.secret_vars.locals.secret_header_value
+  allowed_origin           = local.secret_vars.locals.allowed_origin
+}
+
+terraform {
+  source = "${path_relative_from_include()}//modules/beanstalk"
+}
+
+# Include all settings from the root terragrunt.hcl file
+# include "root" { # VS code doesnt highlight this correctly
+include {
+  path = find_in_parent_folders()
+}
+
+inputs = {
+  aws_region                 = local.aws_region
+  aws_availability_zones     = ["us-east-1a", "us-east-1b"]
+  vpc_cidr_block             = "10.10.0.0/16"
+  aws_acm_certificate_domain = "mentorpal.org"
+  aws_route53_zone_name      = "mentorpal.org"
+  site_domain_name           = "sbert-dev.mentorpal.org"
+  eb_env_namespace           = "mentorpal"
+  eb_env_stage               = local.env
+  eb_env_name                = "sbert"
+  eb_env_instance_type       = "c6i.xlarge" # compute-optimized, 120$/month, a1.xlarge, c6i.xlarge
+  enable_alarms              = true
+  slack_channel              = "ls-alerts-qa"
+  slack_username             = "uscictlsalerts"
+  cloudwatch_slack_webhook   = local.cloudwatch_slack_webhook
+    secret_header_name       = local.secret_header_name
+  secret_header_value        = local.secret_header_value
+  allowed_origin             = local.allowed_origin
+
+  # scaling:
+  eb_env_autoscale_min = 1 # 
+  eb_env_autoscale_max = 1 # 
+  # seems like there's no way to set desired capacity and it seems to be max by default?
+  eb_env_autoscale_measure_name    = "CPUUtilization"
+  eb_env_autoscale_statistic       = "Average"
+  eb_env_autoscale_unit            = "Percent"
+  eb_env_autoscale_lower_bound     = "20"
+  eb_env_autoscale_lower_increment = "1"
+  eb_env_autoscale_upper_bound     = "80"
+  eb_env_autoscale_upper_increment = "1"
+
+  eb_env_env_vars = {
+    STAGE                           = local.env,
+    IS_SENTRY_ENABLED               = "true",
+    SENTRY_DSN_MENTOR_SBERT_SERVICE = local.sentry_dsn_sbert,
+    LOG_LEVEL_SBERT_SERVICE         = "DEBUG",
+    API_SECRET_KEY                  = local.api_secret_key,
+    JWT_SECRET_KEY                  = local.jwt_secret_key,
+
+  }
+
+  # logging:
+  eb_env_enable_stream_logs                   = true
+  eb_env_logs_delete_on_terminate             = false
+  eb_env_logs_retention_in_days               = 30
+  eb_env_health_streaming_enabled             = true
+  eb_env_health_streaming_delete_on_terminate = false
+  eb_env_health_streaming_retention_in_days   = 7
+  eb_env_tags = {
+    Terraform   = "true"
+    Environment = local.env
+    Project     = "mentorpal"
+    Service     = "sbert"
+  }
+}

infrastructure/aws/terraform/modules/beanstalk/main.tf

@@ -60,6 +60,8 @@ module "elastic_beanstalk_application" {
 
 
 ###
+# IMPORTANT NOTE: After deployment, you must manually change the IP address type of the load balancer to dualstack.
+#                 This is a new feature that is not yet supported in terraform.
 # the main elastic beanstalk env
 ###
 module "elastic_beanstalk_environment" {