Merge pull request #61 from WolframResearch/was/removing-minio-aws

[WAS-636] - Removed minio from AWS

EnvironmentSetup/AWS/README.md

@@ -53,14 +53,32 @@ This will interactively prompt for your AWS IAM user access key, secret key and
 
 	cd WAS-Kubernetes/EnvironmentSetup/AWS/
 
-**Step 3.** Run the following command to set up EKS and deploy WAS:
+**Step 3.** Create two S3 buckets to use for WAS, these are needed for resource-manager, resourceinfo-bucket and nodefileinfo-bucket:
+
+	If the buckets will be in 'us-east-1'
+
+	aws s3api create-bucket --bucket <RESOURCEINFOBUCKETNAME>
+	aws s3api create-bucket --bucket <NODEFILEINFOBUCKETNAME>
+
+	If will be in any other regions(us-east-2, us-west-1 etc.)
+
+	aws s3api create-bucket --bucket <RESOURCEINFOBUCKETNAME> --region <REGION> --create-bucket-configuration LocationConstraint=<REGION>
+	aws s3api create-bucket --bucket <NODEFILEINFOBUCKETNAME> --region <REGION> --create-bucket-configuration LocationConstraint=<REGION>
+
+**Step 4.** Update buckets file with these buckets
+
+	resourceinfo-bucket:<RESOURCEINFOBUCKETNAME>
+	nodefiles-bucket:<NODEFILEINFOBUCKETNAME>
+
+
+**Step 5.** Run the following command to set up EKS and deploy WAS:
 
 	mkdir -p ~/.kube && docker-compose up --build -d && clear && docker exec -it aws-setup-manager bash setup --create && sudo chown -R $USER ~/.kube
 
 **Note:** This can take approximately 45 minutes to complete.
 
 
-**Step 4.** Run the following command to retrieve your base URL and application URLs:
+**Step 6.** Run the following command to retrieve your base URL and application URLs:
 
 	docker-compose up --build -d && clear && docker exec -it aws-setup-manager bash setup --endpoint-info
 
@@ -81,15 +99,15 @@ The output of this command will follow this pattern:
 
 
 
-**Step 5.** After completion, run this command to shutdown the aws-setup-manager:
+**Step 7.** After completion, run this command to shutdown the aws-setup-manager:
 
 	docker-compose down
 
 
-**Step 6.** Get a license file from your Wolfram Research sales representative.
+**Step 8.** Get a license file from your Wolfram Research sales representative.
 
 
-**Step 7.** This file needs to be deployed to WAS as a node file in the conventional location `.Wolfram/Licensing/mathpass`. From a Wolfram Language client, this may be achieved using the following code: 
+**Step 9.** This file needs to be deployed to WAS as a node file in the conventional location `.Wolfram/Licensing/mathpass`. From a Wolfram Language client, this may be achieved using the following code: 
 
     was = ServiceConnect["WolframApplicationServer", "http://<your-base-url>"];
     ServiceExecute[was, "DeployNodeFile",
@@ -103,7 +121,7 @@ Alternatively you may use the [node files REST API](../../Documentation/API/Node
     PacletInstall["WolframApplicationServer"];
     Needs["WolframApplicationServer`"]
 
-**Step 8.** Restart the application using the [restart API](../../Documentation/API/Utilities.md) to enable your Wolfram Engines.
+**Step 10.** Restart the application using the [restart API](../../Documentation/API/Utilities.md) to enable your Wolfram Engines.
 
 URL: `http://<your-base-url>/.applicationserver/kernel/restart`
 

EnvironmentSetup/AWS/Source/buckets

@@ -0,0 +1,2 @@
+resourceinfo-bucket:
+nodefiles-bucket:

EnvironmentSetup/AWS/Source/deployments/active-web-elements-server-deployment.yaml

@@ -33,6 +33,14 @@ spec:
           value: kafka-persistent-kafka-bootstrap.kafka.svc.cluster.local:9092
         - name : API_REQUEST_TOPIC
           value: "api-request"
+        - name : RESOURCE_INFO_TOPIC
+          value: "resource-info"
+        - name : NODEFILE_INFO_TOPIC
+          value: "nodefile-info"
+        - name : ENDPOINT_INFO_TOPIC
+          value: "endpoint-info"
+        - name: API_RESPONSE_TOPIC
+          value: "api-response"
         - name: applicationserver.nodefiles.cachedirectory
           value: /opt/.wolframcache/nodefiles/
         - name: applicationserver.version

EnvironmentSetup/AWS/Source/deployments/endpoint-manager-deployment.yaml

@@ -67,6 +67,10 @@ spec:
           value: /opt/app/logs
         - name: KAFKA.BOOTSTRAP-SERVERS
           value: kafka-persistent-kafka-bootstrap.kafka.svc.cluster.local:9092
+        - name : ENDPOINT_INFO_TOPIC
+          value: "endpoint-info"
+        - name : RESOURCE_INFO_TOPIC
+          value: "resource-info"
         image: wolframapplicationserver/endpoint-manager:1.2.5
         name: endpoint-manager
         ports:
@@ -92,9 +96,6 @@ spec:
         - mountPath: "/opt/app/logs"
           name: endpoint-logs-storage
       initContainers:
-      - name: init-minio
-        image: bash
-        command: ["bash", "-c", "for i in $(seq 1 3000); do nc -zvw1 minio 9000 && exit 0 || sleep 3; done; exit 1"]
       - name: init-kafka
         image: bash
         command: ["bash", "-c", "for i in $(seq 1 3000); do nc -zvw1 kafka-persistent-kafka-bootstrap.kafka.svc.cluster.local 9092 && exit 0 || sleep 3; done; exit 1"]

EnvironmentSetup/AWS/Source/deployments/resource-manager-deployment.yaml

@@ -29,12 +29,24 @@ spec:
           value: docker
         - name: LOG_LOCATION
           value: /opt/app/logs
+        - name: RESOURCE_INFO_TOPIC
+          value: "resource-info"
+        - name: NODEFILE_INFO_TOPIC
+          value: "nodefile-info"
         - name: KAFKA.BOOTSTRAP-SERVERS
           value: kafka-persistent-kafka-bootstrap.kafka.svc.cluster.local:9092
-        - name: MINIOACCESSKEY
-          value: <your-access-key>
-        - name: MINIOSECRETKEY
-          value: <your-secret-key>
+        - name: STORAGE_PROVIDER
+          value: "s3"
+        - name: SPRING_CLOUD_AWS_S3_ENABLED
+          value: "true"
+        - name: SPRING_CLOUD_AWS_S3_ENDPOINT
+          value: <endpoint>
+        - name: SPRING_CLOUD_AWS_S3_REGION
+          value: <region>
+        - name: SPRING_CLOUD_AWS_CREDENTIALS_ACCESS_KEY
+          value: <access-key>
+        - name: SPRING_CLOUD_AWS_CREDENTIALS_SECRET_KEY
+          value: <secret-key>
         - name: RESOURCEINFO.BUCKET
           value: <your-resourceinfo-bucketname>
         - name: NODEFILES.BUCKET
@@ -68,9 +80,6 @@ spec:
         - mountPath: "/opt/app/logs"
           name: resources-logs-storage
       initContainers:
-      - name: init-minio
-        image: bash
-        command: ["bash", "-c", "for i in $(seq 1 3000); do nc -zvw1 minio 9000 && exit 0 || sleep 3; done; exit 1"]
       - name: init-kafka
         image: bash
         command: ["bash", "-c", "for i in $(seq 1 3000); do nc -zvw1 kafka-persistent-kafka-bootstrap.kafka.svc.cluster.local 9092 && exit 0 || sleep 3; done; exit 1"]

EnvironmentSetup/AWS/Source/setup

@@ -423,14 +423,14 @@ saveCredsFunction() {
   echo "GET_CLUSTER_NAME:$GET_CLUSTER_NAME" >>creds
   echo "GET_REGION:$GET_REGION" >>creds
   echo "TERRAFORM_BUCKET:$terraform_bucket_name" >>creds
-  echo "MINIO_ACCESS_KEY:$MINIO_ACCESS_KEY" >>creds
-  echo "MINIO_SECRET_KEY:$MINIO_SECRET_KEY" >>creds
+  echo "ACCESS_KEY:$ACCESS_KEY" >>creds
+  echo "SECRET_KEY:$SECRET_KEY" >>creds
   echo "DYNAMODB_TABLE:$dynamodb_table" >>creds
   echo "WORKSPACE:$workspace" >>creds
   echo "S3_KEY:$s3_key" >>creds
   echo "EFS_FS_ID:$filesystem_id" >>creds
-  echo "RESOURCE_INFO_BUCKET:$resources_bucket_name" >>creds
-  echo "NODE_FILES_BUCKET:$nodefiles_bucket_name" >>creds
+  echo "RESOURCE_INFO_BUCKET:$RESOURCEINFO_BUCKET" >>creds
+  echo "NODEFILES_BUCKET:$NODEFILES_BUCKET" >>creds
   echo "BASE_URL:http://$BASE_URL/" >>creds
   echo "RESOURCE_MANAGER_URL:http://$BASE_URL/resources/" >>creds
   echo "NODEFILES_MANAGER_URL:http://$BASE_URL/nodefiles/" >>creds
@@ -454,8 +454,8 @@ saveCredsFunction() {
 awsKeysFunction() {
   CFILE=~/.aws/credentials
   if [ -f "$CFILE" ]; then
-    MINIO_ACCESS_KEY=$(cat ~/.aws/credentials | grep aws_access_key_id | awk '{print $3}')
-    MINIO_SECRET_KEY=$(cat ~/.aws/credentials | grep aws_secret_access_key | awk '{print $3}')
+    ACCESS_KEY=$(cat ~/.aws/credentials | grep aws_access_key_id | awk '{print $3}')
+    SECRET_KEY=$(cat ~/.aws/credentials | grep aws_secret_access_key | awk '{print $3}')
   else
     echo
     log_error "Cannot find aws access key and secret key. Did you run 'aws configure' ?"
@@ -475,6 +475,17 @@ SetupFunction() {
   GET_REGION=$(echo $GET_REGION | tr -d '\r')
 
   cd ..
+
+  RESOURCEINFO_BUCKET=$(cat buckets | grep resourceinfo-bucket | cut -d':' -f2 | tr -d '\n' | tr -d '\r')
+  NODEFILES_BUCKET=$(cat buckets | grep nodefiles-bucket | cut -d':' -f2 | tr -d '\n' | tr -d '\r')
+
+  if [ -z "$RESOURCEINFO_BUCKET" ] || [ -z "$NODEFILES_BUCKET" ]; then
+    echo
+    log_error "Check the Source/buckets file and make sure that it's already filled."
+    echo
+    exit 1
+  fi
+
   echo
   log_debug "Phase 1 of 5: Prequisites Checks"
   printf "${YELLOW}☑${CYAN}□□□□${NORMAL} Phase ${YELLOW}1${NORMAL} of ${GREEN}5${NORMAL}: Prequisites Checks\\n"
@@ -804,24 +815,26 @@ EOF
     fi
   done
 
-  # Minio deployment setup
-  CURRENT_MINIO_ACCESS_KEY=$(grep -A1 -i MINIO_ACCESS_KEY deployments/minio-deployment.yaml | grep -i value: | awk '{print $2}')
-  sed -i -e 's~'"$CURRENT_MINIO_ACCESS_KEY"'~'"$MINIO_ACCESS_KEY"'~g' deployments/minio-deployment.yaml
-  CURRENT_MINIO_SECRET_KEY=$(grep -A1 -i MINIO_SECRET_KEY deployments/minio-deployment.yaml | grep -i value: | awk '{print $2}')
-  sed -i -e 's~'"$CURRENT_MINIO_SECRET_KEY"'~'"$MINIO_SECRET_KEY"'~g' deployments/minio-deployment.yaml
-
   # Resource Manager deployment setup
-  CURRENT_MINIOACCESSKEY=$(grep -A1 -i MINIOACCESSKEY deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
-  sed -i -e 's~'"$CURRENT_MINIOACCESSKEY"'~'"$MINIO_ACCESS_KEY"'~g' deployments/resource-manager-deployment.yaml
-  CURRENT_MINIOSECRETKEY=$(grep -A1 -i MINIOSECRETKEY deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
-  sed -i -e 's~'"$CURRENT_MINIOSECRETKEY"'~'"$MINIO_SECRET_KEY"'~g' deployments/resource-manager-deployment.yaml
-  resources_bucket_name="$lower_cluster_name-resources-$account_id"
-  nodefiles_bucket_name="$lower_cluster_name-nodefiles-$account_id"
+  CURRENT_ACCESS_KEY=$(grep -A1 -i SPRING_CLOUD_AWS_CREDENTIALS_ACCESS_KEY deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
+  sed -i -e 's~'"$CURRENT_ACCESS_KEY"'~'"$ACCESS_KEY"'~g' deployments/resource-manager-deployment.yaml
+
+  CURRENT_SECRETKEY=$(grep -A1 -i SPRING_CLOUD_AWS_CREDENTIALS_SECRET_KEY deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
+  sed -i -e 's~'"$CURRENT_SECRETKEY"'~'"$SECRET_KEY"'~g' deployments/resource-manager-deployment.yaml
+
+  s3_endpoint="https://s3.$GET_REGION.amazonaws.com"
+
+  CURRENT_S3_REGION=$(grep -A1 -i SPRING_CLOUD_AWS_S3_REGION deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
+  sed -i -e 's~'"$CURRENT_S3_REGION"'~'"$GET_REGION"'~g' deployments/resource-manager-deployment.yaml
+
+  CURRENT_S3_ENDPOINT=$(grep -A1 -i SPRING_CLOUD_AWS_S3_ENDPOINT deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
+  sed -i -e 's~'"$CURRENT_S3_ENDPOINT"'~'"$s3_endpoint"'~g' deployments/resource-manager-deployment.yaml
+
   CURRENT_RESOURCEINFOBUCKET=$(grep -A1 -i RESOURCEINFO.BUCKET deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
-  sed -i -e 's~'"$CURRENT_RESOURCEINFOBUCKET"'~'"$resources_bucket_name"'~g' deployments/resource-manager-deployment.yaml
+  sed -i -e 's~'"$CURRENT_RESOURCEINFOBUCKET"'~'"$RESOURCEINFO_BUCKET"'~g' deployments/resource-manager-deployment.yaml
   CURRENT_NODEFILESBUCKET=$(grep -A1 -i NODEFILES.BUCKET deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}' | cut -d ' ' -f 1)
   CURRENT_NODEFILESBUCKET=$(echo $CURRENT_NODEFILESBUCKET | awk '{print $1}')
-  sed -i -e 's~'"$CURRENT_NODEFILESBUCKET"'~'"$nodefiles_bucket_name"'~g' deployments/resource-manager-deployment.yaml
+  sed -i -e 's~'"$CURRENT_NODEFILESBUCKET"'~'"$NODEFILES_BUCKET"'~g' deployments/resource-manager-deployment.yaml
   CURRENT_RESOURCE_BUCKET_REGION=$(grep -A1 -i RESOURCE.BUCKET.REGION deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')
   sed -i -e 's~'"$CURRENT_RESOURCE_BUCKET_REGION"'~'"$GET_REGION"'~g' deployments/resource-manager-deployment.yaml
   CURRENT_NODEFILES_BUCKET_REGION=$(grep -A1 -i NODEFILES.BUCKET.REGION deployments/resource-manager-deployment.yaml | grep -i value: | awk '{print $2}')