[Draft for Initial Review] Automating Model Autotracing and Uploading Process

.ci/run-repository.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Called by entry point `run-test` use this script to add your repository specific test commands
+# Called by entry point `run-test` use this script to add your repository specific task commands
 # Once called opensearch is up and running and the following parameters are available to this script
 
 # OPENSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease)
@@ -16,7 +16,7 @@ set -e
 echo -e "\033[34;1mINFO:\033[0m URL ${opensearch_url}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m EXTERNAL OS URL ${external_opensearch_url}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m VERSION ${OPENSEARCH_VERSION}\033[0m"
-echo -e "\033[34;1mINFO:\033[0m IS_DOC: ${IS_DOC}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m TASK_TYPE: ${TASK_TYPE}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS ${PYTHON_CONNECTION_CLASS}\033[0m"
@@ -33,7 +33,8 @@ docker build \
 echo -e "\033[1m>>>>> Run [opensearch-project/opensearch-py-ml container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
 
-if [[ "$IS_DOC" == "false" ]]; then
+if [[ "$TASK_TYPE" == "test" ]]; then
+  # Set up OpenSearch cluster & Run test (Invoked by integration.yml workflow)
   docker run \
   --network=${network_name} \
   --env "STACK_VERSION=${STACK_VERSION}" \
@@ -45,10 +46,11 @@ if [[ "$IS_DOC" == "false" ]]; then
   --name opensearch-py-ml-test-runner \
   opensearch-project/opensearch-py-ml \
   nox -s "test-${PYTHON_VERSION}(pandas_version='${PANDAS_VERSION}')"
+
   docker cp opensearch-py-ml-test-runner:/code/opensearch-py-ml/junit/ ./junit/
-
   docker rm opensearch-py-ml-test-runner
-else
+elif [[ "$TASK_TYPE" == "doc" ]]; then
+  # Set up OpenSearch cluster & Run docs (Invoked by build_deploy_doc.yml workflow)
   docker run \
   --network=${network_name} \
   --env "STACK_VERSION=${STACK_VERSION}" \
@@ -60,7 +62,29 @@ else
   --name opensearch-py-ml-doc-runner \
   opensearch-project/opensearch-py-ml \
   nox -s docs
+
   docker cp opensearch-py-ml-doc-runner:/code/opensearch-py-ml/docs/build/ ./docs/
-
   docker rm opensearch-py-ml-doc-runner
-fi
+elif [[ "$TASK_TYPE" == "trace" ]]; then
+  # Set up OpenSearch cluster & Run model autotracing (Invoked by model_uploader.yml workflow)
+  echo -e "\033[34;1mINFO:\033[0m MODEL_ID: ${MODEL_ID}\033[0m"
+  echo -e "\033[34;1mINFO:\033[0m MODEL_VERSION: ${MODEL_VERSION}\033[0m"
+  echo -e "\033[34;1mINFO:\033[0m TRACING_FORMAT: ${TRACING_FORMAT}\033[0m"
+  echo -e "\033[34;1mINFO:\033[0m EMBEDDING_DIMENSION: ${EMBEDDING_DIMENSION:-N/A}\033[0m"
+  echo -e "\033[34;1mINFO:\033[0m POOLING_MODE: ${POOLING_MODE:-N/A}\033[0m"
+
+  docker run \
+  --network=${network_name} \
+  --env "STACK_VERSION=${STACK_VERSION}" \
+  --env "OPENSEARCH_URL=${opensearch_url}" \
+  --env "OPENSEARCH_VERSION=${OPENSEARCH_VERSION}" \
+  --env "TEST_SUITE=${TEST_SUITE}" \
+  --env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \
+  --env "TEST_TYPE=server" \
+  --name opensearch-py-ml-trace-runner \
+  opensearch-project/opensearch-py-ml \
+  nox -s trace -- ${MODEL_ID} ${MODEL_VERSION} ${TRACING_FORMAT} -ed ${EMBEDDING_DIMENSION} -pm ${POOLING_MODE}
+
+  docker cp opensearch-py-ml-trace-runner:/code/opensearch-py-ml/upload/ ./upload/
+  docker rm opensearch-py-ml-trace-runner
+fi

.ci/run-tests

@@ -10,7 +10,7 @@ export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection
 export CLUSTER="${1:-opensearch}"
 export SECURE_INTEGRATION="${2:-true}"
 export OPENSEARCH_VERSION="${3:-latest}"
-export IS_DOC="${4:-false}"
+export TASK_TYPE="${4:-test}"
 if [[ "$SECURE_INTEGRATION" == "true" ]]; then
     export OPENSEARCH_URL_EXTENSION="https"
 else

.github/CODEOWNERS

@@ -1 +1 @@
-*   @dhrubo-os  @greaa-aws @ylwu-amzn @b4sjoo @jngz-es @rbhavna
+*   @thanawan-atc

.github/workflows/build_deploy_doc.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v2
       - name: Integ ${{ matrix.cluster }} secured=${{ matrix.secured }} version=${{matrix.entry.opensearch_version}}
-        run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }} true"
+        run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }} doc"
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
         with:

.github/workflows/delete_model_uploader_branch.yml

@@ -0,0 +1,15 @@
+name: Delete merged branch of the model upload PRs
+on: 
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  delete-branch:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.head.ref,'model-uploader/')
+    steps:
+      - name: Delete merged branch
+        uses: SvanBoxel/delete-merged-branch@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/integration.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Integ ${{ matrix.cluster }} secured=${{ matrix.secured }} version=${{matrix.entry.opensearch_version}}
-        run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }}"
+        run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }} test"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
         with: