[WIP] mongo7 upgrade

.github/workflows/test.yml

@@ -21,15 +21,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - python-version: '3.7'
+          - python-version: '3.9.19'
             mongo-version: '3.6'
-          # - python-version: '3.7'
-          #   mongo-version: '7.0.4'
+          - python-version: '3.9.19'
+            mongo-version: '7.0.4'
     services:
       mongo:
         image: mongo:${{matrix.mongo-version}}
         ports:
-            - 27017:27017
+            - 27018:27017
         options: --name mongo${{matrix.mongo-version}}
 
     steps:
@@ -43,6 +43,12 @@ jobs:
         with:
           python-version: ${{matrix.python-version}}
 
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/v2.32.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
+
       - name: Install dependencies and set up test config
         shell: bash
         env:
@@ -52,7 +58,7 @@ jobs:
         run: |
 
           # test mongo connection
-          curl http://localhost:27017
+          curl http://localhost:27018
           returncode=$?
           if [ $returncode != 0 ]; then exit $returncode; fi
 
@@ -77,6 +83,10 @@ jobs:
           sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg
           sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg
 
+          # setup env variables in docker-compose_nms.yml file
+          echo "ADMIN_USER=$ADMIN_USER"
+          echo "ADMIN_USER=$ADMIN_USER" >> $GITHUB_ENV
+
       - name: Run tests
         shell: bash
         run: make test

Makefile

@@ -103,7 +103,7 @@ setup-tests:
 	mkdir -p $(TESTDIR)/nms
 	rsync -av lib/biokbase/* $(TESTLIB)/biokbase/. --exclude *.bak-*
 	rsync -av kbapi_common/lib/biokbase/* $(TESTLIB)/biokbase/.
-	cd narrative_method_store; make; make build-classpath-list;
+#	cd narrative_method_store; make; make build-classpath-list;
 #	rsync -av narrative_method_store/lib/biokbase/* $(TESTLIB)/biokbase/.
 
 

requirements.txt

@@ -1,4 +1,4 @@
-pymongo==3.10
+pymongo==4.7.2
 docker>=3.5
 gitpython
 pyyaml

test/docker-compose_nms.yml

@@ -0,0 +1,47 @@
+version: '3.4'
+
+services:
+  nms:
+    image: ghcr.io/kbase/narrative_method_store:v0.3.12
+    platform: linux/amd64
+    ports:
+      - "7125:7125"
+    depends_on: ["mongo"]
+    environment:
+      # see deployment/conf/.templates for more environment variables
+      - method_spec_git_repo=https://github.com/kbase/narrative_method_specs_ci
+      - method_spec_git_repo_branch=master
+      - method_spec_git_repo_local_dir=narrative_method_specs_recreated_at_startup
+      - method_spec_git_repo_refresh_rate=2
+      - method_spec_cache_size=5000
+      - method_spec_temp_dir=narrative_method_store_temp
+      - method_spec_mongo_host=mongo:27017
+      - method_spec_mongo_dbname=method_store_repo_db
+      - method_spec_admin_users=${ADMIN_USER}
+      - endpoint_host=https://ci.kbase.us
+      - endpoint_base=/services
+      - method_spec_default_tag=dev
+      - auth_service_url=https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login
+      - auth_service_url_allow_insecure=false
+      - service_port=7125
+    command:
+      - "-wait"
+      - "tcp://mongo:27017"
+      - "-timeout"
+      - "120s"
+      - "-template"
+      - "/kb/deployment/conf/.templates/deployment.cfg.templ:/kb/deployment/conf/deployment.cfg"
+      - "-template"
+      - "/kb/deployment/conf/.templates/http.ini.templ:/kb/deployment/services/narrative_method_store/start.d/http.ini"
+      - "-template"
+      - "/kb/deployment/conf/.templates/server.ini.templ:/kb/deployment/services/narrative_method_store/start.d/server.ini"
+      - "-template"
+      - "/kb/deployment/conf/.templates/start_server.sh.templ:/kb/deployment/bin/start_server.sh"
+      - "-stdout"
+      - "/kb/deployment/services/narrative_method_store/logs/request.log"
+      - "/kb/deployment/bin/start_server.sh"
+
+  mongo:
+    image: "mongo:7.0.4"
+    ports:
+      - "27017:27017"

test/mock_auth/server.py

@@ -0,0 +1,141 @@
+import os
+import sys
+import json
+import jsonschema
+import traceback
+import flask
+from jsonschema.exceptions import ValidationError
+
+# Load the endpoints data, the schema, and validate the structure
+
+# For validating every config file
+with open('endpoint_schema.json') as fd:
+    endpoint_schema = json.load(fd)
+
+if not os.path.exists('/config'):
+    sys.stderr.write('Path not found: /config\n')
+    sys.exit(1)
+
+endpoints = []
+catalog_mock_auth = ['auth_admin.json', 'auth_invalid.json', 'auth_missing.json', 'auth_non_admin.json']
+for path in catalog_mock_auth:
+    if path.endswith('.json'):
+        full_path = '/config/' + path
+        with open(full_path) as fd:
+            try:
+                endpoint = json.load(fd)
+            except ValueError as err:
+                sys.stderr.write(f'JSON parsing error:\n{err}')
+                sys.exit(1)
+            try:
+                jsonschema.validate(endpoint, endpoint_schema)
+            except ValidationError as err:
+                sys.stderr.write(f'JSON Schema validation Error for {path}:\n')
+                sys.stderr.write(str(err) + '\n')
+                sys.exit(1)
+            endpoints.append(endpoint)
+
+print(f'Loaded {len(endpoints)} mock endpoints')
+
+# Start the Flask app
+app = flask.Flask(__name__)
+methods = ['GET', 'POST', 'PUT', 'DELETE']
+
+
+@app.route('/', defaults={'path': ''}, methods=methods)
+@app.route('/<path:path>', methods=methods)
+def handle_request(path):
+    """
+    Catch-all: handle any request against the endpoints.json data.
+    """
+    print('-' * 80)
+    path = '/' + path
+    req_body = flask.request.get_data().decode() or ''
+    method = flask.request.method
+    # Find the first endpoint that matches path, method, headers, and body
+    for endpoint in endpoints:
+        if endpoint['path'] == path:
+            print('Matched path:', path)
+        else:
+            continue
+        expected_methods = endpoint.get('methods', ['GET'])
+        if method in expected_methods:
+            print('Matched method')
+        else:
+            msg = f'Mismatch on method: {method} vs {expected_methods}'
+            print(msg)
+            continue
+        if match_headers(endpoint):
+            print('Matched headers')
+        else:
+            hs = dict(flask.request.headers)
+            expected_hs = endpoint.get('headers')
+            msg = f'Mismatch on headers:\n  got:      {hs}\n  expected: {expected_hs}'
+            print(msg)
+            continue
+        expected_body = endpoint.get('body', '')
+        if isinstance(expected_body, dict):
+            expected_body_json = json.dumps(expected_body)
+            try:
+                given_body_json = json.dumps(json.loads(req_body))
+            except Exception as err:
+                print('Error parsing json body:', str(err))
+                continue
+            body_ok = expected_body_json == given_body_json
+        else:
+            body_ok = expected_body.strip() == req_body.strip()
+        if body_ok:
+            print('Matched body')
+        else:
+            msg = f'Mismatch on body:\n  got:      {req_body}\n  expected: {expected_body}'
+            print(msg)
+            continue
+        print('Matched endpoint {} {}'.format(method, path))
+        return mock_response(endpoint.get('response', {}))
+    raise Exception('Unable to match endpoint: %s %s' % (method, path))
+
+
+@app.errorhandler(Exception)
+def any_exception(err):
+    """Catch any error with a JSON response."""
+    class_name = err.__class__.__name__
+    print(traceback.format_exc())
+    resp = {'error': str(err), 'class': class_name}
+    return (flask.jsonify(resp), 500)
+
+
+def match_headers(endpoint):
+    """
+    Either check that there are no headers to match, or match that all headers
+    in the endpoint are present and equal in the request.
+    """
+    if 'headers' not in endpoint and 'absent_headers' not in endpoint:
+        return True
+    headers = dict(flask.request.headers)
+    if 'headers' in endpoint:
+        for (key, val) in endpoint['headers'].items():
+            if val != headers.get(key):
+                return False
+    # Enforce that certain headers must be absent
+    if 'absent_headers' in endpoint:
+        header_keys = set(key.lower() for key in headers.keys())
+        print('headers are', headers)
+        for key in endpoint['absent_headers']:
+            print('checking absent', key)
+            if key.lower() in header_keys:
+                return False
+    return True
+
+
+def mock_response(config):
+    """
+    Create a mock flask response from the endpoints.json configuration
+    """
+    resp_body = config.get('body')
+    if isinstance(resp_body, dict):
+        resp_body = json.dumps(resp_body)
+    resp = flask.Response(resp_body)
+    resp.status = config.get('status', '200')
+    for (header, val) in config.get('headers', {}).items():
+        resp.headers[header] = val
+    return resp

test/run_tests.sh

@@ -11,16 +11,15 @@
 # start the test NMS endpoint
 echo 'Starting NMS...'
 export KB_DEPLOYMENT_CONFIG=test.cfg
-classpath=`cat ../narrative_method_store/dist/jar.classpath.txt`
-java -cp $classpath us.kbase.narrativemethodstore.NarrativeMethodStoreServer 7125 > nms/error.log 2>&1 &
+docker-compose -f docker-compose_nms.yml up -d
 NMS_PID=$!
 
 echo 'Starting Mock Auth API...'
-docker run -d --rm -v ${PWD}/mock_auth:/config -p 7777:5000 --name mock-auth mockservices/mock_json_service
+docker run -d --rm -v ${PWD}/mock_auth:/config -v ${PWD}/mock_auth/server.py:/server/server.py -p 7777:5000 --name mock-auth mockservices/mock_json_service
 
 echo 'Waiting for NMS to start...'
 sleep 25
-curl -d '{"id":"1","params":[],"method":"NarrativeMethodStore.ver","version":"1.1"}' http://localhost:7125
+curl -d '{"id":"1","params":[],"method":"NarrativeMethodStore.ver","version":"1.1"}' http://localhost:7125/rpc
 if [ $? -ne 0 ]; then
     kill -9 $NMS_PID
     echo 'NMS did not startup in time.  Fail.'
@@ -57,7 +56,7 @@ echo "unit tests returned with error code=${TEST_RETURN_CODE}"
 ####  SHUTDOWN stuff and exit
 
 # stop NMS
-kill -9 $NMS_PID
+docker-compose -f docker-compose_nms.yml down
 
 #stop Docker containers
 docker stop mock-auth

test/test.cfg.example

@@ -24,8 +24,8 @@ test-user-2 = wstester2
 test-module-repo-1 = https://github.com/kbaseIncubator/catalog_test_module
 test-module-repo-2 = https://github.com/kbaseIncubator/catalog_test_module.git
 
-# host where mongo lives, e.g. localhost:27017, for tests there should be not authentication required
-mongodb-host = localhost:27017
+# host where mongo lives, e.g. localhost:27018, for tests there should be not authentication required
+mongodb-host = localhost:27018
 
 # docker registry host endpoint
 docker-registry-host = localhost:5000