From 31c757198cc1031f1e309d493a17e78dabdc4790 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 31 May 2019 07:59:48 +0100 Subject: [PATCH 1/2] update e2e tests to allow standalone testing --- testing/scripts/conftest.py | 32 ++- testing/scripts/go_utils.py | 3 +- testing/scripts/k8s_utils.py | 15 +- testing/scripts/seldon_utils.py | 12 +- testing/scripts/test_dev.ipynb | 205 ++++++++++++++++++ .../scripts/test_helm_charts_clusterwide.py | 27 +++ 6 files changed, 273 insertions(+), 21 deletions(-) create mode 100644 testing/scripts/test_dev.ipynb diff --git a/testing/scripts/conftest.py b/testing/scripts/conftest.py index 48746db92d..761c5327dc 100644 --- a/testing/scripts/conftest.py +++ b/testing/scripts/conftest.py @@ -6,26 +6,40 @@ @pytest.fixture(scope="module") def clusterwide_seldon_helm(request): - version = get_seldon_version() - create_seldon_clusterwide_helm(request,version) - port_forward(request) + do_clusterwide_seldon_helm(request) @pytest.fixture(scope="module") -def setup_python_s2i(request): - build_python_s2i_images() +def setup_python_s2i(): + do_setup_python_s2i() @pytest.fixture(scope="module") def s2i_python_version(): - return get_s2i_python_version() + return do_s2i_python_version() @pytest.fixture(scope="session") def seldon_images(request): + do_seldon_images() + +@pytest.fixture(scope="session") +def seldon_version(): + return get_seldon_version() + +#### Implementatiosn below + +def do_s2i_python_version(): + return get_s2i_python_version() + +def do_clusterwide_seldon_helm(request=None): + version = get_seldon_version() + create_seldon_clusterwide_helm(request,version) + port_forward(request) + +def do_seldon_images(request=None): create_docker_repo(request) port_forward_docker_repo(request) build_java_images() version = get_seldon_version() build_go_images(version) -@pytest.fixture(scope="session") -def seldon_version(): - return get_seldon_version() +def do_setup_python_s2i(): + build_python_s2i_images() \ No newline at end of file diff --git a/testing/scripts/go_utils.py b/testing/scripts/go_utils.py index b320963432..4f2f4ad6c8 100644 --- a/testing/scripts/go_utils.py +++ b/testing/scripts/go_utils.py @@ -7,5 +7,6 @@ def build_go_images(version): - run("rm -rf ${PWD}/go && export GOPATH=${PWD}/go && mkdir -p $GOPATH/src/github.com/seldonio/ && cd ./go/src/github.com/seldonio && git clone https://github.com/SeldonIO/seldon-operator.git && cd seldon-operator && make docker-build docker-push-local-private VERSION="+version, shell=True, check=True) + #run("rm -rf ${PWD}/go && export GOPATH=${PWD}/go && mkdir -p $GOPATH/src/github.com/seldonio/ && cd ./go/src/github.com/seldonio && git clone https://github.com/SeldonIO/seldon-operator.git && cd seldon-operator && make docker-build docker-push-local-private VERSION="+version, shell=True, check=True) + run("rm -rf ${PWD}/go && export GOPATH=${PWD}/go && mkdir -p $GOPATH/src/github.com/seldonio/ && cd ./go/src/github.com/seldonio && git clone --single-branch --branch ambassador_update https://github.com/cliveseldon/seldon-operator.git && cd seldon-operator && make docker-build docker-push-local-private VERSION="+version, shell=True, check=True) diff --git a/testing/scripts/k8s_utils.py b/testing/scripts/k8s_utils.py index 0f84ea46a5..80937bdb4c 100644 --- a/testing/scripts/k8s_utils.py +++ b/testing/scripts/k8s_utils.py @@ -44,7 +44,8 @@ def fin(): run("kubectl delete namespace seldon", shell=True) run("kubectl delete namespace test1", shell=True) - request.addfinalizer(fin) + if not request is None: + request.addfinalizer(fin) def get_seldon_version(): @@ -76,7 +77,8 @@ def create_seldon_clusterwide_helm(request, version): run('kubectl rollout status statefulset.apps/seldon-operator-controller-manager -n seldon-system', shell=True) create_ambassador() create_seldon_gateway(version) - setup_finalizer_helm(request) + if not request is None: + setup_finalizer_helm(request) @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) @@ -95,7 +97,8 @@ def fin(): os.killpg(os.getpgid(p2.pid), signal.SIGTERM) os.killpg(os.getpgid(p3.pid), signal.SIGTERM) - request.addfinalizer(fin) + if not request is None: + request.addfinalizer(fin) def create_docker_repo(request): @@ -110,7 +113,8 @@ def fin(): run('kubectl delete -f ../resources/docker-private-registry-proxy.json --ignore-not-found=true -n default', shell=True) - request.addfinalizer(fin) + if not request is None: + request.addfinalizer(fin) @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) @@ -124,4 +128,5 @@ def fin(): print("teardown port-foward docker") os.killpg(os.getpgid(p1.pid), signal.SIGTERM) - request.addfinalizer(fin) + if not request is None: + request.addfinalizer(fin) diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_utils.py index e79b81638e..a708a27252 100644 --- a/testing/scripts/seldon_utils.py +++ b/testing/scripts/seldon_utils.py @@ -126,15 +126,15 @@ def grpc_request_ambassador(deploymentName,namespace,endpoint="localhost:8004",d def grpc_request_ambassador2(deploymentName,namespace,endpoint="localhost:8004",data_size=5,rows=1,data=None): try: - grpc_request_ambassador(deploymentName,namespace,endpoint=endpoint,data_size=data_size,rows=rows,data=data) + return grpc_request_ambassador(deploymentName,namespace,endpoint=endpoint,data_size=data_size,rows=rows,data=data) except: print("Warning - caught exception") - grpc_request_ambassador(deploymentName,namespace,endpoint=endpoint,data_size=data_size,rows=rows,data=data) + return grpc_request_ambassador(deploymentName,namespace,endpoint=endpoint,data_size=data_size,rows=rows,data=data) def grpc_request_api_gateway2(oauth_key,oauth_secret,namespace,rest_endpoint="localhost:8002",grpc_endpoint="localhost:8003",data_size=5,rows=1,data=None): try: - grpc_request_api_gateway(oauth_key,oauth_secret,namespace,rest_endpoint=rest_endpoint,grpc_endpoint=grpc_endpoint,data_size=data_size,rows=rows,data=data) + return grpc_request_api_gateway(oauth_key,oauth_secret,namespace,rest_endpoint=rest_endpoint,grpc_endpoint=grpc_endpoint,data_size=data_size,rows=rows,data=data) except: - print("Warning - caught exception") - grpc_request_api_gateway(oauth_key,oauth_secret,namespace,rest_endpoint=rest_endpoint,grpc_endpoint=grpc_endpoint,data_size=data_size,rows=rows,data=data) - + print("Warning - caught exception") + return grpc_request_api_gateway(oauth_key,oauth_secret,namespace,rest_endpoint=rest_endpoint,grpc_endpoint=grpc_endpoint,data_size=data_size,rows=rows,data=data) + diff --git a/testing/scripts/test_dev.ipynb b/testing/scripts/test_dev.ipynb new file mode 100644 index 0000000000..8b6dc71a23 --- /dev/null +++ b/testing/scripts/test_dev.ipynb @@ -0,0 +1,205 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setup env. Assumes a running cluster." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from conftest import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "do_s2i_python_version()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "do_seldon_images()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "do_clusterwide_seldon_helm()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run Tests" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from test_helm_charts_clusterwide import *\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "tester = ClusterWideTests()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tester.test_single_model()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"access_token\":\"fedaadcc-8a83-485a-8831-9db6add3e5a4\",\"token_type\":\"bearer\",\"expires_in\":24752,\"scope\":\"read write\"}\n", + "Test Oauth REST gateway\n", + "{\"access_token\":\"fedaadcc-8a83-485a-8831-9db6add3e5a4\",\"token_type\":\"bearer\",\"expires_in\":24751,\"scope\":\"read write\"}\n", + "{'meta': {'puid': '28g3qdutjabe5en7fl8euokv18', 'tags': {}, 'routing': {'myabtest': 0}, 'requestPath': {'myabtest': '', 'classifier-1': 'seldonio/mock_classifier:1.0'}, 'metrics': []}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.0762405970028679]}}}\n", + "Test OAuth gRPC gateway\n", + "{'meta': {'puid': 'pt4djf6nq7uo9rgl09hlv3v7t2', 'tags': {}, 'routing': {'myabtest': 1}, 'requestPath': {'myabtest': '', 'classifier-2': 'seldonio/mock_classifier:1.0'}, 'metrics': []}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.09633845221756757]}}}\n", + "Test OAuth gRPC gateway\n", + "{\"access_token\":\"fedaadcc-8a83-485a-8831-9db6add3e5a4\",\"token_type\":\"bearer\",\"expires_in\":24751,\"scope\":\"read write\"}\n", + "meta {\n", + " puid: \"anoicdnugsq3u2nkbjga2jktt9\"\n", + " routing {\n", + " key: \"myabtest\"\n", + " value: 1\n", + " }\n", + " requestPath {\n", + " key: \"classifier-2\"\n", + " value: \"seldonio/mock_classifier:1.0\"\n", + " }\n", + " requestPath {\n", + " key: \"myabtest\"\n", + " value: \"\"\n", + " }\n", + "}\n", + "data {\n", + " names: \"proba\"\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.07028857926652379\n", + " }\n", + "}\n", + "\n", + "Test Ambassador gRPC gateway\n", + "Warning - caught exception\n" + ] + }, + { + "ename": "_Rendezvous", + "evalue": "<_Rendezvous of RPC that terminated with:\n\tstatus = StatusCode.UNIMPLEMENTED\n\tdetails = \"\"\n\tdebug_error_string = \"{\"created\":\"@1559230254.930839422\",\"description\":\"Error received from peer\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1095,\"grpc_message\":\"\",\"grpc_status\":12}\"\n>", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31m_Rendezvous\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/work/seldon-core/fork-seldon-core/testing/scripts/seldon_utils.py\u001b[0m in \u001b[0;36mgrpc_request_ambassador2\u001b[0;34m(deploymentName, namespace, endpoint, data_size, rows, data)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mgrpc_request_ambassador\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeploymentName\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mendpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mendpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrows\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 130\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/work/seldon-core/fork-seldon-core/testing/scripts/seldon_utils.py\u001b[0m in \u001b[0;36mgrpc_request_ambassador\u001b[0;34m(deploymentName, namespace, endpoint, data_size, rows, data)\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0mmetadata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'seldon'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdeploymentName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'namespace'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/seldon-core/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, request, timeout, metadata, credentials)\u001b[0m\n\u001b[1;32m 513\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_blocking\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcredentials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 514\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_end_unary_response_blocking\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 515\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/seldon-core/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m_end_unary_response_blocking\u001b[0;34m(state, call, with_call, deadline)\u001b[0m\n\u001b[1;32m 447\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0m_Rendezvous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeadline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31m_Rendezvous\u001b[0m: <_Rendezvous of RPC that terminated with:\n\tstatus = StatusCode.UNIMPLEMENTED\n\tdetails = \"\"\n\tdebug_error_string = \"{\"created\":\"@1559230254.927240153\",\"description\":\"Error received from peer\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1095,\"grpc_message\":\"\",\"grpc_status\":12}\"\n>", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31m_Rendezvous\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtester\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_abtest_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Start second test\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mtester\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_mab_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/work/seldon-core/fork-seldon-core/testing/scripts/test_helm_charts_clusterwide.py\u001b[0m in \u001b[0;36mtest_abtest_model\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 96\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Test Ambassador gRPC gateway\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrpc_request_ambassador2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"myabtest\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"test1\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mAPI_AMBASSADOR\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 99\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 100\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"helm delete myabtest --purge\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshell\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/work/seldon-core/fork-seldon-core/testing/scripts/seldon_utils.py\u001b[0m in \u001b[0;36mgrpc_request_ambassador2\u001b[0;34m(deploymentName, namespace, endpoint, data_size, rows, data)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Warning - caught exception\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mgrpc_request_ambassador\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdeploymentName\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mendpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mendpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrows\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgrpc_request_api_gateway2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moauth_key\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0moauth_secret\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrest_endpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"localhost:8002\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mgrpc_endpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"localhost:8003\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/work/seldon-core/fork-seldon-core/testing/scripts/seldon_utils.py\u001b[0m in \u001b[0;36mgrpc_request_ambassador\u001b[0;34m(deploymentName, namespace, endpoint, data_size, rows, data)\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 123\u001b[0m \u001b[0mmetadata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'seldon'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdeploymentName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'namespace'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnamespace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 125\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/seldon-core/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, request, timeout, metadata, credentials)\u001b[0m\n\u001b[1;32m 512\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcredentials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 513\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_blocking\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcredentials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 514\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_end_unary_response_blocking\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 515\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 516\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mwith_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcredentials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/seldon-core/lib/python3.6/site-packages/grpc/_channel.py\u001b[0m in \u001b[0;36m_end_unary_response_blocking\u001b[0;34m(state, call, with_call, deadline)\u001b[0m\n\u001b[1;32m 446\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 447\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0m_Rendezvous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeadline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31m_Rendezvous\u001b[0m: <_Rendezvous of RPC that terminated with:\n\tstatus = StatusCode.UNIMPLEMENTED\n\tdetails = \"\"\n\tdebug_error_string = \"{\"created\":\"@1559230254.930839422\",\"description\":\"Error received from peer\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1095,\"grpc_message\":\"\",\"grpc_status\":12}\"\n>" + ] + } + ], + "source": [ + "tester.test_abtest_model()\n", + "print(\"Start second test\")\n", + "tester.test_mab_model()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from seldon_core.seldon_client import SeldonClient\n", + "sc = SeldonClient(deployment_name=\"mymab\",namespace=\"test1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\")\n", + "print(r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py index bbeffda8d6..4f193fecf1 100644 --- a/testing/scripts/test_helm_charts_clusterwide.py +++ b/testing/scripts/test_helm_charts_clusterwide.py @@ -31,24 +31,43 @@ def initial_rest_request(): @pytest.mark.usefixtures("clusterwide_seldon_helm") class TestClusterWide(object): + def __init__(self): + self.tester = ClusterWideTests() + + def test_single_model(self): + self.tester.test_single_model() + + def test_abtest_model(self): + self.tester.test_abtest_model() + + def test_mab_model(self): + self.tester.test_mab_model() + + +class ClusterWideTests(object): + # Test singe model helm script with 4 API methods def test_single_model(self): run("helm delete mymodel --purge", shell=True) run("helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", shell=True, check=True) wait_for_rollout("mymodel-mymodel-7cd068f") r = initial_rest_request() + print("Test OAuth REST gateway") r = rest_request_api_gateway("oauth-key","oauth-secret","test1",API_GATEWAY_REST) res = r.json() print(res) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 + print("Test Ambassador REST gateway") r = rest_request_ambassador("mymodel","test1",API_AMBASSADOR) res = r.json() print(res) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 + print("Test Oauth gRPC gateway") r = grpc_request_api_gateway2("oauth-key","oauth-secret","test1",rest_endpoint=API_GATEWAY_REST,grpc_endpoint=API_GATEWAY_GRPC) print(r) + print("Test Ambassador gRPC gateway") r = grpc_request_ambassador2("mymodel","test1",API_AMBASSADOR) print(r) run("helm delete mymodel --purge", shell=True) @@ -60,18 +79,22 @@ def test_abtest_model(self): wait_for_rollout("myabtest-abtest-41de5b8") wait_for_rollout("myabtest-abtest-df66c5c") r = initial_rest_request() + print("Test Oauth REST gateway") r = rest_request_api_gateway("oauth-key","oauth-secret","test1",API_GATEWAY_REST) res = r.json() print(res) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 + print("Test OAuth gRPC gateway") r = rest_request_ambassador("myabtest","test1",API_AMBASSADOR) res = r.json() print(res) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 + print("Test OAuth gRPC gateway") r = grpc_request_api_gateway2("oauth-key","oauth-secret","test1",rest_endpoint=API_GATEWAY_REST,grpc_endpoint=API_GATEWAY_GRPC) print(r) + print("Test Ambassador gRPC gateway") r = grpc_request_ambassador2("myabtest","test1",API_AMBASSADOR) print(r) run("helm delete myabtest --purge", shell=True) @@ -84,18 +107,22 @@ def test_mab_model(self): wait_for_rollout("mymab-abtest-b8038b2") wait_for_rollout("mymab-abtest-df66c5c") r = initial_rest_request() + print("Test Oauth REST gateway") r = rest_request_api_gateway("oauth-key","oauth-secret","test1",API_GATEWAY_REST) res = r.json() print(res) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 + print("Test Ambassador REST gateway") r = rest_request_ambassador("mymab","test1",API_AMBASSADOR) res = r.json() print(res) assert r.status_code == 200 assert len(r.json()["data"]["tensor"]["values"]) == 1 + print("Test OAuth gRPC gateway") r = grpc_request_api_gateway2("oauth-key","oauth-secret","test1",rest_endpoint=API_GATEWAY_REST,grpc_endpoint=API_GATEWAY_GRPC) print(r) + print("Test Ambassador gRPC gateway") r = grpc_request_ambassador2("mymab","test1",API_AMBASSADOR) print(r) run("helm delete mymab --purge", shell=True) From 35acc8435ef7f7ac78987412b5d0583e4b969736 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 31 May 2019 08:01:48 +0100 Subject: [PATCH 2/2] remove branch usage in go compile for tests --- testing/scripts/go_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/scripts/go_utils.py b/testing/scripts/go_utils.py index 4f2f4ad6c8..d680ea5dad 100644 --- a/testing/scripts/go_utils.py +++ b/testing/scripts/go_utils.py @@ -7,6 +7,6 @@ def build_go_images(version): - #run("rm -rf ${PWD}/go && export GOPATH=${PWD}/go && mkdir -p $GOPATH/src/github.com/seldonio/ && cd ./go/src/github.com/seldonio && git clone https://github.com/SeldonIO/seldon-operator.git && cd seldon-operator && make docker-build docker-push-local-private VERSION="+version, shell=True, check=True) - run("rm -rf ${PWD}/go && export GOPATH=${PWD}/go && mkdir -p $GOPATH/src/github.com/seldonio/ && cd ./go/src/github.com/seldonio && git clone --single-branch --branch ambassador_update https://github.com/cliveseldon/seldon-operator.git && cd seldon-operator && make docker-build docker-push-local-private VERSION="+version, shell=True, check=True) + run("rm -rf ${PWD}/go && export GOPATH=${PWD}/go && mkdir -p $GOPATH/src/github.com/seldonio/ && cd ./go/src/github.com/seldonio && git clone https://github.com/SeldonIO/seldon-operator.git && cd seldon-operator && make docker-build docker-push-local-private VERSION="+version, shell=True, check=True) +