take latest ambassador image

[ryandawsonuk]

will fix #596

after that will need to check #473 again and prob reopen #491

But currently getting errors on E2E tests:

=================================== FAILURES ===================================
______________________ TestClusterWide.test_single_model _______________________

self = <test_helm_charts_clusterwide.TestClusterWide object at 0x7fa1bf758cc0>

    def test_single_model(self):
        run("helm delete mymodel --purge", shell=True)
>       run("helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", shell=True, check=True)

test_helm_charts_clusterwide.py:37: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

input = None, capture_output = False, timeout = None, check = True
popenargs = ('helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1',)
kwargs = {'shell': True}, process = <subprocess.Popen object at 0x7fa1bf758908>
stdout = None, stderr = None, retcode = 1

    def run(*popenargs,
            input=None, capture_output=False, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.
    
        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
    
        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.
    
        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.
    
        There is an optional argument "input", allowing you to
        pass bytes or a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.
    
        By default, all communication is in bytes, and therefore any "input" should
        be bytes, and the stdout and stderr will be bytes. If in text mode, any
        "input" should be a string, and stdout and stderr will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode is
        triggered by setting any of text, encoding, errors or universal_newlines.
    
        The other arguments are the same as for the Popen constructor.
        """
        if input is not None:
            if 'stdin' in kwargs:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE
    
        if capture_output:
            if ('stdout' in kwargs) or ('stderr' in kwargs):
                raise ValueError('stdout and stderr arguments may not be used '
                                 'with capture_output.')
            kwargs['stdout'] = PIPE
            kwargs['stderr'] = PIPE
    
        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired:
                process.kill()
                stdout, stderr = process.communicate()
                raise TimeoutExpired(process.args, timeout, output=stdout,
                                     stderr=stderr)
            except:  # Including KeyboardInterrupt, communicate handled that.
                process.kill()
                # We don't call process.wait() as .__exit__ does that for us.
                raise
            retcode = process.poll()
            if check and retcode:
                raise CalledProcessError(retcode, process.args,
>                                        output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command 'helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1' returned non-zero exit status 1.

/home/ryan/anaconda3/lib/python3.7/subprocess.py:481: CalledProcessError
______________________ TestClusterWide.test_abtest_model _______________________

self = <test_helm_charts_clusterwide.TestClusterWide object at 0x7fa1bf7aa550>

    def test_abtest_model(self):
        run("helm delete myabtest --purge", shell=True)
>       run("helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", shell=True, check=True)

test_helm_charts_clusterwide.py:59: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

input = None, capture_output = False, timeout = None, check = True
popenargs = ('helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1',)
kwargs = {'shell': True}, process = <subprocess.Popen object at 0x7fa1bf7aa898>
stdout = None, stderr = None, retcode = 1

    def run(*popenargs,
            input=None, capture_output=False, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.
    
        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
    
        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.
    
        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.
    
        There is an optional argument "input", allowing you to
        pass bytes or a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.
    
        By default, all communication is in bytes, and therefore any "input" should
        be bytes, and the stdout and stderr will be bytes. If in text mode, any
        "input" should be a string, and stdout and stderr will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode is
        triggered by setting any of text, encoding, errors or universal_newlines.
    
        The other arguments are the same as for the Popen constructor.
        """
        if input is not None:
            if 'stdin' in kwargs:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE
    
        if capture_output:
            if ('stdout' in kwargs) or ('stderr' in kwargs):
                raise ValueError('stdout and stderr arguments may not be used '
                                 'with capture_output.')
            kwargs['stdout'] = PIPE
            kwargs['stderr'] = PIPE
    
        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired:
                process.kill()
                stdout, stderr = process.communicate()
                raise TimeoutExpired(process.args, timeout, output=stdout,
                                     stderr=stderr)
            except:  # Including KeyboardInterrupt, communicate handled that.
                process.kill()
                # We don't call process.wait() as .__exit__ does that for us.
                raise
            retcode = process.poll()
            if check and retcode:
                raise CalledProcessError(retcode, process.args,
>                                        output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command 'helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1' returned non-zero exit status 1.

/home/ryan/anaconda3/lib/python3.7/subprocess.py:481: CalledProcessError
________________________ TestClusterWide.test_mab_model ________________________

self = <test_helm_charts_clusterwide.TestClusterWide object at 0x7fa1bf759ac8>

    def test_mab_model(self):
        run("helm delete mymab --purge", shell=True)
>       run("helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", shell=True, check=True)

test_helm_charts_clusterwide.py:82: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

input = None, capture_output = False, timeout = None, check = True
popenargs = ('helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1',)
kwargs = {'shell': True}, process = <subprocess.Popen object at 0x7fa1bf759438>
stdout = None, stderr = None, retcode = 1

    def run(*popenargs,
            input=None, capture_output=False, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.
    
        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
    
        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.
    
        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.
    
        There is an optional argument "input", allowing you to
        pass bytes or a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.
    
        By default, all communication is in bytes, and therefore any "input" should
        be bytes, and the stdout and stderr will be bytes. If in text mode, any
        "input" should be a string, and stdout and stderr will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode is
        triggered by setting any of text, encoding, errors or universal_newlines.
    
        The other arguments are the same as for the Popen constructor.
        """
        if input is not None:
            if 'stdin' in kwargs:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE
    
        if capture_output:
            if ('stdout' in kwargs) or ('stderr' in kwargs):
                raise ValueError('stdout and stderr arguments may not be used '
                                 'with capture_output.')
            kwargs['stdout'] = PIPE
            kwargs['stderr'] = PIPE
    
        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired:
                process.kill()
                stdout, stderr = process.communicate()
                raise TimeoutExpired(process.args, timeout, output=stdout,
                                     stderr=stderr)
            except:  # Including KeyboardInterrupt, communicate handled that.
                process.kill()
                # We don't call process.wait() as .__exit__ does that for us.
                raise
            retcode = process.poll()
            if check and retcode:
                raise CalledProcessError(retcode, process.args,
>                                        output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command 'helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1' returned non-zero exit status 1.

/home/ryan/anaconda3/lib/python3.7/subprocess.py:481: CalledProcessError
==================== 3 failed, 30 passed in 1704.39 seconds ====================

[ryandawsonuk]

Failures might be unrelated - not sure yet but could be due to the fixtures being commented out on those tests https://github.com/SeldonIO/seldon-core/blob/master/testing/scripts/test_helm_charts_clusterwide.py#L30

[ryandawsonuk]

Might be due to this:

partitioned roll out complete: 1 new pods have been updated...
Error: release ambassador failed: customresourcedefinitions.apiextensions.k8s.io "tracingservices.getambassador.io" already exists
Error from server (NotFound): deployments.apps "ambassador" not found

[ryandawsonuk]

Ok now this works but our old friend the GRPC UNIMPLEMENTED error is still present in the latest version:

=================================== FAILURES ===================================
______________________ TestClusterWide.test_abtest_model _______________________

deploymentName = 'myabtest', namespace = 'test1', endpoint = 'localhost:8003'
data_size = 5, rows = 1, data = None

    def grpc_request_ambassador2(deploymentName,namespace,endpoint="localhost:8004",data_size=5,rows=1,data=None):
        try:
>           grpc_request_ambassador(deploymentName,namespace,endpoint=endpoint,data_size=data_size,rows=rows,data=data)

seldon_utils.py:129: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = ('myabtest', 'test1')
kw = {'data': None, 'data_size': 5, 'endpoint': 'localhost:8003', 'rows': 1}

    @six.wraps(f)
    def wrapped_f(*args, **kw):
>       return Retrying(*dargs, **dkw).call(f, *args, **kw)

/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:49: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <retrying.Retrying object at 0x7f45163318d0>
fn = <function grpc_request_ambassador at 0x7f45169adae8>
args = ('myabtest', 'test1')
kwargs = {'data': None, 'data_size': 5, 'endpoint': 'localhost:8003', 'rows': 1}
start_time = 1559121828522, attempt_number = 5
attempt = Attempts: 5, Error:
  File "/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py", line 200, in call
    attem...ckages/grpc/_channel.py", line 466, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)

tb = (<class 'grpc._channel._Rendezvous'>, <_Rendezvous of RPC that terminated with:
	status = StatusCode.UNIMPLEMENTED
	de...ore/lib/surface/call.cc","file_line":1039,"grpc_message":"","grpc_status":12}"
>, <traceback object at 0x7f45162f6bc8>)
delay_since_first_attempt_ms = 24051, sleep = 10000

    def call(self, fn, *args, **kwargs):
        start_time = int(round(time.time() * 1000))
        attempt_number = 1
        while True:
            try:
                attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
            except:
                tb = sys.exc_info()
                attempt = Attempt(tb, attempt_number, True)
    
            if not self.should_reject(attempt):
                return attempt.get(self._wrap_exception)
    
            delay_since_first_attempt_ms = int(round(time.time() * 1000)) - start_time
            if self.stop(attempt_number, delay_since_first_attempt_ms):
                if not self._wrap_exception and attempt.has_exception:
                    # get() on an attempt with an exception should cause it to be raised, but raise just in case
>                   raise attempt.get()

/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:212: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Attempts: 5, Error:
  File "/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py", line 200, in call
    attem...ckages/grpc/_channel.py", line 466, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)

wrap_exception = False

    def get(self, wrap_exception=False):
        """
        Return the return value of this Attempt instance or raise an Exception.
        If wrap_exception is true, this Attempt is wrapped inside of a
        RetryError before being raised.
        """
        if self.has_exception:
            if wrap_exception:
                raise RetryError(self)
            else:
>               six.reraise(self.value[0], self.value[1], self.value[2])

/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:247: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

tp = <class 'grpc._channel._Rendezvous'>, value = None, tb = None

    def reraise(tp, value, tb=None):
        try:
            if value is None:
                value = tp()
            if value.__traceback__ is not tb:
                raise value.with_traceback(tb)
>           raise value

/home/ryan/anaconda3/lib/python3.7/site-packages/six.py:693: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <retrying.Retrying object at 0x7f45163318d0>
fn = <function grpc_request_ambassador at 0x7f45169adae8>
args = ('myabtest', 'test1')
kwargs = {'data': None, 'data_size': 5, 'endpoint': 'localhost:8003', 'rows': 1}
start_time = 1559121828522, attempt_number = 5
attempt = Attempts: 5, Error:
  File "/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py", line 200, in call
    attem...ckages/grpc/_channel.py", line 466, in _end_unary_response_blocking
    raise _Rendezvous(state, None, None, deadline)

tb = (<class 'grpc._channel._Rendezvous'>, <_Rendezvous of RPC that terminated with:
	status = StatusCode.UNIMPLEMENTED
	de...ore/lib/surface/call.cc","file_line":1039,"grpc_message":"","grpc_status":12}"
>, <traceback object at 0x7f45162f6bc8>)
delay_since_first_attempt_ms = 24051, sleep = 10000

    def call(self, fn, *args, **kwargs):
        start_time = int(round(time.time() * 1000))
        attempt_number = 1
        while True:
            try:
>               attempt = Attempt(fn(*args, **kwargs), attempt_number, False)

/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:200: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

deploymentName = 'myabtest', namespace = 'test1', endpoint = 'localhost:8003'
data_size = 5, rows = 1, data = None

    @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5)
    def grpc_request_ambassador(deploymentName,namespace,endpoint="localhost:8004",data_size=5,rows=1,data=None):
        if data is None:
            shape, arr = create_random_data(data_size,rows)
        else:
            shape = data.shape
            arr = data.flatten()
        datadef = prediction_pb2.DefaultData(
                tensor = prediction_pb2.Tensor(
                    shape = shape,
                    values = arr
                    )
                )
        request = prediction_pb2.SeldonMessage(data = datadef)
        channel = grpc.insecure_channel(endpoint)
        stub = prediction_pb2_grpc.SeldonStub(channel)
        if namespace is None:
            metadata = [('seldon',deploymentName)]
        else:
            metadata = [('seldon',deploymentName),('namespace',namespace)]
>       response = stub.Predict(request=request,metadata=metadata)

seldon_utils.py:124: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f4516331b70>
request = data {
  tensor {
    shape: 1
    shape: 5
    values: 0.4023831385326194
    values: 0.2656419296835113
    values: 0.12663783483215063
    values: 0.5956162282056227
    values: 0.4585063357307485
  }
}

timeout = None, metadata = [('seldon', 'myabtest'), ('namespace', 'test1')]
credentials = None, wait_for_ready = None

    def __call__(self,
                 request,
                 timeout=None,
                 metadata=None,
                 credentials=None,
                 wait_for_ready=None):
        state, call, = self._blocking(request, timeout, metadata, credentials,
                                      wait_for_ready)
>       return _end_unary_response_blocking(state, call, False, None)

/home/ryan/anaconda3/lib/python3.7/site-packages/grpc/_channel.py:549: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = <grpc._channel._RPCState object at 0x7f45162de860>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f4516391848>
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _Rendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _Rendezvous(state, None, None, deadline)
E           grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
E           	status = StatusCode.UNIMPLEMENTED
E           	details = ""
E           	debug_error_string = "{"created":"@1559121852.572991445","description":"Error received from peer","file":"src/core/lib/surface/call.cc","file_line":1039,"grpc_message":"","grpc_status":12}"
E           >

/home/ryan/anaconda3/lib/python3.7/site-packages/grpc/_channel.py:466: _Rendezvous

During handling of the above exception, another exception occurred:

self = <test_helm_charts_clusterwide.TestClusterWide object at 0x7f4516331d30>

    def test_abtest_model(self):
        run("helm delete myabtest --purge", shell=True)
        run("helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1", shell=True, check=True)
        wait_for_rollout("myabtest-abtest-41de5b8")
        wait_for_rollout("myabtest-abtest-df66c5c")
        r = initial_rest_request()
        r = rest_request_api_gateway("oauth-key","oauth-secret","test1",API_GATEWAY_REST)
        res = r.json()
        print(res)
        assert r.status_code == 200
        assert len(r.json()["data"]["tensor"]["values"]) == 1
        r = rest_request_ambassador("myabtest","test1",API_AMBASSADOR)
        res = r.json()
        print(res)
        assert r.status_code == 200
        assert len(r.json()["data"]["tensor"]["values"]) == 1
        r = grpc_request_api_gateway2("oauth-key","oauth-secret","test1",rest_endpoint=API_GATEWAY_REST,grpc_endpoint=API_GATEWAY_GRPC)
        print(r)
>       r = grpc_request_ambassador2("myabtest","test1",API_AMBASSADOR)

test_helm_charts_clusterwide.py:75: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
seldon_utils.py:132: in grpc_request_ambassador2
    grpc_request_ambassador(deploymentName,namespace,endpoint=endpoint,data_size=data_size,rows=rows,data=data)
/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:49: in wrapped_f
    return Retrying(*dargs, **dkw).call(f, *args, **kw)
/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:212: in call
    raise attempt.get()
/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:247: in get
    six.reraise(self.value[0], self.value[1], self.value[2])
/home/ryan/anaconda3/lib/python3.7/site-packages/six.py:693: in reraise
    raise value
/home/ryan/anaconda3/lib/python3.7/site-packages/retrying.py:200: in call
    attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
seldon_utils.py:124: in grpc_request_ambassador
    response = stub.Predict(request=request,metadata=metadata)
/home/ryan/anaconda3/lib/python3.7/site-packages/grpc/_channel.py:549: in __call__
    return _end_unary_response_blocking(state, call, False, None)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = <grpc._channel._RPCState object at 0x7f45162de898>
call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f45162f6988>
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _Rendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _Rendezvous(state, None, None, deadline)
E           grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
E           	status = StatusCode.UNIMPLEMENTED
E           	details = ""
E           	debug_error_string = "{"created":"@1559121876.620087775","description":"Error received from peer","file":"src/core/lib/surface/call.cc","file_line":1039,"grpc_message":"","grpc_status":12}"
E           >

/home/ryan/anaconda3/lib/python3.7/site-packages/grpc/_channel.py:466: _Rendezvous
==================== 1 failed, 32 passed in 1898.89 seconds ====================