[Bug] Test flake, may be sandbox import issue

[cretz]

Describe the bug

Sometimes on Python 3.8, there is this test failure:

tests/worker/test_replayer.py::test_replayer_workflow_incomplete - RuntimeError: Failed validating workflow SayHelloWorkflow

Here is the full logged exception:

______________________ test_replayer_workflow_incomplete _______________________

>   import asyncio

tests/worker/test_replayer.py:1: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
temporalio/worker/workflow_sandbox/_importer.py:441: in __call__
    return self.current(*args, **kwargs)
temporalio/worker/workflow_sandbox/_importer.py:234: in _import
    mod = importlib.__import__(name, globals, locals, fromlist, level)
<frozen importlib._bootstrap>:1093: in __import__
    ???
<frozen importlib._bootstrap>:1014: in _gcd_import
    ???
<frozen importlib._bootstrap>:991: in _find_and_load
    ???
<frozen importlib._bootstrap>:961: in _find_and_load_unlocked
    ???
<frozen importlib._bootstrap>:219: in _call_with_frames_removed
    ???
<frozen importlib._bootstrap>:1014: in _gcd_import
    ???
<frozen importlib._bootstrap>:991: in _find_and_load
    ???
<frozen importlib._bootstrap>:975: in _find_and_load_unlocked
    ???
<frozen importlib._bootstrap>:671: in _load_unlocked
    ???
<frozen importlib._bootstrap_external>:843: in exec_module
    ???
<frozen importlib._bootstrap>:219: in _call_with_frames_removed
    ???
../../../.cache/pypoetry/virtualenvs/temporalio-s8Ez09FR-py3.8/lib/python3.8/site-packages/_pytest/assertion/__init__.py:9: in <module>
    from _pytest.assertion import rewrite
temporalio/worker/workflow_sandbox/_importer.py:441: in __call__
    return self.current(*args, **kwargs)
temporalio/worker/workflow_sandbox/_importer.py:234: in _import
    mod = importlib.__import__(name, globals, locals, fromlist, level)
<frozen importlib._bootstrap>:1113: in __import__
    ???
<frozen importlib._bootstrap>:1042: in _handle_fromlist
    ???
<frozen importlib._bootstrap>:219: in _call_with_frames_removed
    ???
<frozen importlib._bootstrap>:1014: in _gcd_import
    ???
<frozen importlib._bootstrap>:991: in _find_and_load
    ???
<frozen importlib._bootstrap>:975: in _find_and_load_unlocked
    ???
<frozen importlib._bootstrap>:671: in _load_unlocked
    ???
<frozen importlib._bootstrap_external>:843: in exec_module
    ???
<frozen importlib._bootstrap>:219: in _call_with_frames_removed
    ???
../../../.cache/pypoetry/virtualenvs/temporalio-s8Ez09FR-py3.8/lib/python3.8/site-packages/_pytest/assertion/rewrite.py:32: in <module>
    from _pytest._io.saferepr import DEFAULT_REPR_MAX_SIZE
        self._on_eviction_hook = on_eviction_hook
        self._disable_safe_eviction = disable_safe_eviction
        self._throw_after_activation: Optional[Exception] = None
    
        # If there's a debug mode or a truthy TEMPORAL_DEBUG env var, disable
        # deadlock detection, otherwise set to 2 seconds
        self._deadlock_timeout_seconds = (
            None if debug_mode or os.environ.get("TEMPORAL_DEBUG") else 2
        )
    
        # Keep track of workflows that could not be evicted
        self._could_not_evict_count = 0
    
        # Set the worker-level failure exception types into the runner
        workflow_runner.set_worker_level_failure_exception_types(
            workflow_failure_exception_types
        )
    
        # Validate and build workflow dict
        self._workflows: Dict[str, temporalio.workflow._Definition] = {}
        self._dynamic_workflow: Optional[temporalio.workflow._Definition] = None
        for workflow in workflows:
            defn = temporalio.workflow._Definition.must_from_class(workflow)
            # Confirm name unique
            if defn.name in self._workflows:
                raise ValueError(f"More than one workflow named {defn.name}")
            # Prepare the workflow with the runner (this will error in the
            # sandbox if an import fails somehow)
            try:
                if defn.sandboxed:
                    workflow_runner.prepare_workflow(defn)
                else:
                    unsandboxed_workflow_runner.prepare_workflow(defn)
            except Exception as err:
>               raise RuntimeError(f"Failed validating workflow {defn.name}") from err
E               RuntimeError: Failed validating workflow SayHelloWorkflow

temporalio/worker/_workflow.py:130: RuntimeError

I suspect this is the previously-known issue in older Python with reimporting (python/cpython#91351). Just opening issue for tracking.