Race condition in test harness teardown

[niftynei]

Some CI tests fail at teardown?

==================================== ERRORS ====================================
_______________ ERROR at teardown of test_backfill_scriptpubkeys _______________
[gw2] linux -- Python 3.6.13 /opt/hostedtoolcache/Python/3.6.13/x64/bin/python3

request = <SubRequest 'directory' for <Function test_backfill_scriptpubkeys>>
test_base_dir = '/tmp/ltests-du69g5u_'
test_name = 'test_backfill_scriptpubkeys'

    @pytest.fixture
    def directory(request, test_base_dir, test_name):
        """Return a per-test specific directory.
    
        This makes a unique test-directory even if a test is rerun multiple times.
    
        """
        global __attempts
        # Auto set value if it isn't in the dict yet
        __attempts[test_name] = __attempts.get(test_name, 0) + 1
        directory = os.path.join(test_base_dir, "{}_{}".format(test_name, __attempts[test_name]))
        request.node.has_errors = False
    
        if not os.path.exists(directory):
            os.makedirs(directory)
    
        yield directory
    
        # This uses the status set in conftest.pytest_runtest_makereport to
        # determine whether we succeeded or failed. Outcome can be None if the
        # failure occurs during the setup phase, hence the use to getattr instead
        # of accessing it directly.
        rep_call = getattr(request.node, 'rep_call', None)
        outcome = 'passed' if rep_call is None else rep_call.outcome
        failed = not outcome or request.node.has_errors or outcome != 'passed'
    
        if not failed:
            try:
>               shutil.rmtree(directory)

contrib/pyln-testing/pyln/testing/fixtures.py:92: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/shutil.py:486: in rmtree
    _rmtree_safe_fd(fd, path, onerror)
/opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/shutil.py:428: in _rmtree_safe_fd
    onerror(os.rmdir, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

topfd = 12, path = '/tmp/ltests-du69g5u_/test_backfill_scriptpubkeys_1'
onerror = <function rmtree.<locals>.onerror at 0x7efd4118b378>

    def _rmtree_safe_fd(topfd, path, onerror):
        names = []
        try:
            names = os.listdir(topfd)
        except OSError as err:
            err.filename = path
            onerror(os.listdir, path, sys.exc_info())
        for name in names:
            fullname = os.path.join(path, name)
            try:
                orig_st = os.stat(name, dir_fd=topfd, follow_symlinks=False)
                mode = orig_st.st_mode
            except OSError:
                mode = 0
            if stat.S_ISDIR(mode):
                try:
                    dirfd = os.open(name, os.O_RDONLY, dir_fd=topfd)
                except OSError:
                    onerror(os.open, fullname, sys.exc_info())
                else:
                    try:
                        if os.path.samestat(orig_st, os.fstat(dirfd)):
                            _rmtree_safe_fd(dirfd, fullname, onerror)
                            try:
>                               os.rmdir(name, dir_fd=topfd)
E                               OSError: [Errno 39] Directory not empty: 'lightning-3'

/opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/shutil.py:426: OSError
---------------------------- Captured stdout setup -----------------------------