diff --git a/src/ansiblelint/cli.py b/src/ansiblelint/cli.py
index 508cb4e913..36741b9eb0 100644
--- a/src/ansiblelint/cli.py
+++ b/src/ansiblelint/cli.py
@@ -314,7 +314,7 @@ def merge_config(file_config: Dict[Any, Any], cli_config: Namespace) -> Namespac
     )
     # maps lists to their default config values
     lists_map = {
-        'exclude_paths': [".cache"],
+        'exclude_paths': [".cache", ".git", ".hg", ".svn", ".tox"],
         'rulesdir': [],
         'skip_list': [],
         'tags': [],
diff --git a/src/ansiblelint/file_utils.py b/src/ansiblelint/file_utils.py
index b8b3420071..791c8927be 100644
--- a/src/ansiblelint/file_utils.py
+++ b/src/ansiblelint/file_utils.py
@@ -11,7 +11,9 @@
 from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Union
 
+# import wcmatch
 import wcmatch.pathlib
+from wcmatch.wcmatch import RECURSIVE, WcMatch
 
 from ansiblelint.config import BASE_KINDS, options
 from ansiblelint.constants import FileType
@@ -205,14 +207,13 @@ def discover_lintables(options: Namespace) -> Dict[str, Any]:
     """Find all files that we know how to lint."""
     # git is preferred as it also considers .gitignore
     git_command = ['git', 'ls-files', '-z']
-    _logger.info("Discovering files to lint: %s", ' '.join(git_command))
-
     out = None
 
     try:
         out = subprocess.check_output(
             git_command, stderr=subprocess.STDOUT, universal_newlines=True
         ).split("\x00")[:-1]
+        _logger.info("Discovered files to lint using: %s", ' '.join(git_command))
     except subprocess.CalledProcessError as exc:
         if not (exc.returncode == 128 and 'fatal: not a git repository' in exc.output):
             _logger.warning(
@@ -224,15 +225,9 @@ def discover_lintables(options: Namespace) -> Dict[str, Any]:
             _logger.warning("Failed to locate command: %s", exc)
 
     if out is None:
-        # TODO(ssbarnea): avoid returning only yaml/yml files but be careful
-        # to avoid accidental return of too many files, especiall as some
-        # directories like .cache, .tox may bring undesireble noise.
-        out = [
-            os.path.join(root, name)
-            for root, dirs, files in os.walk('.')
-            for name in files
-            if name.endswith('.yaml') or name.endswith('.yml')
-        ]
+        exclude_pattern = "|".join(options.exclude_paths)
+        _logger.info("Looking up for files, excluding %s ...", exclude_pattern)
+        out = WcMatch('.', exclude_pattern=exclude_pattern, flags=RECURSIVE).match()
 
     return OrderedDict.fromkeys(sorted(out))
 
diff --git a/test/TestUtils.py b/test/TestUtils.py
index 0035df4020..16f78f9d75 100644
--- a/test/TestUtils.py
+++ b/test/TestUtils.py
@@ -212,27 +212,23 @@ def test_expand_paths_vars(test_path, expected, monkeypatch):
 @pytest.mark.parametrize(
     ('reset_env_var', 'message_prefix'),
     (
+        # simulate absence of git command
         ('PATH', "Failed to locate command: "),
-        ('GIT_DIR', "Discovering files to lint: "),
+        # simulate a missing git repo
+        ('GIT_DIR', "Looking up for files"),
     ),
     ids=('no-git-cli', 'outside-git-repo'),
 )
 def test_discover_lintables_git_verbose(
-    reset_env_var, message_prefix, monkeypatch, caplog
-):
+    reset_env_var: str, message_prefix: str, monkeypatch, caplog
+) -> None:
     """Ensure that autodiscovery lookup failures are logged."""
     options = cli.get_config(['-v'])
     initialize_logger(options.verbosity)
     monkeypatch.setenv(reset_env_var, '')
     file_utils.discover_lintables(options)
 
-    expected_info = (
-        "ansiblelint",
-        logging.INFO,
-        'Discovering files to lint: git ls-files -z',
-    )
-
-    assert expected_info in caplog.record_tuples
+    assert any(m[2].startswith("Looking up for files") for m in caplog.record_tuples)
     assert any(m.startswith(message_prefix) for m in caplog.messages)
 
 
@@ -314,7 +310,7 @@ def test_cli_auto_detect(capfd):
     out, err = capfd.readouterr()
 
     # Confirmation that it runs in auto-detect mode
-    assert "Discovering files to lint: git ls-files -z" in err
+    assert "Discovered files to lint using: git ls-files -z" in err
     # An expected rule match from our examples
     assert (
         "examples/playbooks/empty_playbook.yml:1: "