fix /learn not working for directories

packages/jupyter-ai/jupyter_ai/document_loaders/directory.py

@@ -66,20 +66,28 @@ def split(path, all_files: bool, splitter):
 
     # Check if the path points to a single file
     if os.path.isfile(path):
-        dir = os.path.dirname(path)
-        filenames = [os.path.basename(path)]
+        filepaths = [Path(path)]
     else:
-        for dir, subdirs, filenames in os.walk(path):
-            # Filter out hidden filenames, hidden directories, and excluded directories,
-            # unless "all files" are requested
+        _skip_dirs = []
+        _filepaths = []
+        for dir, _, filenames in os.walk(path):
             if not all_files:
-                subdirs[:] = [
-                    d for d in subdirs if not (d[0] == "." or d in EXCLUDE_DIRS)
-                ]
+                # Filter out hidden filenames, hidden directories, and excluded directories,
+                # unless "all files" are requested
+                if any(dir.startswith(skip) for skip in _skip_dirs):
+                    continue
+                dir_name = os.path.basename(dir)
+                if dir != path and (os.path.basename(dir).startswith(".") or dir_name in EXCLUDE_DIRS):
+                    _skip_dirs.append(dir)
+                    continue
                 filenames = [f for f in filenames if not f[0] == "."]
 
-    for filename in filenames:
-        filepath = Path(os.path.join(dir, filename))
+            _filepaths.append(
+                [Path(os.path.join(dir, filename)) for filename in filenames]
+            )
+        filepaths = [fp for ls in _filepaths for fp in ls]
+
+    for filepath in filepaths:
         # Lower case everything to make sure file extension comparisons are not case sensitive
         if filepath.suffix.lower() not in {j.lower() for j in SUPPORTED_EXTS}:
             continue
@@ -92,6 +100,7 @@ def split(path, all_files: bool, splitter):
     return flattened_chunks
 
 
+
 def join(embeddings):
     embedding_records = []
     metadatas = []