[Bug]: Spark client panics when reading DataFrame from entire reposit…

…ory / storageNamespace (#7955)
treeverse · Jul 4, 2024 · a9ec2d7 · a9ec2d7
1 parent c15ff92
commit a9ec2d7
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/clients/spark/build.sbt b/clients/spark/build.sbt
@@ -1,4 +1,4 @@
-lazy val projectVersion = "0.14.0"
+lazy val projectVersion = "0.14.1"
 version := projectVersion
 lazy val hadoopVersion = "3.2.1"
 ThisBuild / isSnapshot := false

diff --git a/clients/spark/src/main/scala/io/treeverse/clients/LakeFSInputFormat.scala b/clients/spark/src/main/scala/io/treeverse/clients/LakeFSInputFormat.scala
@@ -243,12 +243,12 @@ class LakeFSAllRangesInputFormat extends LakeFSBaseInputFormat {
     if (StringUtils.isBlank(storageNamespace)) {
       storageNamespace = apiClient.getStorageNamespace(repoName, StorageClientType.HadoopFS)
     }
-    val namespaceURI = URI.create(storageNamespace)
+    val namespaceURI =
+      URI.create(if (storageNamespace.endsWith("/")) storageNamespace else storageNamespace + "/")
     val fs = fileSystemGetter(namespaceURI, conf)
     fs.getStatus(new Path(namespaceURI)) // Will throw an exception if namespace doesn't exist
 
-    val metadataURI =
-      namespaceURI.resolve(if (storageNamespace.endsWith("/")) "_lakefs" else "/_lakefs")
+    val metadataURI = namespaceURI.resolve("_lakefs")
     val metadataPath = new Path(metadataURI)
     if (!fs.exists(metadataPath)) {
       return ListBuffer.empty[InputSplit].asJava