diff --git a/build-logic/src/main/kotlin/kotlinx/io/conventions/kotlinx-io-multiplatform.gradle.kts b/build-logic/src/main/kotlin/kotlinx/io/conventions/kotlinx-io-multiplatform.gradle.kts index e9a577e2f..3484c714f 100644 --- a/build-logic/src/main/kotlin/kotlinx/io/conventions/kotlinx-io-multiplatform.gradle.kts +++ b/build-logic/src/main/kotlin/kotlinx/io/conventions/kotlinx-io-multiplatform.gradle.kts @@ -73,13 +73,19 @@ kotlin { applyDefaultHierarchyTemplate { common { group("native") { - group("nonApple") { + group("nativeNonApple") { group("mingw") group("unix") { group("linux") group("androidNative") } } + + group("nativeNonAndroid") { + group("apple") + group("mingw") + group("linux") + } } group("nodeFilesystemShared") { withJs() diff --git a/core/Module.md b/core/Module.md index 9e2da6d11..840b92af8 100644 --- a/core/Module.md +++ b/core/Module.md @@ -82,3 +82,8 @@ Core IO primitives. # Package kotlinx.io.files Basic API for working with files. + +#### Known issues + +- [#312](https://github.com/Kotlin/kotlinx-io/issues/312) For `wasmWasi` target, directory listing ([kotlinx.io.files.FileSystem.list]) does not work with NodeJS runtime on Windows, +as `fd_readdir` function is [not implemented there](https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19). diff --git a/core/androidNative/src/files/FileSystemAndroid.kt b/core/androidNative/src/files/FileSystemAndroid.kt index 8b29a1440..cc80fd650 100644 --- a/core/androidNative/src/files/FileSystemAndroid.kt +++ b/core/androidNative/src/files/FileSystemAndroid.kt @@ -5,10 +5,12 @@ package kotlinx.io.files +import kotlinx.cinterop.CPointer import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.get import kotlinx.cinterop.toKString -import platform.posix.__posix_basename -import platform.posix.dirname +import kotlinx.io.IOException +import platform.posix.* @OptIn(ExperimentalForeignApi::class) internal actual fun dirnameImpl(path: String): String { @@ -24,3 +26,22 @@ internal actual fun basenameImpl(path: String): String { } internal actual fun isAbsoluteImpl(path: String): Boolean = path.startsWith('/') + +@OptIn(ExperimentalForeignApi::class, ExperimentalStdlibApi::class) +internal actual class OpaqueDirEntry constructor(private val dir: CPointer) : AutoCloseable { + actual fun readdir(): String? { + val entry = platform.posix.readdir(dir) ?: return null + return entry[0].d_name.toKString() + } + + override fun close() { + closedir(dir) + } +} + +@OptIn(ExperimentalForeignApi::class) +internal actual fun opendir(path: String): OpaqueDirEntry { + val dirent = platform.posix.opendir(path) + if (dirent != null) return OpaqueDirEntry(dirent) + throw IOException("Can't open directory $path: ${strerror(errno)?.toKString() ?: "reason unknown"}") +} diff --git a/core/api/kotlinx-io-core.api b/core/api/kotlinx-io-core.api index fb08b1197..1f1dff89a 100644 --- a/core/api/kotlinx-io-core.api +++ b/core/api/kotlinx-io-core.api @@ -215,6 +215,7 @@ public abstract interface class kotlinx/io/files/FileSystem { public abstract fun delete (Lkotlinx/io/files/Path;Z)V public static synthetic fun delete$default (Lkotlinx/io/files/FileSystem;Lkotlinx/io/files/Path;ZILjava/lang/Object;)V public abstract fun exists (Lkotlinx/io/files/Path;)Z + public abstract fun list (Lkotlinx/io/files/Path;)Ljava/util/Collection; public abstract fun metadataOrNull (Lkotlinx/io/files/Path;)Lkotlinx/io/files/FileMetadata; public abstract fun resolve (Lkotlinx/io/files/Path;)Lkotlinx/io/files/Path; public abstract fun sink (Lkotlinx/io/files/Path;Z)Lkotlinx/io/RawSink; diff --git a/core/api/kotlinx-io-core.klib.api b/core/api/kotlinx-io-core.klib.api index 2602c51d6..49e886dc6 100644 --- a/core/api/kotlinx-io-core.klib.api +++ b/core/api/kotlinx-io-core.klib.api @@ -164,6 +164,7 @@ sealed interface kotlinx.io.files/FileSystem { // kotlinx.io.files/FileSystem|nu abstract fun createDirectories(kotlinx.io.files/Path, kotlin/Boolean =...) // kotlinx.io.files/FileSystem.createDirectories|createDirectories(kotlinx.io.files.Path;kotlin.Boolean){}[0] abstract fun delete(kotlinx.io.files/Path, kotlin/Boolean =...) // kotlinx.io.files/FileSystem.delete|delete(kotlinx.io.files.Path;kotlin.Boolean){}[0] abstract fun exists(kotlinx.io.files/Path): kotlin/Boolean // kotlinx.io.files/FileSystem.exists|exists(kotlinx.io.files.Path){}[0] + abstract fun list(kotlinx.io.files/Path): kotlin.collections/Collection // kotlinx.io.files/FileSystem.list|list(kotlinx.io.files.Path){}[0] abstract fun metadataOrNull(kotlinx.io.files/Path): kotlinx.io.files/FileMetadata? // kotlinx.io.files/FileSystem.metadataOrNull|metadataOrNull(kotlinx.io.files.Path){}[0] abstract fun resolve(kotlinx.io.files/Path): kotlinx.io.files/Path // kotlinx.io.files/FileSystem.resolve|resolve(kotlinx.io.files.Path){}[0] abstract fun sink(kotlinx.io.files/Path, kotlin/Boolean =...): kotlinx.io/RawSink // kotlinx.io.files/FileSystem.sink|sink(kotlinx.io.files.Path;kotlin.Boolean){}[0] diff --git a/core/apple/src/files/FileSystemApple.kt b/core/apple/src/files/FileSystemApple.kt index ed2632997..75882b967 100644 --- a/core/apple/src/files/FileSystemApple.kt +++ b/core/apple/src/files/FileSystemApple.kt @@ -6,10 +6,7 @@ package kotlinx.io.files -import kotlinx.cinterop.ExperimentalForeignApi -import kotlinx.cinterop.cstr -import kotlinx.cinterop.memScoped -import kotlinx.cinterop.toKString +import kotlinx.cinterop.* import kotlinx.io.IOException import platform.Foundation.* import platform.posix.* diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 49512da53..57cf0a1cc 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -3,6 +3,7 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENCE file. */ +import org.gradle.internal.os.OperatingSystem import org.jetbrains.dokka.gradle.DokkaTaskPartial plugins { @@ -30,6 +31,17 @@ kotlin { } } } + wasmWasi { + nodejs { + testTask { + // fd_readdir is unsupported on Windows: + // https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19 + if (OperatingSystem.current().isWindows) { + filter.setExcludePatterns("*SmokeFileTest.listDirectory") + } + } + } + } sourceSets { commonMain.dependencies { diff --git a/core/common/src/files/FileSystem.kt b/core/common/src/files/FileSystem.kt index 1262f2691..de000d447 100644 --- a/core/common/src/files/FileSystem.kt +++ b/core/common/src/files/FileSystem.kt @@ -145,6 +145,25 @@ public sealed interface FileSystem { * @throws FileNotFoundException if there is no file or directory corresponding to the specified path. */ public fun resolve(path: Path): Path + + /** + * Returns paths corresponding to [directory]'s immediate children. + * + * There are no guarantees on children paths order within a returned collection. + * + * If path [directory] was an absolute path, a returned collection will also contain absolute paths. + * If it was a relative path, a returned collection will contain relative paths. + * + * *For `wasmWasi` target, function does not work with NodeJS runtime on Windows, + * as `fd_readdir` function is [not implemented there](https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19).* + * + * @param directory a directory to list. + * @return a collection of [directory]'s immediate children. + * @throws FileNotFoundException if [directory] does not exist. + * @throws IOException if [directory] points to something other than directory. + * @throws IOException if there was an underlying error preventing listing [directory] children. + */ + public fun list(directory: Path): Collection } internal abstract class SystemFileSystemImpl : FileSystem diff --git a/core/common/test/files/SmokeFileTest.kt b/core/common/test/files/SmokeFileTest.kt index 997095b6f..b8d25b793 100644 --- a/core/common/test/files/SmokeFileTest.kt +++ b/core/common/test/files/SmokeFileTest.kt @@ -443,6 +443,36 @@ class SmokeFileTest { source.close() // there should be no error } + @Test + fun listDirectory() { + assertFailsWith { SystemFileSystem.list(createTempPath()) } + + val tmpFile = createTempPath().also { + SystemFileSystem.sink(it).close() + } + assertFailsWith { SystemFileSystem.list(tmpFile) } + + val dir = createTempPath().also { + SystemFileSystem.createDirectories(it) + } + assertEquals(emptyList(), SystemFileSystem.list(dir)) + + val subdir = Path(dir, "subdir").also { + SystemFileSystem.createDirectories(it) + SystemFileSystem.sink(Path(it, "file")).close() + } + assertEquals(listOf(subdir), SystemFileSystem.list(dir)) + + val file = Path(dir, "file").also { + SystemFileSystem.sink(it).close() + } + assertEquals(setOf(file, subdir), SystemFileSystem.list(dir).toSet()) + + SystemFileSystem.delete(file) + SystemFileSystem.delete(Path(subdir, "file")) + SystemFileSystem.delete(subdir) + } + private fun constructAbsolutePath(vararg parts: String): String { return SystemPathSeparator.toString() + parts.joinToString(SystemPathSeparator.toString()) } diff --git a/core/jvm/src/files/FileSystemJvm.kt b/core/jvm/src/files/FileSystemJvm.kt index 24d7e0674..b85ebdb7e 100644 --- a/core/jvm/src/files/FileSystemJvm.kt +++ b/core/jvm/src/files/FileSystemJvm.kt @@ -96,6 +96,17 @@ public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl() if (!path.file.exists()) throw FileNotFoundException(path.file.absolutePath) return Path(path.file.canonicalFile) } + + override fun list(directory: Path): Collection { + val file = directory.file + if (!file.exists()) throw FileNotFoundException(file.absolutePath) + if (!file.isDirectory) throw IOException("Not a directory: ${file.absolutePath}") + return buildList { + file.list()?.forEach { childName -> + add(Path(directory, childName)) + } + } + } } @JvmField diff --git a/core/native/src/files/FileSystemNative.kt b/core/native/src/files/FileSystemNative.kt index 05b4795b1..79d7aafd3 100644 --- a/core/native/src/files/FileSystemNative.kt +++ b/core/native/src/files/FileSystemNative.kt @@ -12,7 +12,7 @@ import kotlinx.io.RawSource import platform.posix.* import kotlin.experimental.ExperimentalNativeApi -@OptIn(ExperimentalForeignApi::class) +@OptIn(ExperimentalForeignApi::class, ExperimentalStdlibApi::class) public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl() { override fun exists(path: Path): Boolean { return access(path.path, F_OK) == 0 @@ -86,6 +86,22 @@ public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl() ?: throw IOException("Failed to open $path with ${strerror(errno)?.toKString()}") return FileSink(openFile) } + + override fun list(directory: Path): Collection { + val metadata = metadataOrNull(directory) ?: throw FileNotFoundException(directory.path) + if (!metadata.isDirectory) throw IOException("Not a directory: ${directory.path}") + return buildList { + opendir(directory.path).use { + var child = it.readdir() + while (child != null) { + if (child != "." && child != "..") { + add(Path(directory, child)) + } + child = it.readdir() + } + } + } + } } internal expect fun metadataOrNullImpl(path: Path): FileMetadata? @@ -105,3 +121,10 @@ internal const val PermissionAllowAll: UShort = 511u @OptIn(ExperimentalNativeApi::class) internal actual val isWindows: Boolean = Platform.osFamily == OsFamily.WINDOWS + +@OptIn(ExperimentalStdlibApi::class) +internal expect class OpaqueDirEntry : AutoCloseable { + fun readdir(): String? +} + +internal expect fun opendir(path: String): OpaqueDirEntry diff --git a/core/nativeNonAndroid/src/files/FileSystemNativeNonAndroid.kt b/core/nativeNonAndroid/src/files/FileSystemNativeNonAndroid.kt new file mode 100644 index 000000000..d15a19139 --- /dev/null +++ b/core/nativeNonAndroid/src/files/FileSystemNativeNonAndroid.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package kotlinx.io.files + +import kotlinx.cinterop.CPointer +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.cinterop.get +import kotlinx.cinterop.toKString +import kotlinx.io.IOException +import platform.posix.DIR +import platform.posix.closedir +import platform.posix.errno +import platform.posix.strerror + +@OptIn(ExperimentalForeignApi::class, ExperimentalStdlibApi::class) +internal actual class OpaqueDirEntry constructor(private val dir: CPointer) : AutoCloseable { + actual fun readdir(): String? { + val entry = platform.posix.readdir(dir) ?: return null + return entry[0].d_name.toKString() + } + + override fun close() { + closedir(dir) + } +} + +@OptIn(ExperimentalForeignApi::class) +internal actual fun opendir(path: String): OpaqueDirEntry { + val dirent = platform.posix.opendir(path) + if (dirent != null) return OpaqueDirEntry(dirent) + throw IOException("Can't open directory $path: ${strerror(errno)?.toKString() ?: "reason unknown"}") +} diff --git a/core/nonApple/src/files/FileSystemNonApple.kt b/core/nativeNonApple/src/files/FileSystemNativeNonApple.kt similarity index 100% rename from core/nonApple/src/files/FileSystemNonApple.kt rename to core/nativeNonApple/src/files/FileSystemNativeNonApple.kt diff --git a/core/nodeFilesystemShared/src/files/FileSystemNodeJs.kt b/core/nodeFilesystemShared/src/files/FileSystemNodeJs.kt index df9bac572..810b5ba02 100644 --- a/core/nodeFilesystemShared/src/files/FileSystemNodeJs.kt +++ b/core/nodeFilesystemShared/src/files/FileSystemNodeJs.kt @@ -100,6 +100,23 @@ public actual val SystemFileSystem: FileSystem = object : SystemFileSystemImpl() if (!exists(path)) throw FileNotFoundException(path.path) return Path(fs.realpathSync.native(path.path)) } + + override fun list(directory: Path): Collection { + val metadata = metadataOrNull(directory) ?: throw FileNotFoundException(directory.path) + if (!metadata.isDirectory) throw IOException("Not a directory: ${directory.path}") + val dir = fs.opendirSync(directory.path) ?: throw IOException("Unable to read directory: ${directory.path}") + try { + return buildList { + var child = dir.readSync() + while (child != null) { + add(Path(directory, child.name)) + child = dir.readSync() + } + } + } finally { + dir.closeSync() + } + } } public actual val SystemTemporaryDirectory: Path diff --git a/core/nodeFilesystemShared/src/node/fs.kt b/core/nodeFilesystemShared/src/node/fs.kt index 84c9ba989..429f5d6d6 100644 --- a/core/nodeFilesystemShared/src/node/fs.kt +++ b/core/nodeFilesystemShared/src/node/fs.kt @@ -56,6 +56,11 @@ internal external interface Fs { */ fun writeFileSync(fd: Int, buffer: Buffer) + /** + * See https://nodejs.org/api/fs.html#fsopendirsyncpath-options + */ + fun opendirSync(path: String): Dir? + val realpathSync: realpathSync val constants: constants @@ -86,4 +91,20 @@ internal external interface realpathSync { fun native(path: String): String } +/** + * See https://nodejs.org/api/fs.html#class-fsdir + */ +internal external interface Dir { + fun closeSync() + + fun readSync(): Dirent? +} + +/** + * See https://nodejs.org/api/fs.html#class-fsdirent + */ +internal external interface Dirent { + val name: String +} + internal expect val fs: Fs diff --git a/core/wasmWasi/src/-WasmUtils.kt b/core/wasmWasi/src/-WasmUtils.kt index 647e38e49..6ed622d09 100644 --- a/core/wasmWasi/src/-WasmUtils.kt +++ b/core/wasmWasi/src/-WasmUtils.kt @@ -109,3 +109,14 @@ internal fun Pointer.allocateString(value: String): Int { */ @UnsafeWasmMemoryApi internal fun MemoryAllocator.allocateInt(): Pointer = allocate(Int.SIZE_BYTES) + +/** + * Decodes zero-terminated string from a sequence of bytes that should not exceed [maxLength] bytes in length. + */ +@UnsafeWasmMemoryApi +internal fun Pointer.loadString(maxLength: Int): String { + val bytes = loadBytes(maxLength) + val firstZeroByte = bytes.indexOf(0) + val length = if (firstZeroByte == -1) maxLength else firstZeroByte + return bytes.decodeToString(0, length) +} diff --git a/core/wasmWasi/src/files/FileSystemWasm.kt b/core/wasmWasi/src/files/FileSystemWasm.kt index 450322dc8..20e8069f3 100644 --- a/core/wasmWasi/src/files/FileSystemWasm.kt +++ b/core/wasmWasi/src/files/FileSystemWasm.kt @@ -93,8 +93,10 @@ internal object WasiFileSystem : SystemFileSystemImpl() { if (res == Errno.success) { created = true } else if (res != Errno.exist) { - throw IOException("Can't create directory $path. " + - "Creation of an intermediate directory $segment failed: ${res.description}") + throw IOException( + "Can't create directory $path. " + + "Creation of an intermediate directory $segment failed: ${res.description}" + ) } } if (mustCreate && !created) throw IOException("Directory already exists: $path") @@ -120,8 +122,12 @@ internal object WasiFileSystem : SystemFileSystemImpl() { val res = Errno( path_rename( - oldFd = sourcePreOpen.fd, oldPathPtr = sourceBuffer.address.toInt(), oldPathLen = sourceBufferLength, - newFd = destPreOpen.fd, newPathPtr = destBuffer.address.toInt(), newPathLen = destBufferLength + oldFd = sourcePreOpen.fd, + oldPathPtr = sourceBuffer.address.toInt(), + oldPathLen = sourceBufferLength, + newFd = destPreOpen.fd, + newPathPtr = destBuffer.address.toInt(), + newPathLen = destBufferLength ) ) when (res) { @@ -270,6 +276,77 @@ internal object WasiFileSystem : SystemFileSystemImpl() { throw IOException("Can't create symbolic link $target pointing to $linked: ${res.description}") } } + + override fun list(directory: Path): Collection { + val preOpen = PreOpens.findPreopen(directory) + + val metadata = metadataOrNullInternal(preOpen.fd, directory, true) + ?: throw FileNotFoundException(directory.path) + if (metadata.filetype != FileType.directory) throw IOException("Not a directory: ${directory.path}") + + val children = mutableListOf() + val dir_fd = withScopedMemoryAllocator { allocator -> + val fdPtr = allocator.allocateInt() + val (stringBuffer, stringBufferLength) = allocator.storeString(directory.path) + + val res = Errno( + path_open( + fd = preOpen.fd, + dirflags = listOf(LookupFlags.symlink_follow).toBitset(), + pathPtr = stringBuffer.address.toInt(), pathLen = stringBufferLength, + oflags = setOf(OpenFlags.directory).toBitset(), + fsRightsBase = listOf(Rights.fd_readdir, Rights.fd_read).toBitset(), + fsRightsInheriting = 0, + fdFlags = 0, + resultPtr = fdPtr.address.toInt() + ) + ) + if (res != Errno.success) throw IOException("Can't open directory ${directory.path}: ${res.description}") + fdPtr.loadInt() + } + try { + withScopedMemoryAllocator { allocator -> + val resultSizePtr = allocator.allocateInt() + // directory's filesize expected to be larger than the actual buffer size required to fit all entries + val bufferSize = metadata.filesize.toInt() + val buffer = allocator.allocate(bufferSize) + val resultSize: Int + // Unsuported on Windows and Android: + // https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19 + val res = Errno( + fd_readdir( + fd = dir_fd, + bufPtr = buffer.address.toInt(), + bufLen = bufferSize, + cookie = 0L, + resultPtr = resultSizePtr.address.toInt() + ) + ) + if (res != Errno.success) { + throw IOException("Can't read directory ${directory.path}: ${res.description}") + } + resultSize = resultSizePtr.loadInt() + check(resultSize <= bufferSize) { "Result size: $resultSize, buffer size: $bufferSize" } + var entryPtr = buffer + val endPtr = entryPtr + resultSize + while (entryPtr.address < endPtr.address) { + // read dirent: https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md#-dirent-record + // Each entry is 24-byte-wide dirent with filename length at offset 16, followed by + // filename length bytes of data. + val entryLen = entryPtr.loadInt(16) + entryPtr += 24 + val name = entryPtr.loadString(entryLen) + entryPtr += entryLen + if (name != "." && name != "..") { + children.add(Path(directory, name)) + } + } + } + return children + } finally { + fd_close(dir_fd) + } + } } private fun Path.normalized(): Path { @@ -332,7 +409,8 @@ internal object PreOpens { } internal fun findPreopen(path: Path): PreOpen { - return findPreopenOrNull(path) ?: throw IOException("Path does not belong to any preopened directory: $path") + return findPreopenOrNull(path) + ?: throw IOException("Path does not belong to any preopened directory: $path") } @OptIn(UnsafeWasmMemoryApi::class) diff --git a/core/wasmWasi/src/wasi/functions.kt b/core/wasmWasi/src/wasi/functions.kt index a6153666f..b4176801d 100644 --- a/core/wasmWasi/src/wasi/functions.kt +++ b/core/wasmWasi/src/wasi/functions.kt @@ -23,9 +23,6 @@ internal external fun fd_read(fd: Fd, iovecPtr: Int, iovecLen: Int, resultPtr: I @WasmImport("wasi_snapshot_preview1", "fd_readdir") internal external fun fd_readdir(fd: Fd, bufPtr: Int, bufLen: Int, cookie: Long, resultPtr: Int): Int -@WasmImport("wasi_snapshot_preview1", "fd_datasync") -internal external fun fd_datasync(fd: Fd): Int - @WasmImport("wasi_snapshot_preview1", "fd_sync") internal external fun fd_sync(fd: Fd): Int diff --git a/core/wasmWasi/test/WasiFsTest.kt b/core/wasmWasi/test/WasiFsTest.kt index 16fd2da61..f712b6198 100644 --- a/core/wasmWasi/test/WasiFsTest.kt +++ b/core/wasmWasi/test/WasiFsTest.kt @@ -219,4 +219,14 @@ class WasiFsTest { SystemFileSystem.delete(Path("/tmp/a")) } } + + // https://github.com/nodejs/node/blob/6f4d6011ea1b448cf21f5d363c44e4a4c56ca34c/deps/uvwasi/src/uvwasi.c#L19 + @Test + fun readdirUnsupportedOnWindows() { + if (!isWindows) return + + assertFailsWith { + SystemFileSystem.list(PreOpens.roots.first()) + } + } }