From 435dd96d561887865fcad54f909fd8933762ab04 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Fri, 8 Aug 2025 09:58:51 +0200 Subject: [PATCH 01/13] Implement model and SMIL parsing for Guided Navigation --- .../shared/guided/GuidedNavigationDocument.kt | 61 ++++++ .../r2/shared/guided/GuidedNavigationRole.kt | 36 ++++ .../services/GuidedNavigationService.kt | 50 +++++ .../r2/streamer/parser/epub/Constants.kt | 2 +- .../r2/streamer/parser/epub/EpubParser.kt | 7 +- .../streamer/parser/epub/ManifestAdapter.kt | 13 +- .../parser/epub/MediaOverlaysService.kt | 203 ++++++++++++++++++ .../streamer/parser/epub/ResourceAdapter.kt | 7 +- .../r2/streamer/parser/epub/SmilParser.kt | 87 -------- .../parser/epub/PackageDocumentTest.kt | 2 +- 10 files changed, 372 insertions(+), 96 deletions(-) create mode 100644 readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt create mode 100644 readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt create mode 100644 readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt create mode 100644 readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt delete mode 100644 readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt new file mode 100644 index 0000000000..7e23ac1a5f --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.guided + +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.Url + +public data class GuidedNavigationDocument( + val links: List, + val guided: List, +) + +public sealed interface GuidedNavigationObject { + public val roles: Set +} + +public data class GuidedNavigationLeaf( + val text: String?, + val refs: Set, + override val roles: Set, +) : GuidedNavigationObject + +public data class GuidedNavigationContainer( + val children: List, + override val roles: Set, +) : GuidedNavigationObject + +@JvmInline +public value class SsmlString(public val value: String) + +@ConsistentCopyVisibility +public data class GuidedNavigationText private constructor( + val plain: String?, + val ssml: SsmlString?, + val language: Language?, +) { + init { + require(plain != null || ssml != null) + require(plain == null || plain.isNotEmpty()) + require(ssml == null || ssml.value.isNotEmpty()) + } +} + +public sealed interface GuidedNavigationRef + +public data class GuidedNavigationTextRef( + val url: Url, +) : GuidedNavigationRef + +public data class GuidedNavigationImageRef( + val url: Url, +) : GuidedNavigationRef + +public data class GuidedNavigationAudioRef( + val url: Url, +) : GuidedNavigationRef diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt new file mode 100644 index 0000000000..14dc7a8945 --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.guided + +@JvmInline +public value class GuidedNavigationRole(public val value: String) { + + public companion object { + + /** + * Inherited from HTML and/or ARIA + */ + public val ASIDE: GuidedNavigationRole = GuidedNavigationRole("aside") + public val CELL: GuidedNavigationRole = GuidedNavigationRole("cell") + public val DEFINITION: GuidedNavigationRole = GuidedNavigationRole("definition") + public val FIGURE: GuidedNavigationRole = GuidedNavigationRole("figure") + public val LIST: GuidedNavigationRole = GuidedNavigationRole("list") + public val LIST_ITEM: GuidedNavigationRole = GuidedNavigationRole("listItem") + public val ROW: GuidedNavigationRole = GuidedNavigationRole("row") + public val TABLE: GuidedNavigationRole = GuidedNavigationRole("table") + public val TERM: GuidedNavigationRole = GuidedNavigationRole("term") + + /** + * Inherited from EPUB 3 Structural Semantics Vocabulary 1.1 + */ + public val LANDMARKS: GuidedNavigationRole = GuidedNavigationRole("landmarks") + public val LOA: GuidedNavigationRole = GuidedNavigationRole("loa") + public val LOI: GuidedNavigationRole = GuidedNavigationRole("loi") + public val LOT: GuidedNavigationRole = GuidedNavigationRole("lot") + public val LOV: GuidedNavigationRole = GuidedNavigationRole("lov") + } +} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt new file mode 100644 index 0000000000..2d723b7e73 --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.publication.services + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.PublicationServicesHolder +import org.readium.r2.shared.util.Closeable +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.data.ReadError + +@ExperimentalReadiumApi +public interface GuidedNavigationService : Publication.Service { + + public fun iterator(): GuidedNavigationIterator +} + +@ExperimentalReadiumApi +public interface GuidedNavigationIterator : Closeable { + + /** + * Prepares an element for retrieval by the invocation of next. + * + * Does nothing if the the end has been reached. + */ + public suspend operator fun hasNext(): Boolean + + /** + * Retrieves the next guided navigation document, prepared by the preceding call to [hasNext], + * or throws an IllegalStateException if hasNext was not invoked. + */ + public suspend operator fun next(): Try + + /** + * Closes any resources allocated by the iterator. + */ + override fun close() {} +} + +@ExperimentalReadiumApi +public val PublicationServicesHolder.guidedNavigationService: GuidedNavigationService? + get() { + findService(GuidedNavigationService::class)?.let { return it } + return null + } diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt index 49ba1cd52b..f82a4c2d72 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt @@ -27,7 +27,7 @@ internal object Vocabularies { const val MEDIA = "http://www.idpf.org/epub/vocab/overlays/#" const val RENDITION = "http://www.idpf.org/vocab/rendition/#" - const val TYPE = "http://idpf.org/epub/vocab/structure/#" // this is a guessed value + const val TYPE = "http://idpf.org/epub/vocab/structure/#" const val DCTERMS = "http://purl.org/dc/terms/" const val A11Y = "http://www.idpf.org/epub/vocab/package/a11y/#" diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt index 86c8c296e5..6e502fa89d 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt @@ -14,6 +14,7 @@ import org.readium.r2.shared.publication.Link import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.encryption.Encryption import org.readium.r2.shared.publication.epub.EpubEncryptionParser +import org.readium.r2.shared.publication.services.GuidedNavigationService import org.readium.r2.shared.publication.services.content.DefaultContentService import org.readium.r2.shared.publication.services.content.iterators.HtmlResourceContentIterator import org.readium.r2.shared.publication.services.search.StringSearchService @@ -86,13 +87,15 @@ public class EpubParser( val encryptionData = parseEncryptionData(asset.container) - val manifest = ManifestAdapter( + val (manifest, mediaOverlays) = ManifestAdapter( packageDocument = packageDocument, navigationData = parseNavigationData(packageDocument, asset.container), encryptionData = encryptionData, displayOptions = parseDisplayOptions(asset.container) ).adapt() + val smils = mediaOverlays.map { it.url() } + var container = asset.container manifest.metadata.identifier?.let { id -> val deobfuscator = EpubDeobfuscator(id, encryptionData) @@ -110,7 +113,7 @@ public class EpubParser( HtmlResourceContentIterator.Factory() ) ) - ) + ).also { it[GuidedNavigationService::class] = MediaOverlaysService.createFactory(smils) } ) return Try.success(builder) diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt index 61a979498c..c50e38344a 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt @@ -24,10 +24,15 @@ internal class ManifestAdapter( private val encryptionData: Map = emptyMap(), private val displayOptions: Map = emptyMap(), ) { + + data class Result( + val manifest: Manifest, + val mediaOverlays: List, + ) private val epubVersion = packageDocument.epubVersion private val spine = packageDocument.spine - fun adapt(): Manifest { + fun adapt(): Result { // Compute metadata val metadata = MetadataAdapter( epubVersion, @@ -37,7 +42,7 @@ internal class ManifestAdapter( ).adapt(packageDocument.metadata) // Compute links - val (readingOrder, resources) = ResourceAdapter( + val (readingOrder, resources, mediaOverlays) = ResourceAdapter( packageDocument.spine, packageDocument.manifest, encryptionData, @@ -70,7 +75,7 @@ internal class ManifestAdapter( } // Build Publication object - return Manifest( + val manifest = Manifest( metadata = metadata.metadata, links = emptyList(), readingOrder = readingOrder, @@ -78,5 +83,7 @@ internal class ManifestAdapter( tableOfContents = toc, subcollections = subcollections.toMap() ) + + return Result(manifest = manifest, mediaOverlays = mediaOverlays) } } diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt new file mode 100644 index 0000000000..adb8e57565 --- /dev/null +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt @@ -0,0 +1,203 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class) + +package org.readium.r2.streamer.parser.epub + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationContainer +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.guided.GuidedNavigationLeaf +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.data.Container +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.data.decodeXml +import org.readium.r2.shared.util.data.readDecodeOrElse +import org.readium.r2.shared.util.fromEpubHref +import org.readium.r2.shared.util.resource.Resource +import org.readium.r2.shared.util.use +import org.readium.r2.shared.util.xml.ElementNode + +@ExperimentalReadiumApi +public class MediaOverlaysService( + private val smils: List, + private val container: Container, +) : GuidedNavigationService { + + override fun iterator(): GuidedNavigationIterator { + return MediaOverlaysIterator(smils, container) + } + + public companion object { + + public fun createFactory( + smils: List, + ): ( + Publication.Service.Context, + ) -> MediaOverlaysService = + { context -> + MediaOverlaysService( + smils = smils, + container = context.container + ) + } + } +} + +@OptIn(ExperimentalReadiumApi::class) +internal class MediaOverlaysIterator( + private val smils: List, + private val container: Container, +) : GuidedNavigationIterator { + + private var lastRead: Int = -1 + private var nextDoc: Try? = null + + override suspend fun hasNext(): Boolean { + if (lastRead >= smils.size - 1) { + return false + } + + lastRead++ + nextDoc = parse(smils[lastRead]) + return true + } + + private suspend fun parse(href: Url): Try { + val resource = container[href] + ?: throw IllegalStateException("Cannot find resource in the container: $href") + + val xmlDoc = resource.use { res -> + res.readDecodeOrElse( + decode = { it.decodeXml() }, + recover = { return Try.failure(it) } + ) + } + + return SmilParser.parse(xmlDoc, href) + ?.let { Try.success(it) } + ?: Try.failure(ReadError.Decoding("Cannot parse SMIL file $href")) + } + + override suspend fun next(): Try { + return nextDoc ?: throw IllegalStateException("next was called before hasNext.") + } +} + +internal object SmilParser { + + fun parse(document: ElementNode, fileHref: Url): GuidedNavigationDocument? { + val docPrefixes = document.getAttrNs("prefix", Namespaces.OPS) + ?.let { parsePrefixes(it) }.orEmpty() + val prefixMap = CONTENT_RESERVED_PREFIXES + docPrefixes // prefix element overrides reserved prefixes + val body = document.getFirst("body", Namespaces.SMIL) ?: return null + return parseSeq(body, fileHref, prefixMap) + ?.let { GuidedNavigationDocument(links = emptyList(), guided = it.children) } + } + + private fun parseSeq( + node: ElementNode, + filePath: Url, + prefixMap: Map, + ): GuidedNavigationContainer? { + val roles = parseRoles(node, prefixMap) + val children: MutableList = mutableListOf() + for (child in node.getAll()) { + if (child.name == "par" && child.namespace == Namespaces.SMIL) { + parsePar(child, filePath, prefixMap)?.let { children.add(it) } + } else if (child.name == "seq" && child.namespace == Namespaces.SMIL) { + parseSeq(child, filePath, prefixMap)?.let { children.add(it) } + } + } + + return GuidedNavigationContainer(children = children, roles = roles) + } + + private fun parsePar( + node: ElementNode, + filePath: Url, + prefixMap: Map, + ): GuidedNavigationObject? { + val roles = parseRoles(node, prefixMap) + val text = node.getFirst("text", Namespaces.SMIL) + ?.getAttr("src") + ?.let { Url.fromEpubHref(it) } + ?: return null + val audio = node.getFirst("audio", Namespaces.SMIL) + ?.let { audioNode -> + val src = audioNode.getAttr("src") + ?.let { Url.fromEpubHref(it) } + ?.toString() + ?: return null + val begin = audioNode.getAttr("clipBegin") + ?.let { ClockValueParser.parse(it) } + ?: "" + val end = audioNode.getAttr("clipEnd") + ?.let { ClockValueParser.parse(it) } + ?: "" + Url("$src#t=$begin,$end") + } + + val refs = setOfNotNull( + GuidedNavigationTextRef(filePath.resolve(text)), + audio?.let { GuidedNavigationAudioRef(filePath.resolve(it)) } + ) + + return GuidedNavigationLeaf( + text = null, + refs = refs, + roles = roles + ) + } + + private fun parseRoles( + node: ElementNode, + prefixMap: Map, + ): Set { + val typeAttr = node.getAttrNs("type", Namespaces.OPS) ?: "" + val candidates = if (typeAttr.isNotEmpty()) { + parseProperties(typeAttr).map { + resolveProperty( + it, + prefixMap, + DEFAULT_VOCAB.TYPE + ) + }.toSet() + } else { + emptySet() + } + + return candidates.mapNotNull { + when (it.removePrefix(Vocabularies.TYPE)) { + "aside" -> GuidedNavigationRole.ASIDE + "table-cell" -> GuidedNavigationRole.CELL + "glossdef" -> GuidedNavigationRole.DEFINITION + "figure" -> GuidedNavigationRole.FIGURE + "list" -> GuidedNavigationRole.LIST + "list-item" -> GuidedNavigationRole.LIST_ITEM + "table-row" -> GuidedNavigationRole.ROW + "table" -> GuidedNavigationRole.TABLE + "glossterm" -> GuidedNavigationRole.TERM + "landmarks" -> GuidedNavigationRole.LANDMARKS + "loa" -> GuidedNavigationRole.LOA + "loi" -> GuidedNavigationRole.LOI + "lot" -> GuidedNavigationRole.LOT + "lov" -> GuidedNavigationRole.LOV + else -> null + } + }.toSet() + } +} diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt index 16af59e154..2f352a49b0 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt @@ -27,6 +27,7 @@ internal class ResourceAdapter( data class Links( val readingOrder: List, val resources: List, + val mediaOverlays: List, ) @Suppress("Unchecked_cast") @@ -47,9 +48,11 @@ internal class ResourceAdapter( } } val readingOrderAllIds = computeIdsWithFallbacks(readingOrderIds) - val resourceItems = manifest.filterNot { it.id in readingOrderAllIds } + val smilIds = manifest.mapNotNull { it.mediaOverlay } + val smils = smilIds.mapNotNull { id -> itemById[id]?.let { item -> computeLink(item) } } + val resourceItems = manifest.filterNot { it.id in readingOrderAllIds || it.id in smilIds } val resources = resourceItems.map { computeLink(it) } - return Links(readingOrder, resources) + return Links(readingOrder, resources, smils) } /** Recursively find the ids contained in fallback chains of items with [ids]. */ diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt deleted file mode 100644 index 0f43398f44..0000000000 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(InternalReadiumApi::class) - -package org.readium.r2.streamer.parser.epub - -import org.readium.r2.shared.DelicateReadiumApi -import org.readium.r2.shared.InternalReadiumApi -import org.readium.r2.shared.MediaOverlayNode -import org.readium.r2.shared.MediaOverlays -import org.readium.r2.shared.util.Url -import org.readium.r2.shared.util.fromEpubHref -import org.readium.r2.shared.util.xml.ElementNode - -internal object SmilParser { - /* According to https://www.w3.org/publishing/epub3/epub-mediaoverlays.html#sec-overlays-content-conf - a Media Overlay Document MAY refer to more than one EPUB Content Document - This might be possible only using Canonical Fragment Identifiers - since the unique body and each seq element MUST reference - one EPUB Content Document by means of its attribute epub:textref - */ - - fun parse(document: ElementNode, filePath: Url): MediaOverlays? { - val body = document.getFirst("body", Namespaces.SMIL) ?: return null - return parseSeq(body, filePath)?.let { MediaOverlays(it) } - } - - @OptIn(DelicateReadiumApi::class) - private fun parseSeq(node: ElementNode, filePath: Url): List? { - val children: MutableList = mutableListOf() - for (child in node.getAll()) { - if (child.name == "par" && child.namespace == Namespaces.SMIL) { - parsePar(child, filePath)?.let { children.add(it) } - } else if (child.name == "seq" && child.namespace == Namespaces.SMIL) { - parseSeq(child, filePath)?.let { children.addAll(it) } - } - } - - /* No wrapping media overlay can be created unless: - - all child media overlays reference the same audio file - - the seq element has an textref attribute (this is mandatory according to the EPUB spec) - */ - val textref = node.getAttrNs("textref", Namespaces.OPS) - ?.let { Url.fromEpubHref(it) } - val audioFiles = children.mapNotNull(MediaOverlayNode::audioFile) - return if (textref != null && audioFiles.distinct().size == 1) { // hierarchy - val normalizedTextref = filePath.resolve(textref) - listOf(mediaOverlayFromChildren(normalizedTextref, children)) - } else { - children - } - } - - private fun parsePar(node: ElementNode, filePath: Url): MediaOverlayNode? { - val text = node.getFirst("text", Namespaces.SMIL) - ?.getAttr("src") - ?.let { Url.fromEpubHref(it) } - ?: return null - val audio = node.getFirst("audio", Namespaces.SMIL) - ?.let { audioNode -> - val src = audioNode.getAttr("src") - val begin = audioNode.getAttr("clipBegin")?.let { ClockValueParser.parse(it) } ?: "" - val end = audioNode.getAttr("clipEnd")?.let { ClockValueParser.parse(it) } ?: "" - "$src#t=$begin,$end" - } - ?.let { Url.fromEpubHref(it) } - - return MediaOverlayNode( - filePath.resolve(text), - audio?.let { filePath.resolve(audio) } - ) - } - - private fun mediaOverlayFromChildren(text: Url, children: List): MediaOverlayNode { - require(children.isNotEmpty() && children.mapNotNull { it.audioFile }.distinct().size <= 1) - val audioChildren = children.mapNotNull { if (it.audioFile != null) it else null } - val file = audioChildren.first().audioFile - val start = audioChildren.first().clip.start ?: "" - val end = audioChildren.last().clip.end ?: "" - val audio = Url.fromEpubHref("$file#t=$start,$end") - return MediaOverlayNode(text, audio, children, listOf("section")) - } -} diff --git a/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt index 9a70986678..d759c6ed70 100644 --- a/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt +++ b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt @@ -37,7 +37,7 @@ fun parsePackageDocument(path: String): Manifest { ?.let { ManifestAdapter(it) } ?.adapt() checkNotNull(pub) - return pub + return pub.manifest } const val PARSE_PUB_TIMEOUT = 1000L // milliseconds From 1fce006235b13ccc902b9a893b8fb542b9d07638 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Fri, 8 Aug 2025 10:31:06 +0200 Subject: [PATCH 02/13] Small fix --- .../org/readium/r2/shared/guided/GuidedNavigationDocument.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt index 7e23ac1a5f..a9bc686d58 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt @@ -20,7 +20,7 @@ public sealed interface GuidedNavigationObject { } public data class GuidedNavigationLeaf( - val text: String?, + val text: GuidedNavigationText?, val refs: Set, override val roles: Set, ) : GuidedNavigationObject From b80f11ad66fab24af5faa2bae76e6b8ddfb8f1e5 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Sun, 17 Aug 2025 11:19:03 +0200 Subject: [PATCH 03/13] Draft a ReadAloudNavigator --- .../exoplayer/audio/ExoPlayerDataSource.kt | 6 +- .../exoplayer/readaloud/build.gradle.kts | 30 ++++ .../exoplayer/readaloud/gradle.properties | 1 + .../readaloud/src/main/AndroidManifest.xml | 3 + .../exoplayer/readaloud/ExoPlayerEngine.kt | 149 ++++++++++++++++++ .../readaloud/ExoPlayerEngineProvider.kt | 30 ++++ .../navigator/media/common/MediaNavigator.kt | 5 + .../media/readaloud/build.gradle.kts | 25 +++ .../media/readaloud/gradle.properties | 1 + .../readaloud/src/main/AndroidManifest.xml | 3 + .../readaloud/GuidedNavigationAdapter.kt | 53 +++++++ .../media/readaloud/ReadAloudEngine.kt | 67 ++++++++ .../media/readaloud/ReadAloudModel.kt | 126 +++++++++++++++ .../media/readaloud/ReadAloudNavigator.kt | 123 +++++++++++++++ .../readaloud/ReadAloudNavigatorFactory.kt | 85 ++++++++++ .../media/readaloud/ReadAloudStateMachine.kt | 141 +++++++++++++++++ .../r2/shared/guided/GuidedNavigationRole.kt | 6 + .../r2/shared/util/TemporalFragmentParser.kt | 52 ++++++ .../shared/util/TemporalFragmentParserTest.kt | 52 ++++++ settings.gradle.kts | 8 + 20 files changed, 964 insertions(+), 2 deletions(-) create mode 100644 readium/adapters/exoplayer/readaloud/build.gradle.kts create mode 100644 readium/adapters/exoplayer/readaloud/gradle.properties create mode 100644 readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml create mode 100644 readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt create mode 100644 readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt create mode 100644 readium/navigators/media/readaloud/build.gradle.kts create mode 100644 readium/navigators/media/readaloud/gradle.properties create mode 100644 readium/navigators/media/readaloud/src/main/AndroidManifest.xml create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt create mode 100644 readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt create mode 100644 readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt diff --git a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt index bc7a5c81ba..09a6a98268 100644 --- a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt +++ b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt @@ -14,6 +14,7 @@ import androidx.media3.datasource.DataSource import androidx.media3.datasource.DataSpec import androidx.media3.datasource.TransferListener import kotlinx.coroutines.runBlocking +import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.DebugError import org.readium.r2.shared.util.data.ReadError @@ -28,11 +29,12 @@ import timber.log.Timber * An ExoPlayer's [DataSource] which retrieves resources from a [Publication]. */ @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class ExoPlayerDataSource internal constructor( +@InternalReadiumApi +public class ExoPlayerDataSource internal constructor( private val publication: Publication, ) : BaseDataSource(/* isNetwork = */ true) { - class Factory( + public class Factory( private val publication: Publication, private val transferListener: TransferListener? = null, ) : DataSource.Factory { diff --git a/readium/adapters/exoplayer/readaloud/build.gradle.kts b/readium/adapters/exoplayer/readaloud/build.gradle.kts new file mode 100644 index 0000000000..5cdeca3b20 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +plugins { + id("readium.library-conventions") + alias(libs.plugins.kotlin.serialization) +} + +android { + namespace = "org.readium.adapter.exoplayer.readaloud" +} + +dependencies { + api(project(":readium:readium-shared")) + api(project(":readium:navigators:media:readium-navigator-media-readaloud")) + + implementation(project(":readium:adapters:exoplayer:readium-adapter-exoplayer-audio")) + + implementation(libs.androidx.media3.common) + implementation(libs.androidx.media3.exoplayer) + implementation(libs.timber) + implementation(libs.kotlinx.coroutines.android) + implementation(libs.kotlinx.serialization.json) + + // Tests + testImplementation(libs.junit) +} diff --git a/readium/adapters/exoplayer/readaloud/gradle.properties b/readium/adapters/exoplayer/readaloud/gradle.properties new file mode 100644 index 0000000000..78bc7e9a97 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/gradle.properties @@ -0,0 +1 @@ +pom.artifactId=readium-adapter-exoplayer-readaloud diff --git a/readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml b/readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..2d10029868 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt new file mode 100644 index 0000000000..75719e386e --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -0,0 +1,149 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.adapter.exoplayer.readaloud + +import android.app.Application +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackParameters +import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSource +import androidx.media3.exoplayer.ExoPlaybackException +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import org.readium.navigator.media.readaloud.AudioEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.findInstance +import org.readium.r2.shared.util.ThrowableError +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.data.ReadException + +/** + * An [AudioEngine] based on Media3 ExoPlayer. + */ +@ExperimentalReadiumApi +@OptIn(ExperimentalCoroutinesApi::class) +@androidx.annotation.OptIn(UnstableApi::class) +public class ExoPlayerEngine private constructor( + private val exoPlayer: ExoPlayer, + private val listener: AudioEngine.Listener, +) : AudioEngine { + + public companion object { + + public operator fun invoke( + application: Application, + dataSourceFactory: DataSource.Factory, + listener: AudioEngine.Listener, + ): ExoPlayerEngine { + val exoPlayer = ExoPlayer.Builder(application) + .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) + .setAudioAttributes( + AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC) + .setUsage(C.USAGE_MEDIA) + .build(), + true + ) + .setHandleAudioBecomingNoisy(true) + .build() + + return ExoPlayerEngine(exoPlayer, listener) + } + } + + private inner class Listener : Player.Listener { + + override fun onPlaybackParametersChanged(playbackParameters: PlaybackParameters) { + } + + override fun onEvents(player: Player, events: Player.Events) { + if (events.contains(Player.EVENT_PLAYBACK_STATE_CHANGED) && player.playbackState == Player.STATE_ENDED) { + this@ExoPlayerEngine.listener.onPlaybackEnded() + } + } + } + + public sealed class Error( + override val message: String, + override val cause: org.readium.r2.shared.util.Error?, + ) : org.readium.r2.shared.util.Error { + + public data class Engine(override val cause: ThrowableError) : + Error("An error occurred in the ExoPlayer engine.", cause) + + public data class Source(override val cause: ReadError) : + Error("An error occurred while trying to read publication content.", cause) + } + + private val coroutineScope: CoroutineScope = + MainScope() + + private var prepareCalled: Boolean = false + + init { + exoPlayer.addListener(Listener()) + } + + override fun setPlaylist( + items: List, + ) { + val mediaItems = items.map { item -> + val clippingConfig = MediaItem.ClippingConfiguration.Builder() + .apply { + item.startOffset?.let { setStartPositionMs(it.inWholeMilliseconds) } + item.endOffset?.let { setEndPositionMs(it.inWholeMilliseconds) } + }.build() + MediaItem.Builder() + .setUri(item.href.toString()) + .setClippingConfiguration(clippingConfig) + .build() + } + exoPlayer.setMediaItems(mediaItems) + + if (!prepareCalled) { + exoPlayer.prepare() + prepareCalled = true + } + } + + override fun resume() { + exoPlayer.play() + } + + override fun pause() { + exoPlayer.pause() + } + + public fun close() { + coroutineScope.cancel() + exoPlayer.release() + } + + @OptIn(InternalReadiumApi::class) + private fun ExoPlaybackException.toError(): Error { + val readError = + if (type == ExoPlaybackException.TYPE_SOURCE) { + sourceException.findInstance(ReadException::class.java)?.error + } else { + null + } + + return if (readError == null) { + Error.Engine(ThrowableError(this)) + } else { + Error.Source(readError) + } + } +} diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt new file mode 100644 index 0000000000..da6ad0c66b --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.adapter.exoplayer.readaloud + +import android.app.Application +import org.readium.adapter.exoplayer.audio.ExoPlayerDataSource +import org.readium.navigator.media.readaloud.AudioEngine +import org.readium.navigator.media.readaloud.AudioEngineProvider +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.publication.Publication + +@OptIn(InternalReadiumApi::class) +@ExperimentalReadiumApi +public class ExoPlayerEngineProvider( + private val application: Application, +) : AudioEngineProvider { + + override fun createEngine( + publication: Publication, + listener: AudioEngine.Listener, + ): ExoPlayerEngine { + val dataSourceFactory = ExoPlayerDataSource.Factory(publication) + return ExoPlayerEngine(application, dataSourceFactory, listener) + } +} diff --git a/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt b/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt index 85353eebc8..8c05ad877b 100644 --- a/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt +++ b/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt @@ -59,6 +59,11 @@ public interface MediaNavigator< /** * State of the playback. */ + /* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ public interface Playback { /** diff --git a/readium/navigators/media/readaloud/build.gradle.kts b/readium/navigators/media/readaloud/build.gradle.kts new file mode 100644 index 0000000000..b1b021874c --- /dev/null +++ b/readium/navigators/media/readaloud/build.gradle.kts @@ -0,0 +1,25 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +plugins { + id("readium.library-conventions") + alias(libs.plugins.kotlin.serialization) +} + +android { + namespace = "org.readium.navigators.media.readaloud" +} + +dependencies { + api(project(":readium:navigators:media:readium-navigator-media-common")) + + implementation(libs.androidx.media3.common) + implementation(libs.androidx.media3.session) + + implementation(libs.timber) + implementation(libs.kotlinx.coroutines.android) + implementation(libs.kotlinx.serialization.json) +} diff --git a/readium/navigators/media/readaloud/gradle.properties b/readium/navigators/media/readaloud/gradle.properties new file mode 100644 index 0000000000..d944c2678c --- /dev/null +++ b/readium/navigators/media/readaloud/gradle.properties @@ -0,0 +1 @@ +pom.artifactId=readium-navigator-media-readaloud diff --git a/readium/navigators/media/readaloud/src/main/AndroidManifest.xml b/readium/navigators/media/readaloud/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..2d10029868 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt new file mode 100644 index 0000000000..aa5d115c13 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationContainer +import org.readium.r2.shared.guided.GuidedNavigationLeaf +import org.readium.r2.shared.guided.GuidedNavigationObject + +@OptIn(ExperimentalReadiumApi::class) +internal class GuidedNavigationAdapter { + + fun adapt(guidedNavTree: GuidedNavigationContainer): ReadAloudInnerNode { + val children = guidedNavTree.children.mapNotNull { adaptNode(it) } + val node = ReadAloudInnerNode(children, guidedNavTree.roles) + setParentInChildren(node) + return node + } + + private fun adaptNode(guidedNavigationObject: GuidedNavigationObject): ReadAloudNode? { + return when (guidedNavigationObject) { + is GuidedNavigationContainer -> + guidedNavigationObject.children + .mapNotNull { adaptNode(it) } + .takeIf { it.isNotEmpty() } + ?.let { ReadAloudInnerNode(it, guidedNavigationObject.roles) } + ?.also { setParentInChildren(it) } + is GuidedNavigationLeaf -> + adaptLeat(guidedNavigationObject) + } + } + + private fun adaptLeat(guidedNavigationLeaf: GuidedNavigationLeaf): ReadAloudLeafNode? { + return ReadAloudLeafNode( + text = guidedNavigationLeaf.text, + refs = guidedNavigationLeaf.refs, + roles = guidedNavigationLeaf.roles + ) + } + + private fun setParentInChildren(parent: ReadAloudInnerNode) { + parent.children.forEach { + when (it) { + is ReadAloudInnerNode -> it.parent = parent + is ReadAloudLeafNode -> it.parent = parent + } + } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt new file mode 100644 index 0000000000..dac3053ec3 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +public sealed interface ReadAloudEngine + +public interface AudioEngine : ReadAloudEngine { + + public interface Listener { + + public fun onItemChanged(index: Int) + + public fun onPlaybackEnded() + } + + public data class Item( + val href: Url, + val interval: TimeInterval?, + ) + + public fun setPlaylist(items: List) + + public fun pause() + + public fun resume() +} + +public interface AudioEngineProvider { + + public fun createEngine( + publication: Publication, + listener: AudioEngine.Listener, + ): AudioEngine +} + +@ExperimentalReadiumApi +public interface PlaybackEngine : ReadAloudEngine { + + public fun play(node: ReadAloudLeafNode) +} + +@ExperimentalReadiumApi +public interface TtsEngine : PlaybackEngine { + + public interface Voice { + + /** + * The languages supported by the voice. + */ + public val languages: Set + } + + /** + * Sets of voices available with this [PlaybackEngine]. + */ + public val voices: Set +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt new file mode 100644 index 0000000000..fdba9931e2 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRef +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationText + +// Modèle différent pour pouvoir garder une référence sur le noeud parent. +@ExperimentalReadiumApi +public sealed interface ReadAloudNode { + + public val roles: Set + + public val parent: ReadAloudNode? + + public val children: List +} + +@ExperimentalReadiumApi +public class ReadAloudInnerNode( + override val children: List, + override val roles: Set, +) : ReadAloudNode { + + override var parent: ReadAloudNode? = null + internal set +} + +@ExperimentalReadiumApi +public data class ReadAloudLeafNode( + val text: GuidedNavigationText?, + val refs: Set, + override val roles: Set, +) : ReadAloudNode { + + override val children: List = + emptyList() + + override lateinit var parent: ReadAloudInnerNode + internal set +} + +@ExperimentalReadiumApi +internal fun ReadAloudNode.isSkippable(): Boolean = + nearestSkippable() != null + +@ExperimentalReadiumApi +internal fun ReadAloudNode.isEscapable(): Boolean = + nearestEscapable() != null + +@ExperimentalReadiumApi +internal fun ReadAloudNode.firstLeaf(): ReadAloudLeafNode? = + when (this) { + is ReadAloudLeafNode -> this + is ReadAloudInnerNode -> children[0].firstLeaf() + } + +@ExperimentalReadiumApi +internal fun ReadAloudNode.nextLeaf(): ReadAloudLeafNode? { + if (children.isNotEmpty()) { + return children[0].nextLeaf() + } + + val siblings = parent?.children ?: return null + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + return currentIndex + .takeIf { it < siblings.size - 1 } + ?.let { siblings[currentIndex + 1].nextLeaf() } + ?: parent!!.skipToNext()?.nextLeaf() +} + +@ExperimentalReadiumApi +internal fun ReadAloudNode.skipToNext(): ReadAloudNode? { + val siblings = parent?.children ?: return null + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + return currentIndex + .takeIf { it < siblings.size - 1 } + ?.let { siblings[currentIndex + 1] } + ?: parent!!.skipToNext() +} + +@ExperimentalReadiumApi +internal fun ReadAloudNode.skipToPrevious(): ReadAloudNode? { + val siblings = parent?.children ?: return null + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + return currentIndex + .takeIf { it > 0 } + ?.let { siblings[currentIndex - 1] } + ?: parent!!.skipToPrevious() +} + +@ExperimentalReadiumApi +internal fun ReadAloudNode.escape(force: Boolean): ReadAloudNode? = + (nearestEscapable() ?: this.takeIf { force })?.skipToNext() + +@ExperimentalReadiumApi +internal fun ReadAloudNode.skip(force: Boolean): ReadAloudNode? = + (nearestSkippable() ?: this.takeIf { force })?.skipToNext() + +@ExperimentalReadiumApi +private fun ReadAloudNode.nearestEscapable(): ReadAloudNode? = + nearestOrNull { roles.any { it in GuidedNavigationRole.ESCAPABLE_ROLES } } + +@ExperimentalReadiumApi +private fun ReadAloudNode.nearestSkippable(): ReadAloudNode? = + nearestOrNull { roles.any { it in GuidedNavigationRole.SKIPPABLE_ROLES } } + +@ExperimentalReadiumApi +private fun ReadAloudNode.nearestOrNull( + predicate: (ReadAloudNode) -> Boolean, +): ReadAloudNode? = + when { + predicate(this) -> this + parent == null -> null + else -> parent!!.nearestOrNull(predicate) + } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt new file mode 100644 index 0000000000..1adac47545 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -0,0 +1,123 @@ + +package org.readium.navigator.media.readaloud + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.withContext +import org.readium.navigator.media.common.MediaNavigator +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationContainer +import org.readium.r2.shared.util.data.ReadError + +@ExperimentalReadiumApi +public class ReadAloudNavigator private constructor( + rootNode: ReadAloudInnerNode, + audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, +) { + + public companion object { + + public suspend operator fun invoke( + guidedNavigationTree: GuidedNavigationContainer, + audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, + ): ReadAloudNavigator = withContext(Dispatchers.Default) { + val tree = GuidedNavigationAdapter().adapt(guidedNavigationTree) + ReadAloudNavigator(tree, audioEngineFactory) + } + } + + public data class Playback( + val state: State, + val playWhenReady: Boolean, + ) + + public sealed interface State { + + public data object Ready : State, MediaNavigator.State.Ready + + public data object Ended : State, MediaNavigator.State.Ended + + public data class Failure(val error: Error) : State, MediaNavigator.State.Failure + } + + public sealed class Error( + override val message: String, + override val cause: org.readium.r2.shared.util.Error?, + ) : org.readium.r2.shared.util.Error { + + public data class EngineError(override val cause: org.readium.r2.shared.util.Error) : + Error("An error occurred in the playback engine.", cause) + + public data class ContentError(override val cause: ReadError) : + Error("An error occurred while trying to read publication content.", cause) + } + + private inner class AudioEngineListener : AudioEngine.Listener { + + override fun onItemChanged(index: Int) { + val state = stateMutable.value as ReadAloudStateMachine.State.Playing + val engineFood = state.engineFood as EngineFood.AudioEngineFood + nodeMutable.value = engineFood.nodes[index] + } + + override fun onPlaybackEnded() { + with(stateMachine) { + stateMutable.value = stateMutable.value.onAudioEngineEnded() + } + } + } + + private val audioEngine = audioEngineFactory(AudioEngineListener()) + + private val stateMachine = ReadAloudStateMachine(audioEngine) + + private val stateMutable = MutableStateFlow(stateMachine.start(rootNode, paused = false)) + + private val playbackMutable: MutableStateFlow = + MutableStateFlow(Playback(state = State.Ready, playWhenReady = true)) + + private val nodeMutable: MutableStateFlow = + MutableStateFlow(rootNode) + + public val playback: StateFlow = + playbackMutable.asStateFlow() + + public val node: StateFlow = + nodeMutable.asStateFlow() + + public fun play() { + with(stateMachine) { + stateMutable.value = stateMutable.value.resume() + } + } + + public fun pause() { + with(stateMachine) { + stateMutable.value = stateMutable.value.pause() + } + } + + public fun go(node: ReadAloudNode) { + with(stateMachine) { + stateMutable.value = stateMutable.value.jump(node) + } + } + + public fun canEscape(): Boolean = + nodeMutable.value.isEscapable() + + public fun canSkip(): Boolean = + nodeMutable.value.isSkippable() + + public fun escape(force: Boolean = true) { + nodeMutable.value.escape(force) + ?.let { go(it) } + } + + public fun skip(force: Boolean = true) { + nodeMutable.value.skip(force) + ?.let { go(it) } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt new file mode 100644 index 0000000000..a651888ede --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import android.app.Application +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationContainer +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.publication.services.guidedNavigationService +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.getOrElse + +@ExperimentalReadiumApi +public class ReadAloudNavigatorFactory private constructor( + private val guidedNavigationService: GuidedNavigationService, + private val audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, +) { + + public companion object { + + public operator fun invoke( + application: Application, + publication: Publication, + audioEngineProvider: AudioEngineProvider, + ): ReadAloudNavigatorFactory? { + val guidedNavService = publication.guidedNavigationService + ?: return null + + val audioEngineFactory = { listener: AudioEngine.Listener -> + audioEngineProvider.createEngine(publication, listener) + } + + return ReadAloudNavigatorFactory( + guidedNavigationService = guidedNavService, + audioEngineFactory = audioEngineFactory + ) + } + } + + public sealed class Error( + override val message: String, + override val cause: org.readium.r2.shared.util.Error?, + ) : org.readium.r2.shared.util.Error { + + public class UnsupportedPublication( + cause: org.readium.r2.shared.util.Error? = null, + ) : Error("Publication is not supported.", cause) + + public class GuidedNavigationService( + override val cause: ReadError, + ) : Error("Failed to acquire guided navigation documents.", cause) + } + + public suspend fun createNavigator(): Try { + val guidedDocs = buildList { + val iterator = guidedNavigationService.iterator() + while (iterator.hasNext()) { + val doc = iterator.next().getOrElse { + return Try.failure(Error.GuidedNavigationService(it)) + } + add(doc) + } + } + + val guidedTree = GuidedNavigationContainer( + children = guidedDocs.map { + GuidedNavigationContainer(it.guided, emptySet()) + }, + roles = emptySet() + ) + + val navigator = ReadAloudNavigator( + guidedNavigationTree = guidedTree, + audioEngineFactory = audioEngineFactory + ) + + return Try.success(navigator) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt new file mode 100644 index 0000000000..62fc738da8 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -0,0 +1,141 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.util.Error +import org.readium.r2.shared.util.TemporalFragmentParser +import org.readium.r2.shared.util.Url + +internal sealed interface EngineFood { + + data class AudioEngineFood( + val items: List, + val nodes: List, + ) : EngineFood { + + companion object { + + fun fromNode(node: ReadAloudNode): AudioEngineFood? { + val firstNode = node.firstLeaf() ?: return null + if (!firstNode.refs.any { it is GuidedNavigationAudioRef }) { + return null + } + + var nextLeaf: ReadAloudLeafNode? = firstNode + val audioItems = mutableListOf() + val nodes = mutableListOf() + + while (nextLeaf != null) { + val audioItem = nextLeaf.refs + .firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } + ?.toAudioItem() + + audioItem?.let { + audioItems.add(it) + nodes.add(nextLeaf) + } + + nextLeaf = nextLeaf.nextLeaf() + } + + return AudioEngineFood( + items = audioItems, + nodes = nodes + ) + } + + fun GuidedNavigationAudioRef.toAudioItem(): AudioEngine.Item { + return AudioEngine.Item( + href = url.removeFragment(), + interval = url.timeInterval + ) + } + + val Url.timeInterval get() = fragment + ?.let { TemporalFragmentParser.parse(it) } + } + } +} + +internal class ReadAloudStateMachine( + private val audioEngine: AudioEngine, +) { + sealed interface State { + + data class Playing( + val paused: Boolean, + val node: ReadAloudLeafNode, + val engineFood: EngineFood, + ) : State + + data object Ended : State + + data class Failure(val error: Error) : State + } + + sealed interface Event { + + data object AudioEngineEnded : Event + } + + fun start(initialNode: ReadAloudNode, paused: Boolean): State { + val firstLeaf = initialNode.nextLeaf() + ?: return State.Ended + val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf) + ?: return State.Ended + + if (paused) audioEngine.pause() else audioEngine.resume() + audioEngine.setPlaylist(engineFood.items) + return State.Playing(paused = paused, node = firstLeaf, engineFood = engineFood) + } + + fun State.pause(): State = + when (this) { + State.Ended -> this + is State.Failure -> this + is State.Playing -> { + if (engineFood is EngineFood.AudioEngineFood) { + audioEngine.pause() + } + copy(paused = true) + } + } + + fun State.resume(): State = + when (this) { + State.Ended -> this + is State.Failure -> this + is State.Playing -> { + if (engineFood is EngineFood.AudioEngineFood) { + audioEngine.resume() + } + copy(paused = false) + } + } + + fun State.jump(node: ReadAloudNode): State = + when (this) { + State.Ended -> start(node, paused = true) + is State.Failure -> this + is State.Playing -> start(node, paused = paused) + } + + fun State.onEvent(event: Event): State = when (event) { + Event.AudioEngineEnded -> onAudioEngineEnded() + } + + fun State.onAudioEngineEnded(): State = + when (this) { + State.Ended -> this + is State.Failure -> this + is State.Playing -> State.Ended + } +} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt index 14dc7a8945..b0121a444f 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt @@ -32,5 +32,11 @@ public value class GuidedNavigationRole(public val value: String) { public val LOI: GuidedNavigationRole = GuidedNavigationRole("loi") public val LOT: GuidedNavigationRole = GuidedNavigationRole("lot") public val LOV: GuidedNavigationRole = GuidedNavigationRole("lov") + + public val SKIPPABLE_ROLES: List = + listOf() + + public val ESCAPABLE_ROLES: List = + listOf() } } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt new file mode 100644 index 0000000000..c8260d5b03 --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.util + +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +/** + * Temporal dimension parser for + * [Media Fragment specification](https://www.w3.org/TR/media-frags/#naming-time). + * + * Supports only Normal Play Time specified as seconds without fractional part at the moment. + */ +public object TemporalFragmentParser { + + public fun parse(value: String): TimeInterval? { + if (!value.startsWith("t=")) { + return null + } + + val nptValue = value.removePrefix("t=").removePrefix("npt:") + return parseNormalPlayTime(nptValue) + } + + private fun parseNormalPlayTime(value: String): TimeInterval? { + val regex = """(\d+)?(,\d+)?""".toRegex() + val result = regex.matchEntire(value) + ?: return null + + val startOffset = result.groupValues[1] + .takeIf { it.isNotEmpty() } + ?.toIntOrNull() + ?.seconds + + val endOffset = result.groupValues[2] + .removePrefix(",") + .takeIf { it.isNotEmpty() } + ?.toIntOrNull() + ?.seconds + + return TimeInterval(startOffset, endOffset) + } +} + +public data class TimeInterval( + val start: Duration?, + val end: Duration?, +) diff --git a/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt b/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt new file mode 100644 index 0000000000..baf68bc935 --- /dev/null +++ b/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.util + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.time.Duration.Companion.seconds + +class TemporalFragmentParserTest { + + @Test + fun `fragment which is not temporal is rejected`() { + assertNull(TemporalFragmentParser.parse("htmlId")) + } + + @Test + fun `start only in seconds is accepted`() { + assertEquals( + TimeInterval(start = 4.seconds, null), + TemporalFragmentParser.parse("t=4") + ) + } + + @Test + fun `end only in seconds is accepted`() { + assertEquals( + TimeInterval(start = null, end = 40.seconds), + TemporalFragmentParser.parse("t=,40") + ) + } + + @Test + fun `start and end in seconds are accepted`() { + assertEquals( + TimeInterval(4.seconds, 60.seconds), + TemporalFragmentParser.parse("t=4,60") + ) + } + + @Test + fun `npt prefix is accepted`() { + assertEquals( + TimeInterval(40.seconds, null), + TemporalFragmentParser.parse("t=npt:40") + ) + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 3f4779fd7f..67e8f25af6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -77,10 +77,18 @@ include(":readium:navigators:media:tts") project(":readium:navigators:media:tts") .name = "readium-navigator-media-tts" +include(":readium:navigators:media:readaloud") +project(":readium:navigators:media:readaloud") + .name = "readium-navigator-media-readaloud" + include(":readium:adapters:exoplayer:audio") project(":readium:adapters:exoplayer:audio") .name = "readium-adapter-exoplayer-audio" +include(":readium:adapters:exoplayer:readaloud") +project(":readium:adapters:exoplayer:readaloud") + .name = "readium-adapter-exoplayer-readaloud" + include(":readium:opds") project(":readium:opds") .name = "readium-opds" From eeeecbb3e4aaadba47434a39c56701e21dc47bbc Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Sun, 17 Aug 2025 16:35:13 +0200 Subject: [PATCH 04/13] Draft a demo component for read aloud --- demos/navigator/build.gradle.kts | 3 +- .../java/org/readium/demo/navigator/DemoUi.kt | 35 ++++-- .../readium/demo/navigator/DemoViewModel.kt | 115 +++++++++++++++++- .../navigator/reader/ReadAloudRendition.kt | 86 +++++++++++++ .../demo/navigator/reader/ReaderOpener.kt | 111 +++++++++-------- .../demo/navigator/reader/ReaderState.kt | 22 +++- .../navigator/reader/SelectNavigatorMenu.kt | 86 +++++++++++++ .../{Rendition.kt => VisualRendition.kt} | 4 +- .../audio/DefaultExoPlayerCacheProvider.kt | 3 + .../exoplayer/readaloud/ExoPlayerEngine.kt | 8 +- .../media/readaloud/ReadAloudModel.kt | 35 ++++-- .../media/readaloud/ReadAloudNavigator.kt | 15 ++- 12 files changed, 430 insertions(+), 93 deletions(-) create mode 100644 demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt create mode 100644 demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt rename demos/navigator/src/main/java/org/readium/demo/navigator/reader/{Rendition.kt => VisualRendition.kt} (99%) diff --git a/demos/navigator/build.gradle.kts b/demos/navigator/build.gradle.kts index 8ed8801cb2..00dffca861 100644 --- a/demos/navigator/build.gradle.kts +++ b/demos/navigator/build.gradle.kts @@ -65,7 +65,8 @@ dependencies { implementation(project(":readium:readium-navigator")) implementation(project(":readium:navigators:web:readium-navigator-web-reflowable")) implementation(project(":readium:navigators:web:readium-navigator-web-fixedlayout")) - implementation(project(":readium:adapters:pdfium")) + implementation(project(":readium:navigators:media:readium-navigator-media-readaloud")) + implementation(project(":readium:adapters:exoplayer:readium-adapter-exoplayer-readaloud")) coreLibraryDesugaring(libs.desugar.jdk.libs) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt index 5c7877af17..3b0b8042fd 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt @@ -21,9 +21,12 @@ import androidx.compose.runtime.State import androidx.compose.runtime.collectAsState import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalTextToolbar import androidx.core.view.WindowInsetsControllerCompat -import org.readium.demo.navigator.reader.Rendition +import org.readium.demo.navigator.reader.ReadAloudReaderState +import org.readium.demo.navigator.reader.ReadAloudRendition +import org.readium.demo.navigator.reader.SelectNavigatorMenu +import org.readium.demo.navigator.reader.VisualReaderState +import org.readium.demo.navigator.reader.VisualRendition import org.readium.demo.navigator.util.Fullscreenable @Composable @@ -42,7 +45,6 @@ fun Scaffold( ) { content.invoke() - LocalTextToolbar SnackbarHost( modifier = Modifier .align(Alignment.BottomCenter) @@ -67,7 +69,11 @@ fun MainContent( DemoViewModel.State.BookSelection -> true is DemoViewModel.State.Error -> false DemoViewModel.State.Loading -> true - is DemoViewModel.State.Reader -> true + is DemoViewModel.State.Reader -> when (viewmodelState.readerState) { + is VisualReaderState<*, *, *, *> -> true + is ReadAloudReaderState -> false + } + is DemoViewModel.State.NavigatorSelection -> true } } @@ -79,6 +85,10 @@ fun MainContent( } } + is DemoViewModel.State.NavigatorSelection -> { + SelectNavigatorMenu(viewmodelState.viewModel) + } + is DemoViewModel.State.Error -> { Placeholder() LaunchedEffect(viewmodelState.error) { @@ -100,10 +110,19 @@ fun MainContent( viewmodel.onBookClosed() } - Rendition( - readerState = viewmodelState.readerState, - fullScreenState = fullscreenState - ) + when (viewmodelState.readerState) { + is ReadAloudReaderState -> { + ReadAloudRendition( + readerState = viewmodelState.readerState + ) + } + is VisualReaderState<*, *, *, *> -> { + VisualRendition( + readerState = viewmodelState.readerState, + fullScreenState = fullscreenState + ) + } + } } } } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index 714b246b92..68360c11ad 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -4,19 +4,38 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(ExperimentalReadiumApi::class) + package org.readium.demo.navigator import android.app.Application import androidx.lifecycle.AndroidViewModel +import androidx.lifecycle.application import androidx.lifecycle.viewModelScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch +import org.readium.adapter.exoplayer.readaloud.ExoPlayerEngineProvider import org.readium.demo.navigator.reader.ReaderOpener import org.readium.demo.navigator.reader.ReaderState +import org.readium.demo.navigator.reader.SelectNavigatorItem +import org.readium.demo.navigator.reader.SelectNavigatorViewModel +import org.readium.demo.navigator.reader.fixedConfig +import org.readium.demo.navigator.reader.reflowableConfig +import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory +import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl +import org.readium.r2.shared.util.DebugError +import org.readium.r2.shared.util.asset.AssetRetriever +import org.readium.r2.shared.util.getOrElse +import org.readium.r2.shared.util.http.DefaultHttpClient import org.readium.r2.shared.util.toDebugDescription +import org.readium.r2.streamer.PublicationOpener +import org.readium.r2.streamer.parser.DefaultPublicationParser import timber.log.Timber class DemoViewModel( @@ -28,6 +47,10 @@ class DemoViewModel( data object BookSelection : State + data class NavigatorSelection( + val viewModel: SelectNavigatorViewModel, + ) : State + data object Loading : State @@ -36,7 +59,7 @@ class DemoViewModel( ) : State data class Reader( - val readerState: ReaderState<*, *, *, *>, + val readerState: ReaderState, ) : State } @@ -44,9 +67,24 @@ class DemoViewModel( Timber.plant(Timber.DebugTree()) } + private val httpClient = + DefaultHttpClient() + + private val assetRetriever = + AssetRetriever(application.contentResolver, httpClient) + + private val publicationParser = + DefaultPublicationParser(application, httpClient, assetRetriever, null) + + private val publicationOpener = + PublicationOpener(publicationParser) + private val readerOpener = ReaderOpener(application) + private val audioEngineProvider = + ExoPlayerEngineProvider(application) + private val stateMutable: MutableStateFlow = MutableStateFlow(State.BookSelection) @@ -56,7 +94,80 @@ class DemoViewModel( stateMutable.value = State.Loading viewModelScope.launch { - readerOpener.open(url) + val asset = assetRetriever.retrieve(url) + .getOrElse { + Timber.d(it.toDebugDescription()) + stateMutable.value = State.Error(it) + return@launch + } + + val publication = publicationOpener.open(asset, allowUserInteraction = false) + .getOrElse { + asset.close() + Timber.d(it.toDebugDescription()) + stateMutable.value = State.Error(it) + return@launch + } + + val reflowableFactory = + ReflowableWebRenditionFactory( + application = application, + publication = publication, + configuration = reflowableConfig + )?.let { SelectNavigatorItem.ReflowableWeb(it) } + + val fixedFactory = + FixedWebRenditionFactory( + application = application, + publication = publication, + configuration = fixedConfig + )?.let { SelectNavigatorItem.FixedWeb(it) } + + val readAloudFactory = + ReadAloudNavigatorFactory( + application = application, + publication = publication, + audioEngineProvider = audioEngineProvider + )?.let { SelectNavigatorItem.ReadAloud(it) } + + val factories = listOfNotNull( + reflowableFactory, + fixedFactory, + readAloudFactory + ) + + when (factories.size) { + 0 -> { + val error = DebugError("Publication not supported") + Timber.d(error.toDebugDescription()) + stateMutable.value = State.Error(error) + } + 1 -> { + onNavigatorSelected(url, publication, factories.first()) + } + else -> { + val selectionViewModel = SelectNavigatorViewModel( + items = factories, + onItemSelected = { onNavigatorSelected(url, publication, it) }, + onMenuDismissed = { stateMutable.value = State.BookSelection } + ) + + stateMutable.value = + State.NavigatorSelection(selectionViewModel) + } + } + } + } + + fun onNavigatorSelected( + url: AbsoluteUrl, + publication: Publication, + navigatorItem: SelectNavigatorItem, + ) { + stateMutable.value = State.Loading + + viewModelScope.launch { + readerOpener.open(url, publication, navigatorItem) .onFailure { Timber.d(it.toDebugDescription()) stateMutable.value = State.Error(it) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt new file mode 100644 index 0000000000..12775d825c --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2024 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.demo.navigator.reader + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowOutward +import androidx.compose.material.icons.filled.Pause +import androidx.compose.material.icons.filled.SkipNext +import androidx.compose.material.icons.filled.SkipPrevious +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.Scaffold +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import org.readium.r2.shared.ExperimentalReadiumApi + +@Composable +fun ReadAloudRendition( + readerState: ReadAloudReaderState, +) { + Scaffold( + modifier = Modifier.fillMaxSize() + ) { contentPadding -> + Column( + modifier = Modifier + .padding(contentPadding) + .fillMaxSize(), + verticalArrangement = Arrangement.Bottom + ) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.Center, + verticalAlignment = Alignment.CenterVertically, + ) { + IconButton( + onClick = { readerState.navigator.skip(force = true) } + ) { + Icon( + imageVector = Icons.Default.SkipPrevious, + contentDescription = "Skip to previous" + ) + } + + IconButton( + onClick = { readerState.navigator.pause() } + ) { + Icon( + imageVector = Icons.Default.Pause, + contentDescription = "Pause" + ) + } + + IconButton( + onClick = { readerState.navigator.skip(force = true) } + ) { + Icon( + imageVector = Icons.Default.SkipNext, + contentDescription = "Skip to next" + ) + } + + IconButton( + onClick = { readerState.navigator.escape(force = true) } + ) { + Icon( + imageVector = Icons.Default.ArrowOutward, + contentDescription = "Escape" + ) + } + } + } + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index 3e11e82818..075e22cd53 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -26,6 +26,7 @@ import org.readium.navigator.common.DecorationLocation import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings import org.readium.navigator.common.SettingsController +import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory import org.readium.navigator.web.fixedlayout.FixedWebGoLocation import org.readium.navigator.web.fixedlayout.FixedWebLocation import org.readium.navigator.web.fixedlayout.FixedWebRenditionController @@ -42,70 +43,56 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl -import org.readium.r2.shared.util.DebugError import org.readium.r2.shared.util.Error import org.readium.r2.shared.util.Try -import org.readium.r2.shared.util.asset.AssetRetriever import org.readium.r2.shared.util.getOrElse -import org.readium.r2.shared.util.http.DefaultHttpClient -import org.readium.r2.streamer.PublicationOpener -import org.readium.r2.streamer.parser.DefaultPublicationParser class ReaderOpener( private val application: Application, ) { - - private val httpClient = - DefaultHttpClient() - - private val assetRetriever = - AssetRetriever(application.contentResolver, httpClient) - - private val publicationParser = - DefaultPublicationParser(application, httpClient, assetRetriever, null) - - private val publicationOpener = - PublicationOpener(publicationParser) - - suspend fun open(url: AbsoluteUrl): Try, Error> { - val asset = assetRetriever.retrieve(url) - .getOrElse { return Try.failure(it) } - - val publication = publicationOpener.open(asset, allowUserInteraction = false) - .getOrElse { - asset.close() - return Try.failure(it) - } - + suspend fun open( + url: AbsoluteUrl, + publication: Publication, + selectedNavigator: SelectNavigatorItem, + ): Try { val initialLocator = LocatorRepository.getLocator(url) - val readerState = ( - createFixedWebReader(url, publication, initialLocator) - ?: createReflowableWebReader(url, publication, initialLocator) - ) - .or { Try.failure(DebugError("Publication not supported")) } - .getOrElse { error -> - publication.close() - return Try.failure(error) - } + val readerState = when (selectedNavigator) { + is SelectNavigatorItem.ReflowableWeb -> + createReflowableWebReader( + url, + publication, + selectedNavigator.factory, + initialLocator + ) + is SelectNavigatorItem.FixedWeb -> + createFixedWebReader( + url, + publication, + selectedNavigator.factory, + initialLocator + ) + is SelectNavigatorItem.ReadAloud -> + createReadAloudReader( + url, + publication, + selectedNavigator.factory, + initialLocator + ) + }.getOrElse { error -> + publication.close() + return Try.failure(error) + } return Try.success(readerState) } - private fun Try?.or(onNull: () -> Try): Try = - this ?: onNull() - private suspend fun createReflowableWebReader( url: AbsoluteUrl, publication: Publication, + navigatorFactory: ReflowableWebRenditionFactory, initialLocator: Locator?, - ): Try, Error>? { - val navigatorFactory = ReflowableWebRenditionFactory( - application = application, - publication = publication, - configuration = reflowableConfig - ) ?: return null - + ): Try, Error> { val initialLocation = initialLocator?.let { ReflowableWebGoLocation(it) } val coroutineScope = MainScope() @@ -140,7 +127,7 @@ class ReaderOpener( val actionModeFactory = SelectionActionModeFactory(highlightsManager) - val readerState = ReaderState( + val readerState = VisualReaderState( url = url, coroutineScope = coroutineScope, publication = publication, @@ -157,14 +144,9 @@ class ReaderOpener( private suspend fun createFixedWebReader( url: AbsoluteUrl, publication: Publication, + navigatorFactory: FixedWebRenditionFactory, initialLocator: Locator?, - ): Try, Error>? { - val navigatorFactory = FixedWebRenditionFactory( - application = application, - publication = publication, - configuration = fixedConfig - ) ?: return null - + ): Try, Error> { val initialLocation = initialLocator?.let { FixedWebGoLocation(it) } val coroutineScope = MainScope() @@ -195,7 +177,7 @@ class ReaderOpener( val actionModeFactory = SelectionActionModeFactory(highlightsManager) - val readerState = ReaderState( + val readerState = VisualReaderState( url = url, coroutineScope = coroutineScope, publication = publication, @@ -209,6 +191,23 @@ class ReaderOpener( return Try.success(readerState) } + private suspend fun createReadAloudReader( + url: AbsoluteUrl, + publication: Publication, + navigatorFactory: ReadAloudNavigatorFactory, + initialLocator: Locator?, + ): Try { + val navigator = navigatorFactory.createNavigator() + .getOrElse { return Try.failure(it) } + + val readerState = ReadAloudReaderState( + url = url, + publication = publication, + navigator = navigator + ) + return Try.success(readerState) + } + private fun applySettings( coroutineScope: CoroutineScope, settingsController: SettingsController, diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt index ba6765f0ca..afb2d971ea 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt @@ -18,11 +18,17 @@ import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.RenditionState import org.readium.navigator.common.SelectionController import org.readium.navigator.common.SelectionLocation +import org.readium.navigator.media.readaloud.ReadAloudNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl -data class ReaderState( +sealed interface ReaderState { + + fun close() +} + +data class VisualReaderState( val url: AbsoluteUrl, val coroutineScope: CoroutineScope, val publication: Publication, @@ -31,10 +37,20 @@ data class ReaderState, val onControllerAvailable: (C) -> Unit, val actionModeFactory: SelectionActionModeFactory, -) where C : NavigationController, C : SelectionController { +) : ReaderState where C : NavigationController, C : SelectionController { - fun close() { + override fun close() { coroutineScope.cancel() publication.close() } } + +data class ReadAloudReaderState( + val url: AbsoluteUrl, + val publication: Publication, + val navigator: ReadAloudNavigator, +) : ReaderState { + + override fun close() { + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt new file mode 100644 index 0000000000..edd5cf295e --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.demo.navigator.reader + +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.window.PopupProperties +import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory +import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory +import org.readium.r2.shared.ExperimentalReadiumApi + +class SelectNavigatorViewModel( + val items: List, + val onItemSelected: (SelectNavigatorItem) -> Unit, + val onMenuDismissed: () -> Unit, +) { + + fun select(item: SelectNavigatorItem) { + onItemSelected(item) + } + + fun cancel() { + onMenuDismissed() + } +} + +sealed class SelectNavigatorItem( + val name: String, +) { + + abstract val factory: Any + + data class ReflowableWeb( + override val factory: ReflowableWebRenditionFactory, + ) : SelectNavigatorItem("Reflowable Web Rendition") + + data class FixedWeb( + override val factory: FixedWebRenditionFactory, + ) : SelectNavigatorItem("Fixed Web Rendition") + + data class ReadAloud( + override val factory: ReadAloudNavigatorFactory, + ) : SelectNavigatorItem("Read Aloud Navigator") +} + +@Composable +fun SelectNavigatorMenu( + viewModel: SelectNavigatorViewModel, +) { + SelectNavigatorMenu( + popupProperties = PopupProperties(), + items = viewModel.items, + onItemSelected = { viewModel.select(it) }, + onDismissRequest = viewModel::cancel + ) +} + +@Composable +private fun SelectNavigatorMenu( + popupProperties: PopupProperties, + items: List, + onItemSelected: (SelectNavigatorItem) -> Unit, + onDismissRequest: () -> Unit, +) { + DropdownMenu( + expanded = true, + properties = popupProperties, + onDismissRequest = onDismissRequest + ) { + for (item in items) { + DropdownMenuItem( + text = { Text(item.name) }, + onClick = { onItemSelected(item) } + ) + } + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/Rendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt similarity index 99% rename from demos/navigator/src/main/java/org/readium/demo/navigator/reader/Rendition.kt rename to demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt index b49a1aae96..05b2302d1c 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/Rendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt @@ -66,8 +66,8 @@ import org.readium.r2.shared.util.toUri @OptIn(ExperimentalMaterial3Api::class) @Composable -fun Rendition( - readerState: ReaderState, +fun VisualRendition( + readerState: VisualReaderState, fullScreenState: MutableState, ) where C : NavigationController, C : SelectionController { val coroutineScope = rememberCoroutineScope() diff --git a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt index de852d41bd..1dd6bbd8cf 100644 --- a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt +++ b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt @@ -1,3 +1,5 @@ +@file:kotlin.OptIn(InternalReadiumApi::class) + package org.readium.adapter.exoplayer.audio import androidx.annotation.OptIn @@ -7,6 +9,7 @@ import androidx.media3.datasource.cache.Cache import androidx.media3.datasource.cache.CacheDataSink import androidx.media3.datasource.cache.CacheDataSource import org.readium.r2.shared.DelicateReadiumApi +import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.toUrl diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index 75719e386e..6bcb05bbbe 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -72,6 +72,10 @@ public class ExoPlayerEngine private constructor( if (events.contains(Player.EVENT_PLAYBACK_STATE_CHANGED) && player.playbackState == Player.STATE_ENDED) { this@ExoPlayerEngine.listener.onPlaybackEnded() } + + if (events.contains(Player.EVENT_MEDIA_ITEM_TRANSITION)) { + this@ExoPlayerEngine.listener.onItemChanged(player.currentMediaItemIndex) + } } } @@ -102,8 +106,8 @@ public class ExoPlayerEngine private constructor( val mediaItems = items.map { item -> val clippingConfig = MediaItem.ClippingConfiguration.Builder() .apply { - item.startOffset?.let { setStartPositionMs(it.inWholeMilliseconds) } - item.endOffset?.let { setEndPositionMs(it.inWholeMilliseconds) } + item.interval?.start?.let { setStartPositionMs(it.inWholeMilliseconds) } + item.interval?.end?.let { setEndPositionMs(it.inWholeMilliseconds) } }.build() MediaItem.Builder() .setUri(item.href.toString()) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt index fdba9931e2..af9098c7a5 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt @@ -62,21 +62,30 @@ internal fun ReadAloudNode.firstLeaf(): ReadAloudLeafNode? = } @ExperimentalReadiumApi -internal fun ReadAloudNode.nextLeaf(): ReadAloudLeafNode? { - if (children.isNotEmpty()) { - return children[0].nextLeaf() +internal fun ReadAloudNode.nextLeaf(): ReadAloudLeafNode? = + when (this) { + is ReadAloudInnerNode -> firstLeaf() + is ReadAloudLeafNode -> { + val siblings = parent.children + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + if (currentIndex < siblings.size - 1) { + val sister = siblings[currentIndex + 1] + when (sister) { + is ReadAloudInnerNode -> sister.firstLeaf() + is ReadAloudLeafNode -> sister + } + } else { + val next = parent.skipToNext() ?: return null + when (next) { + is ReadAloudInnerNode -> next.firstLeaf() + is ReadAloudLeafNode -> next + } + } + } } - val siblings = parent?.children ?: return null - val currentIndex = siblings.indexOf(this) - check(currentIndex != -1) - - return currentIndex - .takeIf { it < siblings.size - 1 } - ?.let { siblings[currentIndex + 1].nextLeaf() } - ?: parent!!.skipToNext()?.nextLeaf() -} - @ExperimentalReadiumApi internal fun ReadAloudNode.skipToNext(): ReadAloudNode? { val siblings = parent?.children ?: return null diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index 1adac47545..f0d4c88b1e 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -13,7 +13,7 @@ import org.readium.r2.shared.util.data.ReadError @ExperimentalReadiumApi public class ReadAloudNavigator private constructor( - rootNode: ReadAloudInnerNode, + firstLeaf: ReadAloudLeafNode, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, ) { @@ -22,9 +22,12 @@ public class ReadAloudNavigator private constructor( public suspend operator fun invoke( guidedNavigationTree: GuidedNavigationContainer, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, - ): ReadAloudNavigator = withContext(Dispatchers.Default) { - val tree = GuidedNavigationAdapter().adapt(guidedNavigationTree) - ReadAloudNavigator(tree, audioEngineFactory) + ): ReadAloudNavigator { + val tree = withContext(Dispatchers.Default) { + GuidedNavigationAdapter().adapt(guidedNavigationTree) + } + val firstLeaf = checkNotNull(tree.firstLeaf()) + return ReadAloudNavigator(firstLeaf, audioEngineFactory) } } @@ -73,13 +76,13 @@ public class ReadAloudNavigator private constructor( private val stateMachine = ReadAloudStateMachine(audioEngine) - private val stateMutable = MutableStateFlow(stateMachine.start(rootNode, paused = false)) + private val stateMutable = MutableStateFlow(stateMachine.start(firstLeaf, paused = false)) private val playbackMutable: MutableStateFlow = MutableStateFlow(Playback(state = State.Ready, playWhenReady = true)) private val nodeMutable: MutableStateFlow = - MutableStateFlow(rootNode) + MutableStateFlow(firstLeaf) public val playback: StateFlow = playbackMutable.asStateFlow() From 5dd1575d6eb659696bd3a3b3714ed23200cfd0bd Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Mon, 18 Aug 2025 09:14:37 +0200 Subject: [PATCH 05/13] Fix a bunch of bugs --- .../media/readaloud/ReadAloudNavigator.kt | 5 ++++- .../media/readaloud/ReadAloudStateMachine.kt | 5 ++--- .../r2/shared/util/TemporalFragmentParser.kt | 17 ++++++----------- .../shared/util/TemporalFragmentParserTest.kt | 8 ++++++++ 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index f0d4c88b1e..7d8d48634e 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -40,6 +40,8 @@ public class ReadAloudNavigator private constructor( public data object Ready : State, MediaNavigator.State.Ready + public data object Buffering : MediaNavigator.State.Buffering + public data object Ended : State, MediaNavigator.State.Ended public data class Failure(val error: Error) : State, MediaNavigator.State.Failure @@ -76,7 +78,8 @@ public class ReadAloudNavigator private constructor( private val stateMachine = ReadAloudStateMachine(audioEngine) - private val stateMutable = MutableStateFlow(stateMachine.start(firstLeaf, paused = false)) + private val stateMutable: MutableStateFlow = + MutableStateFlow(stateMachine.start(firstLeaf, paused = false)) private val playbackMutable: MutableStateFlow = MutableStateFlow(Playback(state = State.Ready, playWhenReady = true)) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 62fc738da8..86d5d14a20 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -23,8 +23,7 @@ internal sealed interface EngineFood { companion object { - fun fromNode(node: ReadAloudNode): AudioEngineFood? { - val firstNode = node.firstLeaf() ?: return null + fun fromNode(firstNode: ReadAloudLeafNode): AudioEngineFood? { if (!firstNode.refs.any { it is GuidedNavigationAudioRef }) { return null } @@ -87,7 +86,7 @@ internal class ReadAloudStateMachine( } fun start(initialNode: ReadAloudNode, paused: Boolean): State { - val firstLeaf = initialNode.nextLeaf() + val firstLeaf = initialNode.firstLeaf() ?: return State.Ended val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf) ?: return State.Ended diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt index c8260d5b03..5c998c7122 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt @@ -13,7 +13,7 @@ import kotlin.time.Duration.Companion.seconds * Temporal dimension parser for * [Media Fragment specification](https://www.w3.org/TR/media-frags/#naming-time). * - * Supports only Normal Play Time specified as seconds without fractional part at the moment. + * Supports only Normal Play Time specified as seconds at the moment. */ public object TemporalFragmentParser { @@ -27,19 +27,14 @@ public object TemporalFragmentParser { } private fun parseNormalPlayTime(value: String): TimeInterval? { - val regex = """(\d+)?(,\d+)?""".toRegex() - val result = regex.matchEntire(value) - ?: return null + val components = value.split(",", limit = 2) - val startOffset = result.groupValues[1] - .takeIf { it.isNotEmpty() } - ?.toIntOrNull() + val startOffset = components.getOrNull(0) + ?.toDoubleOrNull() ?.seconds - val endOffset = result.groupValues[2] - .removePrefix(",") - .takeIf { it.isNotEmpty() } - ?.toIntOrNull() + val endOffset = components.getOrNull(1) + ?.toDoubleOrNull() ?.seconds return TimeInterval(startOffset, endOffset) diff --git a/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt b/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt index baf68bc935..eecfff52b5 100644 --- a/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt +++ b/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt @@ -49,4 +49,12 @@ class TemporalFragmentParserTest { TemporalFragmentParser.parse("t=npt:40") ) } + + @Test + fun `floating point values are accepted`() { + assertEquals( + TimeInterval(40.5.seconds, 83.235.seconds), + TemporalFragmentParser.parse("t=40.500,83.235") + ) + } } From 53a9fdb72a856d1c42828e2e42b735ec717b74e0 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Wed, 20 Aug 2025 15:33:09 +0200 Subject: [PATCH 06/13] Make guided nav model more flexible and revisit playback API --- .../{ReadAloudEngine.kt => AudioEngine.kt} | 30 ++--------- .../readaloud/GuidedNavigationAdapter.kt | 28 ++++++---- .../media/readaloud/PlaybackEngine.kt | 52 +++++++++++++++++++ .../media/readaloud/ReadAloudModel.kt | 5 +- .../media/readaloud/ReadAloudNavigator.kt | 4 +- .../readaloud/ReadAloudNavigatorFactory.kt | 10 ++-- .../shared/guided/GuidedNavigationDocument.kt | 29 ++++------- .../parser/epub/MediaOverlaysService.kt | 9 ++-- 8 files changed, 100 insertions(+), 67 deletions(-) rename readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/{ReadAloudEngine.kt => AudioEngine.kt} (62%) create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt similarity index 62% rename from readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt rename to readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt index dac3053ec3..0a987cd636 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt @@ -8,13 +8,11 @@ package org.readium.navigator.media.readaloud import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.TimeInterval import org.readium.r2.shared.util.Url -public sealed interface ReadAloudEngine - -public interface AudioEngine : ReadAloudEngine { +@ExperimentalReadiumApi +public interface AudioEngine { public interface Listener { @@ -35,6 +33,7 @@ public interface AudioEngine : ReadAloudEngine { public fun resume() } +@ExperimentalReadiumApi public interface AudioEngineProvider { public fun createEngine( @@ -42,26 +41,3 @@ public interface AudioEngineProvider { listener: AudioEngine.Listener, ): AudioEngine } - -@ExperimentalReadiumApi -public interface PlaybackEngine : ReadAloudEngine { - - public fun play(node: ReadAloudLeafNode) -} - -@ExperimentalReadiumApi -public interface TtsEngine : PlaybackEngine { - - public interface Voice { - - /** - * The languages supported by the voice. - */ - public val languages: Set - } - - /** - * Sets of voices available with this [PlaybackEngine]. - */ - public val voices: Set -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt index aa5d115c13..5495e97035 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt @@ -7,34 +7,42 @@ package org.readium.navigator.media.readaloud import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationContainer -import org.readium.r2.shared.guided.GuidedNavigationLeaf import org.readium.r2.shared.guided.GuidedNavigationObject @OptIn(ExperimentalReadiumApi::class) internal class GuidedNavigationAdapter { - fun adapt(guidedNavTree: GuidedNavigationContainer): ReadAloudInnerNode { + fun adapt(guidedNavTree: GuidedNavigationObject): ReadAloudInnerNode { val children = guidedNavTree.children.mapNotNull { adaptNode(it) } - val node = ReadAloudInnerNode(children, guidedNavTree.roles) + val node = ReadAloudInnerNode( + children = children, + roles = guidedNavTree.roles, + refs = guidedNavTree.refs + ) setParentInChildren(node) return node } private fun adaptNode(guidedNavigationObject: GuidedNavigationObject): ReadAloudNode? { - return when (guidedNavigationObject) { - is GuidedNavigationContainer -> + return when (guidedNavigationObject.children.size) { + 0 -> + adaptLeat(guidedNavigationObject) + else -> guidedNavigationObject.children .mapNotNull { adaptNode(it) } .takeIf { it.isNotEmpty() } - ?.let { ReadAloudInnerNode(it, guidedNavigationObject.roles) } + ?.let { + ReadAloudInnerNode( + children = it, + roles = guidedNavigationObject.roles, + refs = guidedNavigationObject.refs + ) + } ?.also { setParentInChildren(it) } - is GuidedNavigationLeaf -> - adaptLeat(guidedNavigationObject) } } - private fun adaptLeat(guidedNavigationLeaf: GuidedNavigationLeaf): ReadAloudLeafNode? { + private fun adaptLeat(guidedNavigationLeaf: GuidedNavigationObject): ReadAloudLeafNode? { return ReadAloudLeafNode( text = guidedNavigationLeaf.text, refs = guidedNavigationLeaf.refs, diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt new file mode 100644 index 0000000000..75e6818974 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt @@ -0,0 +1,52 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +public interface PlaybackEngine { + + public interface Listener { + + public fun onUtterancesReady() + + public fun onPlaybackCompleted() + } + + public fun feed(utterances: List) + + public fun speak(utteranceIndex: Int) +} + +@ExperimentalReadiumApi +public interface PausablePlaybackEngine : PlaybackEngine { + + public fun pause() + + public fun resume() +} + +@ExperimentalReadiumApi +public interface PlaybackEngineProvider { + + public interface Voice { + + /** + * The languages supported by the voice. + */ + public val languages: Set + } + + /** + * Sets of voices available with this [PlaybackEngine]. + */ + public val voices: Set + + public fun createEngine(voice: V): PlaybackEngine +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt index af9098c7a5..71e8c814fb 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt @@ -17,6 +17,8 @@ public sealed interface ReadAloudNode { public val roles: Set + public val refs: Set + public val parent: ReadAloudNode? public val children: List @@ -26,6 +28,7 @@ public sealed interface ReadAloudNode { public class ReadAloudInnerNode( override val children: List, override val roles: Set, + override val refs: Set, ) : ReadAloudNode { override var parent: ReadAloudNode? = null @@ -35,7 +38,7 @@ public class ReadAloudInnerNode( @ExperimentalReadiumApi public data class ReadAloudLeafNode( val text: GuidedNavigationText?, - val refs: Set, + override val refs: Set, override val roles: Set, ) : ReadAloudNode { diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index 7d8d48634e..fee450b836 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -8,7 +8,7 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.withContext import org.readium.navigator.media.common.MediaNavigator import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationContainer +import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.util.data.ReadError @ExperimentalReadiumApi @@ -20,7 +20,7 @@ public class ReadAloudNavigator private constructor( public companion object { public suspend operator fun invoke( - guidedNavigationTree: GuidedNavigationContainer, + guidedNavigationTree: GuidedNavigationObject, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, ): ReadAloudNavigator { val tree = withContext(Dispatchers.Default) { diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index a651888ede..800ecbb542 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -8,7 +8,7 @@ package org.readium.navigator.media.readaloud import android.app.Application import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationContainer +import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.services.GuidedNavigationService import org.readium.r2.shared.publication.services.guidedNavigationService @@ -68,11 +68,13 @@ public class ReadAloudNavigatorFactory private constructor( } } - val guidedTree = GuidedNavigationContainer( + val guidedTree = GuidedNavigationObject( children = guidedDocs.map { - GuidedNavigationContainer(it.guided, emptySet()) + GuidedNavigationObject(it.guided, roles = emptySet(), refs = emptySet(), text = null) }, - roles = emptySet() + roles = emptySet(), + refs = emptySet(), + text = null ) val navigator = ReadAloudNavigator( diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt index a9bc686d58..af23cf0552 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt @@ -14,21 +14,12 @@ public data class GuidedNavigationDocument( val links: List, val guided: List, ) - -public sealed interface GuidedNavigationObject { - public val roles: Set -} - -public data class GuidedNavigationLeaf( - val text: GuidedNavigationText?, - val refs: Set, - override val roles: Set, -) : GuidedNavigationObject - -public data class GuidedNavigationContainer( +public data class GuidedNavigationObject( val children: List, - override val roles: Set, -) : GuidedNavigationObject + val roles: Set, + val refs: Set, + val text: GuidedNavigationText?, +) @JvmInline public value class SsmlString(public val value: String) @@ -46,16 +37,18 @@ public data class GuidedNavigationText private constructor( } } -public sealed interface GuidedNavigationRef +public sealed interface GuidedNavigationRef { + public val url: Url +} public data class GuidedNavigationTextRef( - val url: Url, + override val url: Url, ) : GuidedNavigationRef public data class GuidedNavigationImageRef( - val url: Url, + override val url: Url, ) : GuidedNavigationRef public data class GuidedNavigationAudioRef( - val url: Url, + override val url: Url, ) : GuidedNavigationRef diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt index adb8e57565..497ce4dda5 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt @@ -11,9 +11,7 @@ package org.readium.r2.streamer.parser.epub import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationAudioRef -import org.readium.r2.shared.guided.GuidedNavigationContainer import org.readium.r2.shared.guided.GuidedNavigationDocument -import org.readium.r2.shared.guided.GuidedNavigationLeaf import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.guided.GuidedNavigationRole import org.readium.r2.shared.guided.GuidedNavigationTextRef @@ -112,7 +110,7 @@ internal object SmilParser { node: ElementNode, filePath: Url, prefixMap: Map, - ): GuidedNavigationContainer? { + ): GuidedNavigationObject? { val roles = parseRoles(node, prefixMap) val children: MutableList = mutableListOf() for (child in node.getAll()) { @@ -123,7 +121,7 @@ internal object SmilParser { } } - return GuidedNavigationContainer(children = children, roles = roles) + return GuidedNavigationObject(children = children, roles = roles, refs = emptySet(), text = null) } private fun parsePar( @@ -156,7 +154,8 @@ internal object SmilParser { audio?.let { GuidedNavigationAudioRef(filePath.resolve(it)) } ) - return GuidedNavigationLeaf( + return GuidedNavigationObject( + children = emptyList(), text = null, refs = refs, roles = roles From 46dfe82885374079f89f2a70f47a5d05bde8d634 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 21 Aug 2025 09:39:59 +0200 Subject: [PATCH 07/13] Implement playback state --- .../exoplayer/readaloud/ExoPlayerEngine.kt | 22 +-- .../navigator/media/readaloud/AudioEngine.kt | 14 +- .../media/readaloud/ReadAloudNavigator.kt | 61 +++++--- .../media/readaloud/ReadAloudStateMachine.kt | 139 +++++++++++------- 4 files changed, 150 insertions(+), 86 deletions(-) diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index 6bcb05bbbe..ec292cd656 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -69,8 +69,14 @@ public class ExoPlayerEngine private constructor( } override fun onEvents(player: Player, events: Player.Events) { - if (events.contains(Player.EVENT_PLAYBACK_STATE_CHANGED) && player.playbackState == Player.STATE_ENDED) { - this@ExoPlayerEngine.listener.onPlaybackEnded() + if (events.contains(Player.EVENT_PLAYBACK_STATE_CHANGED)) { + val newState = when (player.playbackState) { + Player.STATE_READY -> AudioEngine.State.Ready + Player.STATE_BUFFERING -> AudioEngine.State.Starved + Player.STATE_ENDED -> AudioEngine.State.Ended + else -> null + } + newState?.let { this@ExoPlayerEngine.listener.onStateChanged(it) } } if (events.contains(Player.EVENT_MEDIA_ITEM_TRANSITION)) { @@ -122,13 +128,11 @@ public class ExoPlayerEngine private constructor( } } - override fun resume() { - exoPlayer.play() - } - - override fun pause() { - exoPlayer.pause() - } + override var playWhenReady: Boolean + get() = exoPlayer.playWhenReady + set(value) { + exoPlayer.playWhenReady = value + } public fun close() { coroutineScope.cancel() diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt index 0a987cd636..ed135d42c2 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt @@ -18,7 +18,13 @@ public interface AudioEngine { public fun onItemChanged(index: Int) - public fun onPlaybackEnded() + public fun onStateChanged(state: State) + } + + public enum class State { + Ready, + Starved, + Ended, } public data class Item( @@ -26,11 +32,9 @@ public interface AudioEngine { val interval: TimeInterval?, ) - public fun setPlaylist(items: List) + public var playWhenReady: Boolean - public fun pause() - - public fun resume() + public fun setPlaylist(items: List) } @ExperimentalReadiumApi diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index fee450b836..ff73b9a308 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -1,13 +1,17 @@ +@file:OptIn(InternalReadiumApi::class) package org.readium.navigator.media.readaloud +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.withContext import org.readium.navigator.media.common.MediaNavigator import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.util.data.ReadError @@ -34,13 +38,14 @@ public class ReadAloudNavigator private constructor( public data class Playback( val state: State, val playWhenReady: Boolean, + val node: ReadAloudLeafNode, ) public sealed interface State { public data object Ready : State, MediaNavigator.State.Ready - public data object Buffering : MediaNavigator.State.Buffering + public data object Buffering : State, MediaNavigator.State.Buffering public data object Ended : State, MediaNavigator.State.Ended @@ -62,14 +67,14 @@ public class ReadAloudNavigator private constructor( private inner class AudioEngineListener : AudioEngine.Listener { override fun onItemChanged(index: Int) { - val state = stateMutable.value as ReadAloudStateMachine.State.Playing - val engineFood = state.engineFood as EngineFood.AudioEngineFood - nodeMutable.value = engineFood.nodes[index] + with(stateMachine) { + stateMutable.value = stateMutable.value.onAudioEngineItemChanged(index) + } } - override fun onPlaybackEnded() { + override fun onStateChanged(state: AudioEngine.State) { with(stateMachine) { - stateMutable.value = stateMutable.value.onAudioEngineEnded() + stateMutable.value = stateMutable.value.onAudioEngineStateChanged(state) } } } @@ -78,20 +83,34 @@ public class ReadAloudNavigator private constructor( private val stateMachine = ReadAloudStateMachine(audioEngine) - private val stateMutable: MutableStateFlow = - MutableStateFlow(stateMachine.start(firstLeaf, paused = false)) - - private val playbackMutable: MutableStateFlow = - MutableStateFlow(Playback(state = State.Ready, playWhenReady = true)) + private val stateMutable: MutableStateFlow = run { + val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf)!! + MutableStateFlow(stateMachine.play(engineFood, playWhenReady = true)) + } - private val nodeMutable: MutableStateFlow = - MutableStateFlow(firstLeaf) + private val coroutineScope: CoroutineScope = + MainScope() public val playback: StateFlow = - playbackMutable.asStateFlow() + stateMutable.mapStateIn(coroutineScope) { state -> + Playback( + playWhenReady = state.playWhenReady, + state = state.playbackState.toState(), + node = state.node + ) + } - public val node: StateFlow = - nodeMutable.asStateFlow() + private fun ReadAloudStateMachine.PlaybackState.toState(): State = + when (this) { + is ReadAloudStateMachine.PlaybackState.Ready -> + State.Ready + is ReadAloudStateMachine.PlaybackState.Starved -> + State.Buffering + is ReadAloudStateMachine.PlaybackState.Ended -> + State.Ended + is ReadAloudStateMachine.PlaybackState.Failure -> + State.Failure(Error.EngineError(error)) + } public fun play() { with(stateMachine) { @@ -112,18 +131,18 @@ public class ReadAloudNavigator private constructor( } public fun canEscape(): Boolean = - nodeMutable.value.isEscapable() + stateMutable.value.node.isEscapable() public fun canSkip(): Boolean = - nodeMutable.value.isSkippable() + stateMutable.value.node.isSkippable() public fun escape(force: Boolean = true) { - nodeMutable.value.escape(force) + stateMutable.value.node.escape(force) ?.let { go(it) } } public fun skip(force: Boolean = true) { - nodeMutable.value.skip(force) + stateMutable.value.node.skip(force) ?.let { go(it) } } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 86d5d14a20..6130ea3185 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -62,79 +62,116 @@ internal sealed interface EngineFood { ?.let { TemporalFragmentParser.parse(it) } } } + + data class TtsFood( + val nodes: List, + ) { + + companion object { + + fun fromNode(firstNode: ReadAloudLeafNode): TtsFood? { + return null + } + } + } } internal class ReadAloudStateMachine( private val audioEngine: AudioEngine, ) { - sealed interface State { - data class Playing( - val paused: Boolean, - val node: ReadAloudLeafNode, - val engineFood: EngineFood, - ) : State + sealed interface PlaybackState { - data object Ended : State + data object Ready : PlaybackState - data class Failure(val error: Error) : State - } + data object Starved : PlaybackState - sealed interface Event { + data object Ended : PlaybackState - data object AudioEngineEnded : Event + data class Failure(val error: Error) : PlaybackState } - fun start(initialNode: ReadAloudNode, paused: Boolean): State { - val firstLeaf = initialNode.firstLeaf() - ?: return State.Ended - val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf) - ?: return State.Ended + data class State( + val playbackState: PlaybackState, + val playWhenReady: Boolean, + val node: ReadAloudLeafNode, + val engineFood: EngineFood, + ) + + sealed interface Event { - if (paused) audioEngine.pause() else audioEngine.resume() - audioEngine.setPlaylist(engineFood.items) - return State.Playing(paused = paused, node = firstLeaf, engineFood = engineFood) + data class AudioEngineStateChanged(val state: AudioEngine.State) : Event + + data class AudioEngineItemChanged(val index: Int) : Event } - fun State.pause(): State = - when (this) { - State.Ended -> this - is State.Failure -> this - is State.Playing -> { - if (engineFood is EngineFood.AudioEngineFood) { - audioEngine.pause() - } - copy(paused = true) + fun play(engineFood: EngineFood, playWhenReady: Boolean): State { + when (engineFood) { + is EngineFood.AudioEngineFood -> { + audioEngine.playWhenReady = !playWhenReady + audioEngine.setPlaylist(engineFood.items) } } - fun State.resume(): State = - when (this) { - State.Ended -> this - is State.Failure -> this - is State.Playing -> { - if (engineFood is EngineFood.AudioEngineFood) { - audioEngine.resume() - } - copy(paused = false) - } - } + return State( + playbackState = PlaybackState.Starved, + playWhenReady = playWhenReady, + node = engineFood.nodes[0], + engineFood = engineFood, + ) + } - fun State.jump(node: ReadAloudNode): State = - when (this) { - State.Ended -> start(node, paused = true) - is State.Failure -> this - is State.Playing -> start(node, paused = paused) - } + fun State.pause(): State { + audioEngine.playWhenReady = false + return copy(playWhenReady = true) + } + + fun State.resume(): State { + audioEngine.playWhenReady = true + return copy(playWhenReady = false) + } + + fun State.jump(node: ReadAloudNode): State { + val firstLeaf = node.firstLeaf() + ?: return copy(playbackState = PlaybackState.Ended) + + val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf) + ?: return copy(playbackState = PlaybackState.Ended) + + return play(engineFood, !playWhenReady) + } fun State.onEvent(event: Event): State = when (event) { - Event.AudioEngineEnded -> onAudioEngineEnded() + is Event.AudioEngineStateChanged -> onAudioEngineStateChanged(event.state) + is Event.AudioEngineItemChanged -> onAudioEngineItemChanged(event.index) } - fun State.onAudioEngineEnded(): State = - when (this) { - State.Ended -> this - is State.Failure -> this - is State.Playing -> State.Ended + fun State.onAudioEngineStateChanged(audioEngineState: AudioEngine.State): State = + when (audioEngineState) { + AudioEngine.State.Ready -> copy(playbackState = PlaybackState.Ready) + AudioEngine.State.Starved -> copy(playbackState = PlaybackState.Starved) + AudioEngine.State.Ended -> onAudioEngineEnded() + } + + private fun State.onAudioEngineEnded(): State { + var nextNode: ReadAloudLeafNode? = node + var nextFood: EngineFood? + + do { + nextNode = nextNode?.nextLeaf() + nextFood = nextNode?.let { EngineFood.AudioEngineFood.fromNode(it) } + } while (nextFood == null && nextNode != null) + + return if (nextNode == null) { + copy(playbackState = PlaybackState.Ended) + } else { + nextFood!! + audioEngine.setPlaylist(nextFood.items) + copy(engineFood = nextFood, playbackState = PlaybackState.Starved, node = nextFood.nodes[0]) } + } + + fun State.onAudioEngineItemChanged(item: Int): State { + return copy(node = (engineFood as EngineFood.AudioEngineFood).nodes[item]) + } } From 33abc5250981f66a6b3e51cb9d7b377acc7183b0 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 26 Aug 2025 15:03:06 +0200 Subject: [PATCH 08/13] Implement playback state and location --- .idea/kotlinc.xml | 2 +- .../navigator/reader/ReadAloudRendition.kt | 115 ++++++++++++------ .../navigator/common/LocationElements.kt | 6 + .../org/readium/navigator/common/Locations.kt | 9 ++ .../media/readaloud/build.gradle.kts | 1 + .../media/readaloud/ReadAloudLocations.kt | 86 +++++++++++++ .../media/readaloud/ReadAloudModel.kt | 32 ++++- .../media/readaloud/ReadAloudNavigator.kt | 97 +++++++++++++-- .../readaloud/ReadAloudNavigatorFactory.kt | 21 +++- .../media/readaloud/ReadAloudPublication.kt | 28 +++++ .../media/readaloud/ReadAloudStateMachine.kt | 8 +- 11 files changed, 351 insertions(+), 54 deletions(-) create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml index 131e44d798..c224ad564b 100644 --- a/.idea/kotlinc.xml +++ b/.idea/kotlinc.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt index 12775d825c..5c87d934ec 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt @@ -17,14 +17,18 @@ import androidx.compose.foundation.layout.padding import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.ArrowOutward import androidx.compose.material.icons.filled.Pause +import androidx.compose.material.icons.filled.PlayArrow import androidx.compose.material.icons.filled.SkipNext import androidx.compose.material.icons.filled.SkipPrevious import androidx.compose.material3.Icon import androidx.compose.material3.IconButton import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp import org.readium.r2.shared.ExperimentalReadiumApi @Composable @@ -38,49 +42,88 @@ fun ReadAloudRendition( modifier = Modifier .padding(contentPadding) .fillMaxSize(), - verticalArrangement = Arrangement.Bottom ) { - Row( - modifier = Modifier.fillMaxWidth(), - horizontalArrangement = Arrangement.Center, - verticalAlignment = Alignment.CenterVertically, + Column( + modifier = Modifier.weight(1f), + verticalArrangement = Arrangement.spacedBy(20.dp, alignment = Alignment.Top) ) { - IconButton( - onClick = { readerState.navigator.skip(force = true) } - ) { - Icon( - imageVector = Icons.Default.SkipPrevious, - contentDescription = "Skip to previous" - ) - } + val playbackState = readerState.navigator.playback.collectAsState() - IconButton( - onClick = { readerState.navigator.pause() } - ) { - Icon( - imageVector = Icons.Default.Pause, - contentDescription = "Pause" - ) - } + Text("Playback State: ${playbackState.value.state}") - IconButton( - onClick = { readerState.navigator.skip(force = true) } - ) { - Icon( - imageVector = Icons.Default.SkipNext, - contentDescription = "Skip to next" - ) - } + Text("Play When Ready: ${playbackState.value.playWhenReady}") + + Text("Resource Href: ${playbackState.value.utteranceLocation.href}") + + Text("Utterance Selector ${playbackState.value.utteranceLocation.cssSelector?.value}") + } + + Toolbar(readerState) + } + } +} + +@Composable +private fun Toolbar( + readerState: ReadAloudReaderState, +) { + val playbackState = readerState.navigator.playback.collectAsState() - IconButton( - onClick = { readerState.navigator.escape(force = true) } - ) { - Icon( - imageVector = Icons.Default.ArrowOutward, - contentDescription = "Escape" - ) + Row( + modifier = Modifier + .fillMaxWidth(), + horizontalArrangement = Arrangement.Center, + verticalAlignment = Alignment.Bottom, + ) { + IconButton( + onClick = { readerState.navigator.skipToPrevious(force = true) } + ) { + Icon( + imageVector = Icons.Default.SkipPrevious, + contentDescription = "Skip to previous" + ) + } + + if (playbackState.value.playWhenReady) { + IconButton( + onClick = { + readerState.navigator.pause() } + ) { + Icon( + imageVector = Icons.Default.Pause, + contentDescription = "Pause" + ) + } + } else { + IconButton( + onClick = { + readerState.navigator.play() + } + ) { + Icon( + imageVector = Icons.Default.PlayArrow, + contentDescription = "Play" + ) } } + + IconButton( + onClick = { readerState.navigator.skipToNext(force = true) } + ) { + Icon( + imageVector = Icons.Default.SkipNext, + contentDescription = "Skip to next" + ) + } + + IconButton( + onClick = { readerState.navigator.escape(force = true) } + ) { + Icon( + imageVector = Icons.Default.ArrowOutward, + contentDescription = "Escape" + ) + } } } diff --git a/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt b/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt index 7164bcea2c..6da3d29421 100644 --- a/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt +++ b/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt @@ -60,3 +60,9 @@ public data class TextQuote( val prefix: String, val suffix: String, ) + +@ExperimentalReadiumApi +public data class TextAnchor( + val prefix: String, + val suffix: String, +) diff --git a/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt b/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt index 97c9ba97a4..b1f1ec9145 100644 --- a/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt +++ b/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt @@ -37,6 +37,15 @@ public interface TextQuoteLocation : Location { public val textQuote: TextQuote } +/** + * A [Location] including a [TextAnchor]. + */ +@ExperimentalReadiumApi +public interface TextAnchorLocation : Location { + + public val textAnchor: TextAnchor +} + /** * A [Location] including a [CssSelector]. */ diff --git a/readium/navigators/media/readaloud/build.gradle.kts b/readium/navigators/media/readaloud/build.gradle.kts index b1b021874c..f100c98a4f 100644 --- a/readium/navigators/media/readaloud/build.gradle.kts +++ b/readium/navigators/media/readaloud/build.gradle.kts @@ -14,6 +14,7 @@ android { } dependencies { + api(project(":readium:navigators:readium-navigator-common")) api(project(":readium:navigators:media:readium-navigator-media-common")) implementation(libs.androidx.media3.common) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt new file mode 100644 index 0000000000..081e429c75 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.navigator.common.CssSelector +import org.readium.navigator.common.CssSelectorLocation +import org.readium.navigator.common.ExportableLocation +import org.readium.navigator.common.GoLocation +import org.readium.navigator.common.Location +import org.readium.navigator.common.TextAnchor +import org.readium.navigator.common.TextAnchorLocation +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.Locator.Locations +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.mediatype.MediaType + +@ExperimentalReadiumApi +public data class ReadAloudGoLocation( + override val href: Url, + val cssSelector: CssSelector?, + val textAnchor: TextAnchor?, +) : GoLocation { + + public constructor(location: Location) : this( + href = location.href, + cssSelector = (location as? CssSelectorLocation)?.cssSelector, + textAnchor = (location as? TextAnchorLocation)?.textAnchor + ) +} + +@ExperimentalReadiumApi +public sealed interface ReadAloudLocation : ExportableLocation { + override val href: Url + public val textAnchor: TextAnchor? + public val cssSelector: CssSelector? +} + +@ExperimentalReadiumApi +internal data class MediaOverlaysLocation( + override val href: Url, + private val mediaType: MediaType?, + override val cssSelector: CssSelector?, +) : ReadAloudLocation, CssSelectorLocation { + + override fun toLocator(): Locator = + Locator( + href = href, + mediaType = mediaType ?: MediaType.XHTML, + locations = Locations() // TODO + ) + + override val textAnchor: TextAnchor? = null +} + +@ExperimentalReadiumApi +internal data class TtsLocation( + override val href: Url, + private val mediaType: MediaType?, + override val cssSelector: CssSelector?, + override val textAnchor: TextAnchor, +) : ReadAloudLocation, CssSelectorLocation, TextAnchorLocation { + + override fun toLocator(): Locator = + Locator( + href = href, + mediaType = mediaType ?: MediaType.XHTML, + locations = Locations() // TODO + ) +} + +@ExperimentalReadiumApi +public data class UtteranceLocation( + override val href: Url, + private val mediaType: MediaType?, + override val cssSelector: CssSelector?, +) : ExportableLocation, CssSelectorLocation { + + override fun toLocator(): Locator { + TODO("Not yet implemented") + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt index 71e8c814fb..3348a2b4a8 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt @@ -49,6 +49,22 @@ public data class ReadAloudLeafNode( internal set } +@ExperimentalReadiumApi +internal fun ReadAloudNode.firstMatchingLocation(location: ReadAloudGoLocation): ReadAloudLeafNode? = + firstDescendantOrNull { + when (it) { + is ReadAloudInnerNode -> false + is ReadAloudLeafNode -> it.matchLocation(location) + } + } as? ReadAloudLeafNode + +@ExperimentalReadiumApi +private fun ReadAloudLeafNode.matchLocation(location: ReadAloudGoLocation): Boolean = + refs.any { ref -> + ref.url.removeFragment() == location.href && + ref.url.fragment == location.cssSelector?.value?.removePrefix("#") + } + @ExperimentalReadiumApi internal fun ReadAloudNode.isSkippable(): Boolean = nearestSkippable() != null @@ -116,9 +132,13 @@ internal fun ReadAloudNode.escape(force: Boolean): ReadAloudNode? = (nearestEscapable() ?: this.takeIf { force })?.skipToNext() @ExperimentalReadiumApi -internal fun ReadAloudNode.skip(force: Boolean): ReadAloudNode? = +internal fun ReadAloudNode.skipToNext(force: Boolean): ReadAloudNode? = (nearestSkippable() ?: this.takeIf { force })?.skipToNext() +@ExperimentalReadiumApi +internal fun ReadAloudNode.skipToPrevious(force: Boolean): ReadAloudNode? = + (nearestSkippable() ?: this.takeIf { force })?.skipToPrevious() + @ExperimentalReadiumApi private fun ReadAloudNode.nearestEscapable(): ReadAloudNode? = nearestOrNull { roles.any { it in GuidedNavigationRole.ESCAPABLE_ROLES } } @@ -136,3 +156,13 @@ private fun ReadAloudNode.nearestOrNull( parent == null -> null else -> parent!!.nearestOrNull(predicate) } + +@ExperimentalReadiumApi +private fun ReadAloudNode.firstDescendantOrNull( + predicate: (ReadAloudNode) -> Boolean, +): ReadAloudNode? = + when { + predicate(this) -> this + children.isEmpty() -> null + else -> children.firstNotNullOfOrNull { it.firstDescendantOrNull(predicate) } + } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index ff73b9a308..d0fb0c1d0d 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -8,30 +8,45 @@ import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.withContext +import org.readium.navigator.common.CssSelector import org.readium.navigator.media.common.MediaNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.addPrefix import org.readium.r2.shared.extensions.mapStateIn -import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.data.ReadError @ExperimentalReadiumApi public class ReadAloudNavigator private constructor( firstLeaf: ReadAloudLeafNode, + private val guidedNavigationTree: ReadAloudInnerNode, + private val resources: List, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, ) { - public companion object { - public suspend operator fun invoke( - guidedNavigationTree: GuidedNavigationObject, + internal suspend operator fun invoke( + initialLocation: ReadAloudGoLocation?, + publication: ReadAloudPublication, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, ): ReadAloudNavigator { val tree = withContext(Dispatchers.Default) { - GuidedNavigationAdapter().adapt(guidedNavigationTree) + GuidedNavigationAdapter().adapt(publication.guidedNavigationTree) } - val firstLeaf = checkNotNull(tree.firstLeaf()) - return ReadAloudNavigator(firstLeaf, audioEngineFactory) + val initialLeaf = initialLocation + ?.let { tree.firstMatchingLocation(it) } + ?: tree.firstLeaf() + + checkNotNull(initialLeaf) + + return ReadAloudNavigator( + firstLeaf = initialLeaf, + guidedNavigationTree = tree, + resources = publication.resources, + audioEngineFactory = audioEngineFactory + ) } } @@ -39,6 +54,7 @@ public class ReadAloudNavigator private constructor( val state: State, val playWhenReady: Boolean, val node: ReadAloudLeafNode, + val utteranceLocation: UtteranceLocation, ) public sealed interface State { @@ -96,10 +112,24 @@ public class ReadAloudNavigator private constructor( Playback( playWhenReady = state.playWhenReady, state = state.playbackState.toState(), - node = state.node + node = state.node, + utteranceLocation = state.node.utteranceLocation ) } + private val ReadAloudLeafNode.utteranceLocation: UtteranceLocation get() { + val textref = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + checkNotNull(textref) + val href = textref.url.removeFragment() + val cssSelector = textref.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + return UtteranceLocation( + href = href, + mediaType = resources.first { item -> item.href == href }.mediaType, + cssSelector = cssSelector + ) + } + private fun ReadAloudStateMachine.PlaybackState.toState(): State = when (this) { is ReadAloudStateMachine.PlaybackState.Ready -> @@ -141,8 +171,55 @@ public class ReadAloudNavigator private constructor( ?.let { go(it) } } - public fun skip(force: Boolean = true) { - stateMutable.value.node.skip(force) + public fun skipToPrevious(force: Boolean = true) { + stateMutable.value.node.skipToPrevious(force) ?.let { go(it) } } + + public fun skipToNext(force: Boolean = true) { + stateMutable.value.node.skipToNext(force) + ?.let { go(it) } + } + + public val location: StateFlow = + stateMutable.mapStateIn(coroutineScope) { state -> + val textref = state.node.refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + checkNotNull(textref) + val href = textref.url.removeFragment() + val cssSelector = textref.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + MediaOverlaysLocation( + href = href, + mediaType = resources.first { item -> item.href == href }.mediaType, + cssSelector = cssSelector + ) + } + + public fun goTo(location: ReadAloudGoLocation) { + guidedNavigationTree.firstMatchingLocation(location) + ?.let { go(it) } + } + + public fun goTo(location: ReadAloudLocation) { + val goLocation = when (location) { + is MediaOverlaysLocation -> + ReadAloudGoLocation( + href = location.href, + cssSelector = location.cssSelector, + textAnchor = location.textAnchor + ) + is TtsLocation -> + throw IllegalStateException() + } + goTo(goLocation) + } + + public fun goTo(url: Url) { + val location = ReadAloudGoLocation( + href = url.removeFragment(), + cssSelector = url.fragment?.let { CssSelector(it.addPrefix("#")) }, + textAnchor = null + ) + goTo(location) + } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index 800ecbb542..622c5b0649 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -19,6 +19,7 @@ import org.readium.r2.shared.util.getOrElse @ExperimentalReadiumApi public class ReadAloudNavigatorFactory private constructor( private val guidedNavigationService: GuidedNavigationService, + private val resources: List, private val audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, ) { @@ -36,8 +37,16 @@ public class ReadAloudNavigatorFactory private constructor( audioEngineProvider.createEngine(publication, listener) } + val resources = (publication.readingOrder + publication.resources).map { + ReadAloudPublication.Item( + href = it.url(), + mediaType = it.mediaType + ) + } + return ReadAloudNavigatorFactory( guidedNavigationService = guidedNavService, + resources = resources, audioEngineFactory = audioEngineFactory ) } @@ -57,7 +66,9 @@ public class ReadAloudNavigatorFactory private constructor( ) : Error("Failed to acquire guided navigation documents.", cause) } - public suspend fun createNavigator(): Try { + public suspend fun createNavigator( + initialLocation: ReadAloudGoLocation? = null, + ): Try { val guidedDocs = buildList { val iterator = guidedNavigationService.iterator() while (iterator.hasNext()) { @@ -77,8 +88,14 @@ public class ReadAloudNavigatorFactory private constructor( text = null ) - val navigator = ReadAloudNavigator( + val navigatorPublication = ReadAloudPublication( guidedNavigationTree = guidedTree, + resources = resources, + ) + + val navigator = ReadAloudNavigator( + initialLocation = initialLocation, + publication = navigatorPublication, audioEngineFactory = audioEngineFactory ) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt new file mode 100644 index 0000000000..e5e3db0fc6 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.mediatype.MediaType + +internal class ReadAloudPublication( + val guidedNavigationTree: GuidedNavigationObject, + val resources: List, +) { + data class Item( + val href: Url, + val mediaType: MediaType?, + ) + + val mediaTypes = resources + .mapNotNull { item -> item.mediaType?.let { item.href to it } } + .associate { it } + + fun itemWithHref(href: Url): Item? = + resources.firstOrNull { it.href == href } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 6130ea3185..80943a8dec 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -108,7 +108,7 @@ internal class ReadAloudStateMachine( fun play(engineFood: EngineFood, playWhenReady: Boolean): State { when (engineFood) { is EngineFood.AudioEngineFood -> { - audioEngine.playWhenReady = !playWhenReady + audioEngine.playWhenReady = playWhenReady audioEngine.setPlaylist(engineFood.items) } } @@ -123,12 +123,12 @@ internal class ReadAloudStateMachine( fun State.pause(): State { audioEngine.playWhenReady = false - return copy(playWhenReady = true) + return copy(playWhenReady = false) } fun State.resume(): State { audioEngine.playWhenReady = true - return copy(playWhenReady = false) + return copy(playWhenReady = true) } fun State.jump(node: ReadAloudNode): State { @@ -138,7 +138,7 @@ internal class ReadAloudStateMachine( val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf) ?: return copy(playbackState = PlaybackState.Ended) - return play(engineFood, !playWhenReady) + return play(engineFood, playWhenReady) } fun State.onEvent(event: Event): State = when (event) { From 50173a78a30a82f5092bd25350039436f2af8e19 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 2 Sep 2025 09:19:25 +0200 Subject: [PATCH 09/13] Revisit guided navigation tree --- .../readium/demo/navigator/DemoViewModel.kt | 4 +- .../demo/navigator/reader/ReaderOpener.kt | 16 +- .../exoplayer/readaloud/ExoPlayerEngine.kt | 10 +- .../navigator/media/readaloud/AudioEngine.kt | 47 ----- .../ContentIteratorGuidedNavigationService.kt | 20 ++ .../readaloud/GuidedNavigationAdapter.kt | 61 ------ .../media/readaloud/PlaybackEngine.kt | 52 ------ .../media/readaloud/ReadAloudDataLoader.kt | 68 +++++++ .../media/readaloud/ReadAloudEngine.kt | 112 ++++++++++++ .../media/readaloud/ReadAloudModel.kt | 168 ----------------- .../readaloud/ReadAloudNavigationHelper.kt | 89 +++++++++ .../media/readaloud/ReadAloudNavigator.kt | 111 +++++++---- .../readaloud/ReadAloudNavigatorFactory.kt | 32 +++- .../media/readaloud/ReadAloudNode.kt | 122 ++++++++++++ .../media/readaloud/ReadAloudPublication.kt | 3 + .../media/readaloud/ReadAloudSegment.kt | 152 +++++++++++++++ .../media/readaloud/ReadAloudSettings.kt | 23 +++ .../media/readaloud/ReadAloudStateMachine.kt | 173 +++++++----------- .../shared/guided/GuidedNavigationDocument.kt | 34 ++++ .../r2/shared/guided/GuidedNavigationRole.kt | 57 +++++- .../r2/shared/publication/epub/Services.kt | 34 ++++ .../services/GuidedNavigationService.kt | 17 +- .../r2/streamer/parser/epub/EpubParser.kt | 4 +- .../parser/epub/MediaOverlaysService.kt | 66 +++++-- .../parser/epub/MediaOverlaysServiceTest.kt | 136 ++++++++++++++ .../parser/epub/smil/chapter_001_overlay.smil | 139 ++++++++++++++ .../parser/epub/smil/chapter_002_overlay.smil | 70 +++++++ 27 files changed, 1324 insertions(+), 496 deletions(-) delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt create mode 100644 readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt create mode 100644 readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt create mode 100644 readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil create mode 100644 readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index 68360c11ad..48deb99fe3 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -23,6 +23,7 @@ import org.readium.demo.navigator.reader.SelectNavigatorItem import org.readium.demo.navigator.reader.SelectNavigatorViewModel import org.readium.demo.navigator.reader.fixedConfig import org.readium.demo.navigator.reader.reflowableConfig +import org.readium.navigator.media.readaloud.NullTtsEngineProvider import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory @@ -127,7 +128,8 @@ class DemoViewModel( ReadAloudNavigatorFactory( application = application, publication = publication, - audioEngineProvider = audioEngineProvider + audioEngineProvider = audioEngineProvider, + ttsEngineProvider = NullTtsEngineProvider )?.let { SelectNavigatorItem.ReadAloud(it) } val factories = listOfNotNull( diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index 075e22cd53..45423f988d 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -27,6 +27,7 @@ import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings import org.readium.navigator.common.SettingsController import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.media.readaloud.ReadAloudSettings import org.readium.navigator.web.fixedlayout.FixedWebGoLocation import org.readium.navigator.web.fixedlayout.FixedWebLocation import org.readium.navigator.web.fixedlayout.FixedWebRenditionController @@ -40,10 +41,12 @@ import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebSelectionLocation import org.readium.navigator.web.reflowable.preferences.ReflowableWebPreferences import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl import org.readium.r2.shared.util.Error +import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.getOrElse @@ -197,7 +200,18 @@ class ReaderOpener( navigatorFactory: ReadAloudNavigatorFactory, initialLocator: Locator?, ): Try { - val navigator = navigatorFactory.createNavigator() + val initialSettings = ReadAloudSettings( + language = Language("en"), + overrideContentLanguage = true, + preferRecordedVoices = true, + pitch = 1.0, + speed = 1.0, + voices = emptyMap(), + escapableRoles = GuidedNavigationRole.ESCAPABLE_ROLES.toSet(), + skippableRoles = GuidedNavigationRole.SKIPPABLE_ROLES.toSet() + ) + + val navigator = navigatorFactory.createNavigator(initialSettings) .getOrElse { return Try.failure(it) } val readerState = ReadAloudReaderState( diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index ec292cd656..75c78f98c7 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -59,6 +59,8 @@ public class ExoPlayerEngine private constructor( .setHandleAudioBecomingNoisy(true) .build() + exoPlayer.preloadConfiguration = ExoPlayer.PreloadConfiguration(10_000_000L) + return ExoPlayerEngine(exoPlayer, listener) } } @@ -76,11 +78,11 @@ public class ExoPlayerEngine private constructor( Player.STATE_ENDED -> AudioEngine.State.Ended else -> null } - newState?.let { this@ExoPlayerEngine.listener.onStateChanged(it) } + newState?.let { this@ExoPlayerEngine.listener.onStateChanged(this@ExoPlayerEngine, it) } } if (events.contains(Player.EVENT_MEDIA_ITEM_TRANSITION)) { - this@ExoPlayerEngine.listener.onItemChanged(player.currentMediaItemIndex) + this@ExoPlayerEngine.listener.onItemChanged(this@ExoPlayerEngine, player.currentMediaItemIndex) } } } @@ -128,6 +130,10 @@ public class ExoPlayerEngine private constructor( } } + override fun seekTo(index: Int) { + exoPlayer.seekTo(index, 0L) + } + override var playWhenReady: Boolean get() = exoPlayer.playWhenReady set(value) { diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt deleted file mode 100644 index ed135d42c2..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.util.TimeInterval -import org.readium.r2.shared.util.Url - -@ExperimentalReadiumApi -public interface AudioEngine { - - public interface Listener { - - public fun onItemChanged(index: Int) - - public fun onStateChanged(state: State) - } - - public enum class State { - Ready, - Starved, - Ended, - } - - public data class Item( - val href: Url, - val interval: TimeInterval?, - ) - - public var playWhenReady: Boolean - - public fun setPlaylist(items: List) -} - -@ExperimentalReadiumApi -public interface AudioEngineProvider { - - public fun createEngine( - publication: Publication, - listener: AudioEngine.Listener, - ): AudioEngine -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt new file mode 100644 index 0000000000..5c2495f07a --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.publication.services.GuidedNavigationService + +internal class ContentIteratorGuidedNavigationService : GuidedNavigationService { + + override fun iterator(): GuidedNavigationIterator { + TODO("Not yet implemented") + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt deleted file mode 100644 index 5495e97035..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/GuidedNavigationAdapter.kt +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationObject - -@OptIn(ExperimentalReadiumApi::class) -internal class GuidedNavigationAdapter { - - fun adapt(guidedNavTree: GuidedNavigationObject): ReadAloudInnerNode { - val children = guidedNavTree.children.mapNotNull { adaptNode(it) } - val node = ReadAloudInnerNode( - children = children, - roles = guidedNavTree.roles, - refs = guidedNavTree.refs - ) - setParentInChildren(node) - return node - } - - private fun adaptNode(guidedNavigationObject: GuidedNavigationObject): ReadAloudNode? { - return when (guidedNavigationObject.children.size) { - 0 -> - adaptLeat(guidedNavigationObject) - else -> - guidedNavigationObject.children - .mapNotNull { adaptNode(it) } - .takeIf { it.isNotEmpty() } - ?.let { - ReadAloudInnerNode( - children = it, - roles = guidedNavigationObject.roles, - refs = guidedNavigationObject.refs - ) - } - ?.also { setParentInChildren(it) } - } - } - - private fun adaptLeat(guidedNavigationLeaf: GuidedNavigationObject): ReadAloudLeafNode? { - return ReadAloudLeafNode( - text = guidedNavigationLeaf.text, - refs = guidedNavigationLeaf.refs, - roles = guidedNavigationLeaf.roles - ) - } - - private fun setParentInChildren(parent: ReadAloudInnerNode) { - parent.children.forEach { - when (it) { - is ReadAloudInnerNode -> it.parent = parent - is ReadAloudLeafNode -> it.parent = parent - } - } - } -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt deleted file mode 100644 index 75e6818974..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Language - -@ExperimentalReadiumApi -public interface PlaybackEngine { - - public interface Listener { - - public fun onUtterancesReady() - - public fun onPlaybackCompleted() - } - - public fun feed(utterances: List) - - public fun speak(utteranceIndex: Int) -} - -@ExperimentalReadiumApi -public interface PausablePlaybackEngine : PlaybackEngine { - - public fun pause() - - public fun resume() -} - -@ExperimentalReadiumApi -public interface PlaybackEngineProvider { - - public interface Voice { - - /** - * The languages supported by the voice. - */ - public val languages: Set - } - - /** - * Sets of voices available with this [PlaybackEngine]. - */ - public val voices: Set - - public fun createEngine(voice: V): PlaybackEngine -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt new file mode 100644 index 0000000000..b030877289 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt @@ -0,0 +1,68 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import kotlin.properties.Delegates +import org.readium.r2.shared.ExperimentalReadiumApi + +internal class ReadAloudDataLoader( + private val segmentFactory: ReadAloudSegmentFactory, + initialSettings: ReadAloudSettings, +) { + + sealed interface NodeInfo + + data class ItemRef( + val segment: ReadAloudSegment, + val nodeIndex: Int?, + ) : NodeInfo + + data object EmptyNode : NodeInfo + + var settings by Delegates.observable(initialSettings) { property, oldValue, newValue -> + preloadedRefs.clear() + } + + private val preloadedRefs: MutableMap = mutableMapOf() + + fun onPlaybackProgressed(node: ReadAloudNode) { + val nextNode = node.next() ?: return + + if (nextNode !in preloadedRefs) { + loadSegmentForNode(nextNode) + } + } + + fun getItemRef(node: ReadAloudNode): ItemRef? { + loadSegmentForNode(node) + return preloadedRefs[node] + } + + private fun loadSegmentForNode(node: ReadAloudNode) { + if (node in preloadedRefs) { + return + } + + val segment = segmentFactory.createSegmentFromNode(node) + ?: return // Ended + + val refs = computeRefsForSegment(segment) + preloadedRefs.putAll(refs) + } + + private fun computeRefsForSegment(segment: ReadAloudSegment): Map { + val plainRefs = segment.nodes + .withIndex() + .associate { (index, node) -> node to ItemRef(segment, index) } + + val emptyRefs = segment.emptyNodes.associateWith { ItemRef(segment, null) } + + return plainRefs + emptyRefs + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt new file mode 100644 index 0000000000..131e03dbb0 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt @@ -0,0 +1,112 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +@ExperimentalReadiumApi +public sealed interface ReadAloudEngine { + + public var playWhenReady: Boolean +} + +@ExperimentalReadiumApi +public interface TtsEngine : ReadAloudEngine { + + public interface Listener { + + public fun onUtterancesReady() + + public fun onPlaybackCompleted() + } + + public fun feed(utterances: List) + + public fun speak(utteranceIndex: Int) +} + +@ExperimentalReadiumApi +public interface PausableTtsEngine : TtsEngine { + + public fun pause() + + public fun resume() +} + +@ExperimentalReadiumApi +public interface TtsVoice { + + @JvmInline + public value class Id(public val value: String) + + public val id: Id + + /** + * The languages supported by the voice. + */ + public val languages: Set +} + +@ExperimentalReadiumApi +public interface TtsEngineProvider { + + /** + * Sets of voices available with this [TtsEngineProvider]. + */ + public val voices: Set + + public fun createEngine(voice: TtsVoice): TtsEngine +} + +@ExperimentalReadiumApi +public object NullTtsEngineProvider : TtsEngineProvider { + + override val voices: Set = emptySet() + + override fun createEngine(voice: TtsVoice): TtsEngine { + throw IllegalStateException() + } +} + +@ExperimentalReadiumApi +public interface AudioEngine : ReadAloudEngine { + + public interface Listener { + + public fun onItemChanged(engine: AudioEngine, index: Int) + + public fun onStateChanged(engine: AudioEngine, state: State) + } + + public enum class State { + Ready, + Starved, + Ended, + } + + public data class Item( + val href: Url, + val interval: TimeInterval?, + ) + + public fun setPlaylist(items: List) + + public fun seekTo(index: Int) +} + +@ExperimentalReadiumApi +public interface AudioEngineProvider { + + public fun createEngine( + publication: Publication, + listener: AudioEngine.Listener, + ): AudioEngine +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt deleted file mode 100644 index 3348a2b4a8..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudModel.kt +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationRef -import org.readium.r2.shared.guided.GuidedNavigationRole -import org.readium.r2.shared.guided.GuidedNavigationText - -// Modèle différent pour pouvoir garder une référence sur le noeud parent. -@ExperimentalReadiumApi -public sealed interface ReadAloudNode { - - public val roles: Set - - public val refs: Set - - public val parent: ReadAloudNode? - - public val children: List -} - -@ExperimentalReadiumApi -public class ReadAloudInnerNode( - override val children: List, - override val roles: Set, - override val refs: Set, -) : ReadAloudNode { - - override var parent: ReadAloudNode? = null - internal set -} - -@ExperimentalReadiumApi -public data class ReadAloudLeafNode( - val text: GuidedNavigationText?, - override val refs: Set, - override val roles: Set, -) : ReadAloudNode { - - override val children: List = - emptyList() - - override lateinit var parent: ReadAloudInnerNode - internal set -} - -@ExperimentalReadiumApi -internal fun ReadAloudNode.firstMatchingLocation(location: ReadAloudGoLocation): ReadAloudLeafNode? = - firstDescendantOrNull { - when (it) { - is ReadAloudInnerNode -> false - is ReadAloudLeafNode -> it.matchLocation(location) - } - } as? ReadAloudLeafNode - -@ExperimentalReadiumApi -private fun ReadAloudLeafNode.matchLocation(location: ReadAloudGoLocation): Boolean = - refs.any { ref -> - ref.url.removeFragment() == location.href && - ref.url.fragment == location.cssSelector?.value?.removePrefix("#") - } - -@ExperimentalReadiumApi -internal fun ReadAloudNode.isSkippable(): Boolean = - nearestSkippable() != null - -@ExperimentalReadiumApi -internal fun ReadAloudNode.isEscapable(): Boolean = - nearestEscapable() != null - -@ExperimentalReadiumApi -internal fun ReadAloudNode.firstLeaf(): ReadAloudLeafNode? = - when (this) { - is ReadAloudLeafNode -> this - is ReadAloudInnerNode -> children[0].firstLeaf() - } - -@ExperimentalReadiumApi -internal fun ReadAloudNode.nextLeaf(): ReadAloudLeafNode? = - when (this) { - is ReadAloudInnerNode -> firstLeaf() - is ReadAloudLeafNode -> { - val siblings = parent.children - val currentIndex = siblings.indexOf(this) - check(currentIndex != -1) - - if (currentIndex < siblings.size - 1) { - val sister = siblings[currentIndex + 1] - when (sister) { - is ReadAloudInnerNode -> sister.firstLeaf() - is ReadAloudLeafNode -> sister - } - } else { - val next = parent.skipToNext() ?: return null - when (next) { - is ReadAloudInnerNode -> next.firstLeaf() - is ReadAloudLeafNode -> next - } - } - } - } - -@ExperimentalReadiumApi -internal fun ReadAloudNode.skipToNext(): ReadAloudNode? { - val siblings = parent?.children ?: return null - val currentIndex = siblings.indexOf(this) - check(currentIndex != -1) - return currentIndex - .takeIf { it < siblings.size - 1 } - ?.let { siblings[currentIndex + 1] } - ?: parent!!.skipToNext() -} - -@ExperimentalReadiumApi -internal fun ReadAloudNode.skipToPrevious(): ReadAloudNode? { - val siblings = parent?.children ?: return null - val currentIndex = siblings.indexOf(this) - check(currentIndex != -1) - return currentIndex - .takeIf { it > 0 } - ?.let { siblings[currentIndex - 1] } - ?: parent!!.skipToPrevious() -} - -@ExperimentalReadiumApi -internal fun ReadAloudNode.escape(force: Boolean): ReadAloudNode? = - (nearestEscapable() ?: this.takeIf { force })?.skipToNext() - -@ExperimentalReadiumApi -internal fun ReadAloudNode.skipToNext(force: Boolean): ReadAloudNode? = - (nearestSkippable() ?: this.takeIf { force })?.skipToNext() - -@ExperimentalReadiumApi -internal fun ReadAloudNode.skipToPrevious(force: Boolean): ReadAloudNode? = - (nearestSkippable() ?: this.takeIf { force })?.skipToPrevious() - -@ExperimentalReadiumApi -private fun ReadAloudNode.nearestEscapable(): ReadAloudNode? = - nearestOrNull { roles.any { it in GuidedNavigationRole.ESCAPABLE_ROLES } } - -@ExperimentalReadiumApi -private fun ReadAloudNode.nearestSkippable(): ReadAloudNode? = - nearestOrNull { roles.any { it in GuidedNavigationRole.SKIPPABLE_ROLES } } - -@ExperimentalReadiumApi -private fun ReadAloudNode.nearestOrNull( - predicate: (ReadAloudNode) -> Boolean, -): ReadAloudNode? = - when { - predicate(this) -> this - parent == null -> null - else -> parent!!.nearestOrNull(predicate) - } - -@ExperimentalReadiumApi -private fun ReadAloudNode.firstDescendantOrNull( - predicate: (ReadAloudNode) -> Boolean, -): ReadAloudNode? = - when { - predicate(this) -> this - children.isEmpty() -> null - else -> children.firstNotNullOfOrNull { it.firstDescendantOrNull(predicate) } - } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt new file mode 100644 index 0000000000..aa85026a82 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt @@ -0,0 +1,89 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef + +internal class ReadAloudNavigationHelper( + var settings: ReadAloudSettings, +) { + + fun ReadAloudNode.firstMatchingLocation(location: ReadAloudGoLocation): ReadAloudNode? = + firstDescendantOrNull { it.matchLocation(location) } + + private fun ReadAloudNode.matchLocation(location: ReadAloudGoLocation): Boolean = + refs.any { ref -> + ref.url.removeFragment() == location.href && + ref.url.fragment == location.cssSelector?.value?.removePrefix("#") + } + + fun ReadAloudNode.isSkippable(): Boolean = + nearestSkippable() != null + + fun ReadAloudNode.isEscapable(): Boolean = + nearestEscapable() != null + + fun ReadAloudNode.firstContentNode(): ReadAloudNode? = + firstDescendantOrNull { it.hasContent() } + + fun ReadAloudNode.hasContent() = + text != null || refs.firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } != null + + fun ReadAloudNode.nextContentNode(): ReadAloudNode? { + val parent = parent + ?: return children.firstOrNull()?.nextContentNode() + + val siblings = parent.children + + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + val next = if (currentIndex < siblings.size - 1) { + siblings[currentIndex + 1] + } else { + parent.next() + } + + return next?.firstContentNode() + } + + fun ReadAloudNode.escape(force: Boolean): ReadAloudNode? = + (nearestEscapable() ?: this.takeIf { force })?.next() + + fun ReadAloudNode.skipToNext(force: Boolean): ReadAloudNode? = + (nearestSkippable() ?: this.takeIf { force })?.next() + + fun ReadAloudNode.skipToPrevious(force: Boolean): ReadAloudNode? = + (nearestSkippable() ?: this.takeIf { force })?.previous() + + private fun ReadAloudNode.nearestEscapable(): ReadAloudNode? = + nearestOrNull { roles.any { it in settings.escapableRoles } } + + private fun ReadAloudNode.nearestSkippable(): ReadAloudNode? = + nearestOrNull { roles.any { it in settings.skippableRoles } } + + private fun ReadAloudNode.nearestOrNull( + predicate: (ReadAloudNode) -> Boolean, + ): ReadAloudNode? = + when { + predicate(this) -> this + parent == null -> null + else -> parent!!.nearestOrNull(predicate) + } + + private fun ReadAloudNode.firstDescendantOrNull( + predicate: (ReadAloudNode) -> Boolean, + ): ReadAloudNode? = + when { + predicate(this) -> this + children.isEmpty() -> null + else -> children.firstNotNullOfOrNull { it.firstDescendantOrNull(predicate) } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index d0fb0c1d0d..9c9d6495c4 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -1,7 +1,14 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + @file:OptIn(InternalReadiumApi::class) package org.readium.navigator.media.readaloud +import kotlin.properties.Delegates import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.MainScope @@ -20,32 +27,33 @@ import org.readium.r2.shared.util.data.ReadError @ExperimentalReadiumApi public class ReadAloudNavigator private constructor( - firstLeaf: ReadAloudLeafNode, - private val guidedNavigationTree: ReadAloudInnerNode, + private val guidedNavigationTree: ReadAloudNode, private val resources: List, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, + ttsEngineFactory: () -> TtsEngine, + initialSettings: ReadAloudSettings, + initialLocation: ReadAloudGoLocation?, ) { public companion object { internal suspend operator fun invoke( initialLocation: ReadAloudGoLocation?, + initialSettings: ReadAloudSettings, publication: ReadAloudPublication, audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, + ttsEngineFactory: () -> TtsEngine, ): ReadAloudNavigator { val tree = withContext(Dispatchers.Default) { - GuidedNavigationAdapter().adapt(publication.guidedNavigationTree) + ReadAloudNode.fromGuidedNavigationObject(publication.guidedNavigationTree) } - val initialLeaf = initialLocation - ?.let { tree.firstMatchingLocation(it) } - ?: tree.firstLeaf() - - checkNotNull(initialLeaf) return ReadAloudNavigator( - firstLeaf = initialLeaf, guidedNavigationTree = tree, resources = publication.resources, - audioEngineFactory = audioEngineFactory + audioEngineFactory = audioEngineFactory, + ttsEngineFactory = ttsEngineFactory, + initialSettings = initialSettings, + initialLocation = initialLocation ) } } @@ -53,7 +61,7 @@ public class ReadAloudNavigator private constructor( public data class Playback( val state: State, val playWhenReady: Boolean, - val node: ReadAloudLeafNode, + val node: ReadAloudNode, val utteranceLocation: UtteranceLocation, ) @@ -61,7 +69,7 @@ public class ReadAloudNavigator private constructor( public data object Ready : State, MediaNavigator.State.Ready - public data object Buffering : State, MediaNavigator.State.Buffering + public data object Starved : State, MediaNavigator.State.Buffering public data object Ended : State, MediaNavigator.State.Ended @@ -82,28 +90,46 @@ public class ReadAloudNavigator private constructor( private inner class AudioEngineListener : AudioEngine.Listener { - override fun onItemChanged(index: Int) { + override fun onItemChanged(engine: AudioEngine, index: Int) { with(stateMachine) { - stateMutable.value = stateMutable.value.onAudioEngineItemChanged(index) + stateMutable.value = stateMutable.value.onAudioEngineItemChanged(engine, index) } } - override fun onStateChanged(state: AudioEngine.State) { + override fun onStateChanged(engine: AudioEngine, state: AudioEngine.State) { with(stateMachine) { - stateMutable.value = stateMutable.value.onAudioEngineStateChanged(state) + stateMutable.value = stateMutable.value.onAudioEngineStateChanged(engine, state) } } } - private val audioEngine = audioEngineFactory(AudioEngineListener()) + private val segmentFactory = ReadAloudSegmentFactory( + audioEngineFactory = { audioEngineFactory(AudioEngineListener()) }, + ttsEngineFactory = ttsEngineFactory + ) + + private val dataLoader = ReadAloudDataLoader(segmentFactory, initialSettings) + + private val navigationHelper = ReadAloudNavigationHelper(initialSettings) - private val stateMachine = ReadAloudStateMachine(audioEngine) + private val stateMachine = ReadAloudStateMachine(dataLoader, navigationHelper) - private val stateMutable: MutableStateFlow = run { - val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf)!! - MutableStateFlow(stateMachine.play(engineFood, playWhenReady = true)) + private val initialNode = with(navigationHelper) { + val nodeFromLocation = initialLocation + ?.let { guidedNavigationTree.firstMatchingLocation(it) } + ?.firstContentNode() + nodeFromLocation ?: guidedNavigationTree.firstContentNode() ?: guidedNavigationTree } + private val stateMutable: MutableStateFlow = + MutableStateFlow( + stateMachine.play( + segment = dataLoader.getItemRef(initialNode)!!.segment, + playWhenReady = true, + settings = initialSettings + ) + ) + private val coroutineScope: CoroutineScope = MainScope() @@ -117,7 +143,7 @@ public class ReadAloudNavigator private constructor( ) } - private val ReadAloudLeafNode.utteranceLocation: UtteranceLocation get() { + private val ReadAloudNode.utteranceLocation: UtteranceLocation get() { val textref = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } checkNotNull(textref) val href = textref.url.removeFragment() @@ -135,7 +161,7 @@ public class ReadAloudNavigator private constructor( is ReadAloudStateMachine.PlaybackState.Ready -> State.Ready is ReadAloudStateMachine.PlaybackState.Starved -> - State.Buffering + State.Starved is ReadAloudStateMachine.PlaybackState.Ended -> State.Ended is ReadAloudStateMachine.PlaybackState.Failure -> @@ -161,24 +187,34 @@ public class ReadAloudNavigator private constructor( } public fun canEscape(): Boolean = - stateMutable.value.node.isEscapable() + with(navigationHelper) { + stateMutable.value.node.isEscapable() + } public fun canSkip(): Boolean = - stateMutable.value.node.isSkippable() + with(navigationHelper) { + stateMutable.value.node.isSkippable() + } public fun escape(force: Boolean = true) { - stateMutable.value.node.escape(force) - ?.let { go(it) } + with(navigationHelper) { + stateMutable.value.node.escape(force) + ?.let { go(it) } + } } public fun skipToPrevious(force: Boolean = true) { - stateMutable.value.node.skipToPrevious(force) - ?.let { go(it) } + with(navigationHelper) { + stateMutable.value.node.skipToPrevious(force) + ?.let { go(it) } + } } public fun skipToNext(force: Boolean = true) { - stateMutable.value.node.skipToNext(force) - ?.let { go(it) } + with(navigationHelper) { + stateMutable.value.node.skipToNext(force) + ?.let { go(it) } + } } public val location: StateFlow = @@ -196,8 +232,10 @@ public class ReadAloudNavigator private constructor( } public fun goTo(location: ReadAloudGoLocation) { - guidedNavigationTree.firstMatchingLocation(location) - ?.let { go(it) } + with(navigationHelper) { + guidedNavigationTree.firstMatchingLocation(location) + ?.let { go(it) } + } } public fun goTo(location: ReadAloudLocation) { @@ -222,4 +260,11 @@ public class ReadAloudNavigator private constructor( ) goTo(location) } + + public var settings: ReadAloudSettings by Delegates.observable(initialSettings) { + property, oldValue, newValue -> + with(stateMachine) { + stateMutable.value = stateMutable.value.updateSettings(oldValue, newValue) + } + } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index 622c5b0649..93beee7306 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -10,8 +10,8 @@ import android.app.Application import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.epub.MediaOverlaysService import org.readium.r2.shared.publication.services.GuidedNavigationService -import org.readium.r2.shared.publication.services.guidedNavigationService import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.data.ReadError import org.readium.r2.shared.util.getOrElse @@ -21,6 +21,7 @@ public class ReadAloudNavigatorFactory private constructor( private val guidedNavigationService: GuidedNavigationService, private val resources: List, private val audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, + private val ttsEngineFactory: () -> TtsEngine, ) { public companion object { @@ -29,14 +30,31 @@ public class ReadAloudNavigatorFactory private constructor( application: Application, publication: Publication, audioEngineProvider: AudioEngineProvider, + ttsEngineProvider: TtsEngineProvider, + usePrerecordedVoicesWhenAvailable: Boolean = true, ): ReadAloudNavigatorFactory? { - val guidedNavService = publication.guidedNavigationService - ?: return null + var guidedNavService: GuidedNavigationService? = null + + if (usePrerecordedVoicesWhenAvailable) { + guidedNavService = publication.findService(MediaOverlaysService::class) + } + if (guidedNavService == null) { + guidedNavService = publication.findService(GuidedNavigationService::class) + } + + if (guidedNavService == null) { + return null + } val audioEngineFactory = { listener: AudioEngine.Listener -> audioEngineProvider.createEngine(publication, listener) } + val ttsEngineFactory = { + val voice = ttsEngineProvider.voices.first() + ttsEngineProvider.createEngine(voice) + } + val resources = (publication.readingOrder + publication.resources).map { ReadAloudPublication.Item( href = it.url(), @@ -47,7 +65,8 @@ public class ReadAloudNavigatorFactory private constructor( return ReadAloudNavigatorFactory( guidedNavigationService = guidedNavService, resources = resources, - audioEngineFactory = audioEngineFactory + audioEngineFactory = audioEngineFactory, + ttsEngineFactory = ttsEngineFactory ) } } @@ -67,6 +86,7 @@ public class ReadAloudNavigatorFactory private constructor( } public suspend fun createNavigator( + initialSettings: ReadAloudSettings, initialLocation: ReadAloudGoLocation? = null, ): Try { val guidedDocs = buildList { @@ -94,9 +114,11 @@ public class ReadAloudNavigatorFactory private constructor( ) val navigator = ReadAloudNavigator( + initialSettings = initialSettings, initialLocation = initialLocation, publication = navigatorPublication, - audioEngineFactory = audioEngineFactory + audioEngineFactory = audioEngineFactory, + ttsEngineFactory = ttsEngineFactory ) return Try.success(navigator) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt new file mode 100644 index 0000000000..fee6292d64 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationRef +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationText + +@ExperimentalReadiumApi +public class ReadAloudNode( + public val text: GuidedNavigationText?, + public val refs: Set, + public val roles: Set, + public val children: List, +) { + + public var parent: ReadAloudNode? = null + internal set + + public companion object { + + internal fun fromGuidedNavigationObject( + guidedNavigationObject: GuidedNavigationObject, + ): ReadAloudNode { + val children = guidedNavigationObject.children + .map { fromGuidedNavigationObject(it) } + + val node = ReadAloudNode( + text = guidedNavigationObject.text, + refs = guidedNavigationObject.refs, + roles = guidedNavigationObject.roles, + children = children, + ) + + children.forEach { it.parent = node } + + return node + } + } +} + +@ExperimentalReadiumApi +public fun ReadAloudNode.next(): ReadAloudNode? { + nextDown()?.let { return it } + + nextRight()?.let { return it } + + return nextUp() +} + +@ExperimentalReadiumApi +public fun ReadAloudNode.previous(): ReadAloudNode? { + previousLeft()?.lastDescendant()?.let { return it } + + previousUp()?.lastDescendant()?.let { return it } + + return null +} + +private fun ReadAloudNode.lastDescendant(): ReadAloudNode? { + val lastChild = children.lastOrNull() + + return if (lastChild == null) { + this + } else { + lastChild.lastDescendant() + } +} + +private fun ReadAloudNode.nextDown(): ReadAloudNode? { + if (children.isEmpty()) { + return null + } + + return children.first() +} + +private fun ReadAloudNode.nextUp(): ReadAloudNode? { + return parent?.nextRight() ?: parent?.nextUp() +} + +private fun ReadAloudNode.previousUp(): ReadAloudNode? { + return parent?.previousLeft() ?: parent?.previousUp() +} + +private fun ReadAloudNode.nextRight(): ReadAloudNode? { + val parent = parent ?: return null + + val siblings = parent.children + + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + return if (currentIndex < siblings.size - 1) { + siblings[currentIndex + 1] + } else { + null + } +} + +private fun ReadAloudNode.previousLeft(): ReadAloudNode? { + val parent = parent ?: return null + + val siblings = parent.children + + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + return if (currentIndex > 0) { + siblings[currentIndex - 1] + } else { + null + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt index e5e3db0fc6..78797c53c9 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt @@ -4,8 +4,11 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(ExperimentalReadiumApi::class) + package org.readium.navigator.media.readaloud +import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.mediatype.MediaType diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt new file mode 100644 index 0000000000..b90066596d --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt @@ -0,0 +1,152 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationText +import org.readium.r2.shared.util.TemporalFragmentParser +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +internal sealed interface ReadAloudSegment { + + val nodes: List + + val engine: ReadAloudEngine + + val emptyNodes: Set +} + +internal data class AudioSegment( + override val engine: AudioEngine, + val items: List, + override val nodes: List, + override val emptyNodes: Set, +) : ReadAloudSegment + +internal data class TtsSegment( + override val engine: TtsEngine, + val items: List, + override val nodes: List, + override val emptyNodes: Set, +) : ReadAloudSegment + +internal class ReadAloudSegmentFactory( + private val audioEngineFactory: () -> AudioEngine, + private val ttsEngineFactory: () -> TtsEngine, +) { + + fun createSegmentFromNode(node: ReadAloudNode): ReadAloudSegment? = + createAudioSegmentFromNode(node) + .takeUnless { it.items.isEmpty() } + ?: createTtsSegmentFromNode(node) + .takeUnless { it.items.isEmpty() } + + private fun createAudioSegmentFromNode( + firstNode: ReadAloudNode, + ): AudioSegment { + var nextNode: ReadAloudNode? = firstNode + val audioItems = mutableListOf() + val nodes = mutableListOf() + val emptyNodes = mutableSetOf() + + while (nextNode != null && nextNode.content !is TextContent) { + val audioItem = (nextNode.content as? AudioContent)?.toAudioItem() + + if (audioItem != null) { + audioItems.add(audioItem) + nodes.add(nextNode) + } else { + emptyNodes.add(nextNode) + } + + nextNode = nextNode.next() + } + + val audioEngine = audioEngineFactory() + audioEngine.playWhenReady = false + audioEngine.setPlaylist(audioItems) + + return AudioSegment( + engine = audioEngine, + items = audioItems, + nodes = nodes, + emptyNodes = emptyNodes + ) + } + + private fun createTtsSegmentFromNode( + firstNode: ReadAloudNode, + ): TtsSegment { + var nextNode: ReadAloudNode? = firstNode + val textItems = mutableListOf() + val nodes = mutableListOf() + val emptyNodes = mutableSetOf() + + while (nextNode != null && nextNode.content !is AudioContent) { + val textContent = (nextNode.content as? TextContent) + + if (textContent != null) { + textItems.add(textContent.text) + nodes.add(nextNode) + } else { + emptyNodes.add(nextNode) + } + + nextNode = nextNode.next() + } + + return TtsSegment( + engine = ttsEngineFactory(), + items = textItems, + nodes = nodes, + emptyNodes = emptyNodes + ) + } + + private val ReadAloudNode.content: NodeContent? get() { + refs + .firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } + ?.let { + return AudioContent( + href = it.url.removeFragment(), + interval = it.url.timeInterval + ) + } + + text + ?.let { + return TextContent(it) + } + + return null + } +} + +private sealed interface NodeContent + +private data class AudioContent( + val href: Url, + val interval: TimeInterval?, +) : NodeContent + +private data class TextContent( + val text: GuidedNavigationText, +) : NodeContent + +private fun AudioContent.toAudioItem(): AudioEngine.Item { + return AudioEngine.Item( + href = href, + interval = interval + ) +} + +private val Url.timeInterval get() = fragment + ?.let { TemporalFragmentParser.parse(it) } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt new file mode 100644 index 0000000000..4034f39c5c --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +public data class ReadAloudSettings( + val language: Language, + val overrideContentLanguage: Boolean, + val preferRecordedVoices: Boolean, + val pitch: Double, + val speed: Double, + val voices: Map, + val escapableRoles: Set, + val skippableRoles: Set, +) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 80943a8dec..8f516ee141 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -9,75 +9,11 @@ package org.readium.navigator.media.readaloud import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationAudioRef import org.readium.r2.shared.util.Error -import org.readium.r2.shared.util.TemporalFragmentParser -import org.readium.r2.shared.util.Url - -internal sealed interface EngineFood { - - data class AudioEngineFood( - val items: List, - val nodes: List, - ) : EngineFood { - - companion object { - - fun fromNode(firstNode: ReadAloudLeafNode): AudioEngineFood? { - if (!firstNode.refs.any { it is GuidedNavigationAudioRef }) { - return null - } - - var nextLeaf: ReadAloudLeafNode? = firstNode - val audioItems = mutableListOf() - val nodes = mutableListOf() - - while (nextLeaf != null) { - val audioItem = nextLeaf.refs - .firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } - ?.toAudioItem() - - audioItem?.let { - audioItems.add(it) - nodes.add(nextLeaf) - } - - nextLeaf = nextLeaf.nextLeaf() - } - - return AudioEngineFood( - items = audioItems, - nodes = nodes - ) - } - - fun GuidedNavigationAudioRef.toAudioItem(): AudioEngine.Item { - return AudioEngine.Item( - href = url.removeFragment(), - interval = url.timeInterval - ) - } - - val Url.timeInterval get() = fragment - ?.let { TemporalFragmentParser.parse(it) } - } - } - - data class TtsFood( - val nodes: List, - ) { - - companion object { - - fun fromNode(firstNode: ReadAloudLeafNode): TtsFood? { - return null - } - } - } -} internal class ReadAloudStateMachine( - private val audioEngine: AudioEngine, + private val dataLoader: ReadAloudDataLoader, + private val navigationHelper: ReadAloudNavigationHelper, ) { sealed interface PlaybackState { @@ -94,84 +30,111 @@ internal class ReadAloudStateMachine( data class State( val playbackState: PlaybackState, val playWhenReady: Boolean, - val node: ReadAloudLeafNode, - val engineFood: EngineFood, + val node: ReadAloudNode, + val segment: ReadAloudSegment, + val settings: ReadAloudSettings, ) sealed interface Event { - data class AudioEngineStateChanged(val state: AudioEngine.State) : Event + data class AudioEngineStateChanged( + val engine: AudioEngine, + val state: AudioEngine.State, + ) : Event - data class AudioEngineItemChanged(val index: Int) : Event + data class AudioEngineItemChanged( + val engine: AudioEngine, + val index: Int, + ) : Event } - fun play(engineFood: EngineFood, playWhenReady: Boolean): State { - when (engineFood) { - is EngineFood.AudioEngineFood -> { - audioEngine.playWhenReady = playWhenReady - audioEngine.setPlaylist(engineFood.items) - } - } + fun play(segment: ReadAloudSegment, playWhenReady: Boolean, settings: ReadAloudSettings): State { + segment.engine.playWhenReady = playWhenReady return State( playbackState = PlaybackState.Starved, playWhenReady = playWhenReady, - node = engineFood.nodes[0], - engineFood = engineFood, + node = segment.nodes[0], + segment = segment, + settings = settings ) } fun State.pause(): State { - audioEngine.playWhenReady = false + segment.engine.playWhenReady = false return copy(playWhenReady = false) } fun State.resume(): State { - audioEngine.playWhenReady = true + segment.engine.playWhenReady = true return copy(playWhenReady = true) } fun State.jump(node: ReadAloudNode): State { - val firstLeaf = node.firstLeaf() + val firstContentNode = with(navigationHelper) { node.firstContentNode() } ?: return copy(playbackState = PlaybackState.Ended) - val engineFood = EngineFood.AudioEngineFood.fromNode(firstLeaf) + val itemRef = dataLoader.getItemRef(firstContentNode) ?: return copy(playbackState = PlaybackState.Ended) - return play(engineFood, playWhenReady) + val engineFood = itemRef.segment + + when (engineFood) { + is AudioSegment -> engineFood.engine.seekTo(itemRef.nodeIndex!!) + is TtsSegment -> TODO() + } + + return play(engineFood, playWhenReady, settings) + } + + fun State.updateSettings( + oldSettings: ReadAloudSettings, + newSettings: ReadAloudSettings, + ): State { + navigationHelper.settings = newSettings + dataLoader.settings = newSettings + return copy(settings = newSettings) } fun State.onEvent(event: Event): State = when (event) { - is Event.AudioEngineStateChanged -> onAudioEngineStateChanged(event.state) - is Event.AudioEngineItemChanged -> onAudioEngineItemChanged(event.index) + is Event.AudioEngineStateChanged -> + onAudioEngineStateChanged(event.engine, event.state) + is Event.AudioEngineItemChanged -> + onAudioEngineItemChanged(event.engine, event.index) } - fun State.onAudioEngineStateChanged(audioEngineState: AudioEngine.State): State = - when (audioEngineState) { + fun State.onAudioEngineStateChanged(engine: AudioEngine, audioEngineState: AudioEngine.State): State { + val currentEngine = (segment as? AudioSegment)?.engine + if (currentEngine != engine) { + return this + } + + return when (audioEngineState) { AudioEngine.State.Ready -> copy(playbackState = PlaybackState.Ready) AudioEngine.State.Starved -> copy(playbackState = PlaybackState.Starved) AudioEngine.State.Ended -> onAudioEngineEnded() } + } private fun State.onAudioEngineEnded(): State { - var nextNode: ReadAloudLeafNode? = node - var nextFood: EngineFood? - - do { - nextNode = nextNode?.nextLeaf() - nextFood = nextNode?.let { EngineFood.AudioEngineFood.fromNode(it) } - } while (nextFood == null && nextNode != null) - - return if (nextNode == null) { - copy(playbackState = PlaybackState.Ended) - } else { - nextFood!! - audioEngine.setPlaylist(nextFood.items) - copy(engineFood = nextFood, playbackState = PlaybackState.Starved, node = nextFood.nodes[0]) - } + val nextNode = node.next() + ?: return copy(playbackState = PlaybackState.Ended) + + val newSegment = dataLoader.getItemRef(nextNode)?.segment + ?: return copy(playbackState = PlaybackState.Ended) + + newSegment.engine.playWhenReady = playWhenReady + + return copy(segment = newSegment, playbackState = PlaybackState.Starved, node = newSegment.nodes[0]) } - fun State.onAudioEngineItemChanged(item: Int): State { - return copy(node = (engineFood as EngineFood.AudioEngineFood).nodes[item]) + fun State.onAudioEngineItemChanged(engine: AudioEngine, item: Int): State { + val currentEngine = (segment as? AudioSegment)?.engine + if (currentEngine != engine) { + return this + } + + dataLoader.onPlaybackProgressed(segment.nodes[item]) + return copy(node = segment.nodes[item]) } } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt index af23cf0552..6173aa60d2 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt @@ -6,14 +6,24 @@ package org.readium.r2.shared.guided +import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Link import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.Url +/** + * A guided navigation document. + */ +@ExperimentalReadiumApi public data class GuidedNavigationDocument( val links: List, val guided: List, ) + +/** + * A guided navigation object. + */ +@ExperimentalReadiumApi public data class GuidedNavigationObject( val children: List, val roles: Set, @@ -21,9 +31,17 @@ public data class GuidedNavigationObject( val text: GuidedNavigationText?, ) +/** + * A string containing some SSML markup. + */ +@ExperimentalReadiumApi @JvmInline public value class SsmlString(public val value: String) +/** + * Text holder for a guided navigation object. + */ +@ExperimentalReadiumApi @ConsistentCopyVisibility public data class GuidedNavigationText private constructor( val plain: String?, @@ -37,18 +55,34 @@ public data class GuidedNavigationText private constructor( } } +/** + * A reference to external content. + */ +@ExperimentalReadiumApi public sealed interface GuidedNavigationRef { public val url: Url } +/** + * A reference to external text content. + */ +@ExperimentalReadiumApi public data class GuidedNavigationTextRef( override val url: Url, ) : GuidedNavigationRef +/** + * A reference to external image content. + */ +@ExperimentalReadiumApi public data class GuidedNavigationImageRef( override val url: Url, ) : GuidedNavigationRef +/** + * A reference to external audio content. + */ +@ExperimentalReadiumApi public data class GuidedNavigationAudioRef( override val url: Url, ) : GuidedNavigationRef diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt index b0121a444f..7c31c6cb9a 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt @@ -6,14 +6,18 @@ package org.readium.r2.shared.guided +/** + * A role usable in a guided navigation object. + */ @JvmInline public value class GuidedNavigationRole(public val value: String) { public companion object { - /** + /* * Inherited from HTML and/or ARIA */ + public val ASIDE: GuidedNavigationRole = GuidedNavigationRole("aside") public val CELL: GuidedNavigationRole = GuidedNavigationRole("cell") public val DEFINITION: GuidedNavigationRole = GuidedNavigationRole("definition") @@ -24,9 +28,51 @@ public value class GuidedNavigationRole(public val value: String) { public val TABLE: GuidedNavigationRole = GuidedNavigationRole("table") public val TERM: GuidedNavigationRole = GuidedNavigationRole("term") - /** + /* + * Inherited from DPUB ARIA 1.0 + */ + + public val ABSTRACT: GuidedNavigationRole = GuidedNavigationRole("abstract") + public val ACKNOWLEDGMENTS: GuidedNavigationRole = GuidedNavigationRole("acknowledgments") + public val AFTERWORD: GuidedNavigationRole = GuidedNavigationRole("afterword") + public val APPENDIX: GuidedNavigationRole = GuidedNavigationRole("appendix") + public val BACKLINK: GuidedNavigationRole = GuidedNavigationRole("backlink") + public val BIBLIOGRAPHY: GuidedNavigationRole = GuidedNavigationRole("bibliography") + public val BIBLIOREF: GuidedNavigationRole = GuidedNavigationRole("biblioref") + public val CHAPTER: GuidedNavigationRole = GuidedNavigationRole("chapter") + public val COLOPHON: GuidedNavigationRole = GuidedNavigationRole("colophon") + public val CONCLUSION: GuidedNavigationRole = GuidedNavigationRole("conclusion") + public val COVER: GuidedNavigationRole = GuidedNavigationRole("cover") + public val CREDIT: GuidedNavigationRole = GuidedNavigationRole("credit") + public val CREDITS: GuidedNavigationRole = GuidedNavigationRole("credits") + public val DEDICATION: GuidedNavigationRole = GuidedNavigationRole("dedication") + public val ENDNOTES: GuidedNavigationRole = GuidedNavigationRole("endnotes") + public val EPIGRAPH: GuidedNavigationRole = GuidedNavigationRole("epigraph") + public val EPILOGUE: GuidedNavigationRole = GuidedNavigationRole("epilogue") + public val ERRATA: GuidedNavigationRole = GuidedNavigationRole("errata") + public val EXAMPLE: GuidedNavigationRole = GuidedNavigationRole("example") + public val FOOTNOTE: GuidedNavigationRole = GuidedNavigationRole("footnote") + public val GLOSSARY: GuidedNavigationRole = GuidedNavigationRole("glossary") + public val GLOSSREF: GuidedNavigationRole = GuidedNavigationRole("glossref") + public val INDEX: GuidedNavigationRole = GuidedNavigationRole("index") + public val INTRODUCTION: GuidedNavigationRole = GuidedNavigationRole("introduction") + public val NOTEREF: GuidedNavigationRole = GuidedNavigationRole("noteref") + public val NOTICE: GuidedNavigationRole = GuidedNavigationRole("notice") + public val PAGEBREAK: GuidedNavigationRole = GuidedNavigationRole("pagebreak") + public val PAGELIST: GuidedNavigationRole = GuidedNavigationRole("page-list") + public val PART: GuidedNavigationRole = GuidedNavigationRole("part") + public val PREFACE: GuidedNavigationRole = GuidedNavigationRole("preface") + public val PROLOGUE: GuidedNavigationRole = GuidedNavigationRole("prologue") + public val PULLQUOTE: GuidedNavigationRole = GuidedNavigationRole("pullquote") + public val QNA: GuidedNavigationRole = GuidedNavigationRole("qna") + public val SUBTITLE: GuidedNavigationRole = GuidedNavigationRole("subtitle") + public val TIP: GuidedNavigationRole = GuidedNavigationRole("tip") + public val TOC: GuidedNavigationRole = GuidedNavigationRole("toc") + + /* * Inherited from EPUB 3 Structural Semantics Vocabulary 1.1 */ + public val LANDMARKS: GuidedNavigationRole = GuidedNavigationRole("landmarks") public val LOA: GuidedNavigationRole = GuidedNavigationRole("loa") public val LOI: GuidedNavigationRole = GuidedNavigationRole("loi") @@ -34,9 +80,12 @@ public value class GuidedNavigationRole(public val value: String) { public val LOV: GuidedNavigationRole = GuidedNavigationRole("lov") public val SKIPPABLE_ROLES: List = - listOf() + listOf( + ASIDE, BIBLIOGRAPHY, ENDNOTES, FOOTNOTE, NOTEREF, PULLQUOTE, + LANDMARKS, LOA, LOI, LOT, LOV, PAGEBREAK, TOC + ) public val ESCAPABLE_ROLES: List = - listOf() + listOf(ASIDE, FIGURE, LIST, LIST_ITEM, TABLE, ROW, CELL) } } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt new file mode 100644 index 0000000000..4808bbd43f --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.publication.epub + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.PublicationServicesHolder +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.publication.services.GuidedNavigationService + +/** + * Provides a list of guided navigation documents mimicking media overlays available in Publication. + */ +@ExperimentalReadiumApi +public interface MediaOverlaysService : GuidedNavigationService + +/** + * Returns an iterator providing access to all the guided navigation documents mimicking media + * overlays of the publication. + */ +@ExperimentalReadiumApi +public fun Publication.mediaOverlaysIterator(): GuidedNavigationIterator? = + mediaOverlaysService?.iterator() + +@ExperimentalReadiumApi +private val PublicationServicesHolder.mediaOverlaysService: GuidedNavigationService? + get() { + findService(MediaOverlaysService::class)?.let { return it } + return null + } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt index 2d723b7e73..1d31fabc2c 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt @@ -14,17 +14,23 @@ import org.readium.r2.shared.util.Closeable import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.data.ReadError +/** + * Provides a list of Guided Navigation documents for a Publication. + */ @ExperimentalReadiumApi public interface GuidedNavigationService : Publication.Service { public fun iterator(): GuidedNavigationIterator } +/** + * Iterator providing access to all guided navigation documents of a Publication. + */ @ExperimentalReadiumApi public interface GuidedNavigationIterator : Closeable { /** - * Prepares an element for retrieval by the invocation of next. + * Prepares the next guided navigation document for retrieval by the invocation of next. * * Does nothing if the the end has been reached. */ @@ -42,8 +48,15 @@ public interface GuidedNavigationIterator : Closeable { override fun close() {} } +/** + * Returns an iterator providing access to all the guided navigation documents of the publication. + */ @ExperimentalReadiumApi -public val PublicationServicesHolder.guidedNavigationService: GuidedNavigationService? +public fun Publication.guidedNavigationIterator(): GuidedNavigationIterator? = + guidedNavigationService?.iterator() + +@OptIn(ExperimentalReadiumApi::class) +private val PublicationServicesHolder.guidedNavigationService: GuidedNavigationService? get() { findService(GuidedNavigationService::class)?.let { return it } return null diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt index 6e502fa89d..85b263e6b3 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt @@ -14,7 +14,7 @@ import org.readium.r2.shared.publication.Link import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.encryption.Encryption import org.readium.r2.shared.publication.epub.EpubEncryptionParser -import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.publication.epub.MediaOverlaysService import org.readium.r2.shared.publication.services.content.DefaultContentService import org.readium.r2.shared.publication.services.content.iterators.HtmlResourceContentIterator import org.readium.r2.shared.publication.services.search.StringSearchService @@ -113,7 +113,7 @@ public class EpubParser( HtmlResourceContentIterator.Factory() ) ) - ).also { it[GuidedNavigationService::class] = MediaOverlaysService.createFactory(smils) } + ).also { it[MediaOverlaysService::class] = SmilBasedMediaOverlaysService.createFactory(smils) } ) return Try.success(builder) diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt index 497ce4dda5..02f71ac4bd 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt @@ -1,10 +1,10 @@ /* - * Copyright 2022 Readium Foundation. All rights reserved. + * Copyright 2025 Readium Foundation. All rights reserved. * Use of this source code is governed by the BSD-style license * available in the top-level LICENSE file of the project. */ -@file:OptIn(InternalReadiumApi::class) +@file:OptIn(InternalReadiumApi::class, ExperimentalReadiumApi::class) package org.readium.r2.streamer.parser.epub @@ -16,8 +16,8 @@ import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.guided.GuidedNavigationRole import org.readium.r2.shared.guided.GuidedNavigationTextRef import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.epub.MediaOverlaysService import org.readium.r2.shared.publication.services.GuidedNavigationIterator -import org.readium.r2.shared.publication.services.GuidedNavigationService import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.data.Container @@ -29,34 +29,36 @@ import org.readium.r2.shared.util.resource.Resource import org.readium.r2.shared.util.use import org.readium.r2.shared.util.xml.ElementNode +/** + * A GuidedNavigationService producing guided navigation documents from media overlays SMIL files. + */ @ExperimentalReadiumApi -public class MediaOverlaysService( - private val smils: List, +public class SmilBasedMediaOverlaysService( + private val smilFiles: List, private val container: Container, -) : GuidedNavigationService { +) : MediaOverlaysService { override fun iterator(): GuidedNavigationIterator { - return MediaOverlaysIterator(smils, container) + return MediaOverlaysIterator(smilFiles, container) } public companion object { public fun createFactory( - smils: List, + smilFiles: List, ): ( Publication.Service.Context, - ) -> MediaOverlaysService = + ) -> SmilBasedMediaOverlaysService = { context -> - MediaOverlaysService( - smils = smils, + SmilBasedMediaOverlaysService( + smilFiles = smilFiles, container = context.container ) } } } -@OptIn(ExperimentalReadiumApi::class) -internal class MediaOverlaysIterator( +private class MediaOverlaysIterator( private val smils: List, private val container: Container, ) : GuidedNavigationIterator { @@ -190,6 +192,44 @@ internal object SmilParser { "table-row" -> GuidedNavigationRole.ROW "table" -> GuidedNavigationRole.TABLE "glossterm" -> GuidedNavigationRole.TERM + + "abstract" -> GuidedNavigationRole.ABSTRACT + "acknowledgments" -> GuidedNavigationRole.ACKNOWLEDGMENTS + "afterword" -> GuidedNavigationRole.AFTERWORD + "appendix" -> GuidedNavigationRole.APPENDIX + "backlink" -> GuidedNavigationRole.BACKLINK + "bibliography" -> GuidedNavigationRole.BIBLIOGRAPHY + "biblioref" -> GuidedNavigationRole.BIBLIOREF + "chapter" -> GuidedNavigationRole.CHAPTER + "colophon" -> GuidedNavigationRole.COLOPHON + "conclusion" -> GuidedNavigationRole.CONCLUSION + "cover" -> GuidedNavigationRole.COVER + "credit" -> GuidedNavigationRole.CREDIT + "credits" -> GuidedNavigationRole.CREDITS + "dedication" -> GuidedNavigationRole.DEDICATION + "endnotes" -> GuidedNavigationRole.ENDNOTES + "epigraph" -> GuidedNavigationRole.EPIGRAPH + "epilogue" -> GuidedNavigationRole.EPILOGUE + "errata" -> GuidedNavigationRole.ERRATA + "example" -> GuidedNavigationRole.EXAMPLE + "footnote" -> GuidedNavigationRole.FOOTNOTE + "glossary" -> GuidedNavigationRole.GLOSSARY + "glossref" -> GuidedNavigationRole.GLOSSREF + "index" -> GuidedNavigationRole.INDEX + "introduction" -> GuidedNavigationRole.INTRODUCTION + "noteref" -> GuidedNavigationRole.NOTEREF + "notice" -> GuidedNavigationRole.NOTICE + "pagebreak" -> GuidedNavigationRole.PAGEBREAK + "page-list" -> GuidedNavigationRole.PAGELIST + "part" -> GuidedNavigationRole.PART + "preface" -> GuidedNavigationRole.PREFACE + "prologue" -> GuidedNavigationRole.PROLOGUE + "pullquote" -> GuidedNavigationRole.PULLQUOTE + "qna" -> GuidedNavigationRole.QNA + "subtitle" -> GuidedNavigationRole.SUBTITLE + "tip" -> GuidedNavigationRole.TIP + "toc" -> GuidedNavigationRole.TOC + "landmarks" -> GuidedNavigationRole.LANDMARKS "loa" -> GuidedNavigationRole.LOA "loi" -> GuidedNavigationRole.LOI diff --git a/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt new file mode 100644 index 0000000000..a20411e241 --- /dev/null +++ b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class, InternalReadiumApi::class) + +package org.readium.r2.streamer.parser.epub + +import java.io.File +import kotlin.reflect.KClass +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlinx.coroutines.runBlocking +import org.junit.runner.RunWith +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationRef +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.RelativeUrl +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.data.Container +import org.readium.r2.shared.util.file.DirectoryContainer +import org.readium.r2.shared.util.resource.Resource +import org.readium.r2.shared.util.xml.XmlParser +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class SmilBasedMediaOverlaysServiceTest { + + private val smilDir = requireNotNull( + SmilBasedMediaOverlaysServiceTest::class.java + .getResource("smil") + ?.path + ?.let { File(it) } + ) + + private val smilUrls = listOf( + RelativeUrl("chapter_001_overlay.smil")!!, + RelativeUrl("chapter_002_overlay.smil")!! + ) + + private val container: Container = DirectoryContainer( + root = smilDir, + entries = smilUrls.toSet() + ) + + @Test + fun `smil files are chained`() = runBlocking { + val service = SmilBasedMediaOverlaysService(smilUrls, container) + val iterator = service.iterator() + val docs = mutableListOf() + while (iterator.hasNext()) { + val doc = assertNotNull(iterator.next().getOrNull()) + docs.add(doc) + } + assert(docs.size == 2) + } +} + +@RunWith(RobolectricTestRunner::class) +class SmilParserTest { + + private val chapter1File: File = requireNotNull( + SmilParserTest::class.java + .getResource("smil/chapter_001_overlay.smil") + ?.path + ?.let { File(it) } + ) + + @Suppress("UNCHECKED_CAST") + private fun firstRefOfClass( + nodes: List, + klass: KClass, + ): T? { + for (node in nodes) { + node.refs + .firstOrNull { klass.isInstance(it) } + ?.let { return it as T } + + return firstRefOfClass(node.children, klass) + } + + return null + } + + private fun parseSmilDoc(): GuidedNavigationDocument { + val root = chapter1File + .inputStream() + .use { XmlParser().parse(it) } + + val doc = + SmilParser.parse(root, Url("OPS/chapter_001_overlay.smil")!!) + + return assertNotNull(doc) + } + + @Test + fun `all leaves are parsed`() { + fun assertSize(nodes: List) { + assert(nodes.size == 1 || nodes.size == 27) + + if (nodes.size == 27) { + return + } + for (node in nodes) { + assertSize(node.children) + } + } + + val guidedNavDoc = parseSmilDoc() + assertSize(guidedNavDoc.guided) + } + + @Test + fun `generated href are relative to SMIL`() { + val guidedNavDoc = parseSmilDoc() + val firstTextRef = assertNotNull(firstRefOfClass(guidedNavDoc.guided, GuidedNavigationTextRef::class)) + assertEquals(Url("OPS/chapter_001.xhtml#c01h01")!!, firstTextRef.url) + + val firstAudioRef = assertNotNull(firstRefOfClass(guidedNavDoc.guided, GuidedNavigationAudioRef::class)) + assertEquals(Url("OPS/audio/mobydick_001_002_melville.mp4")!!, firstAudioRef.url.removeFragment()) + } + + @Test + fun `audio clips are correct`() { + val guidedNavDoc = parseSmilDoc() + val firstAudioRef = assertNotNull(firstRefOfClass(guidedNavDoc.guided, GuidedNavigationAudioRef::class)) + assertEquals("t=24.5,29.268", firstAudioRef.url.fragment) + } +} diff --git a/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil new file mode 100644 index 0000000000..cd366fdf10 --- /dev/null +++ b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil @@ -0,0 +1,139 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil new file mode 100644 index 0000000000..16e0d08106 --- /dev/null +++ b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From d62c6dcbac3ea0434a08190772ebb0a58bc525aa Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Wed, 3 Sep 2025 11:42:10 +0200 Subject: [PATCH 10/13] Draft TTS implementation --- .../readium/demo/navigator/DemoViewModel.kt | 12 +- .../navigator/reader/ReadAloudRendition.kt | 6 +- .../demo/navigator/reader/ReaderOpener.kt | 3 +- .../demo/navigator/reader/ReaderState.kt | 3 +- .../navigator/reader/SelectNavigatorMenu.kt | 3 +- .../exoplayer/readaloud/ExoPlayerEngine.kt | 21 +- .../readaloud/ExoPlayerEngineProvider.kt | 3 +- .../media/readaloud/AndroidTtsEngine.kt | 435 ++++++++++++++++++ .../navigator/media/readaloud/AudioEngine.kt | 54 +++ .../ContentIteratorGuidedNavigationService.kt | 20 - .../media/readaloud/ReadAloudDataLoader.kt | 22 +- .../media/readaloud/ReadAloudEngine.kt | 112 ----- .../media/readaloud/ReadAloudLocations.kt | 29 +- .../media/readaloud/ReadAloudNavigator.kt | 162 ++++++- .../readaloud/ReadAloudNavigatorFactory.kt | 38 +- .../media/readaloud/ReadAloudSegment.kt | 70 ++- .../media/readaloud/ReadAloudStateMachine.kt | 100 ++-- .../media/readaloud/SegmentPlayer.kt | 70 +++ .../navigator/media/readaloud/TtsEngine.kt | 106 +++++ .../readaloud/TtsGuidedNavigationService.kt | 108 +++++ .../navigator/media/readaloud/TtsPlayer.kt | 162 +++++++ .../shared/guided/GuidedNavigationDocument.kt | 17 +- .../r2/streamer/parser/epub/EpubParser.kt | 6 +- 23 files changed, 1267 insertions(+), 295 deletions(-) create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index 48deb99fe3..a53dd78afb 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -23,7 +23,7 @@ import org.readium.demo.navigator.reader.SelectNavigatorItem import org.readium.demo.navigator.reader.SelectNavigatorViewModel import org.readium.demo.navigator.reader.fixedConfig import org.readium.demo.navigator.reader.reflowableConfig -import org.readium.navigator.media.readaloud.NullTtsEngineProvider +import org.readium.navigator.media.readaloud.AndroidTtsEngineProvider import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory @@ -124,13 +124,19 @@ class DemoViewModel( configuration = fixedConfig )?.let { SelectNavigatorItem.FixedWeb(it) } - val readAloudFactory = + val readAloudFactory = run { + val ttsEngineProvider = + AndroidTtsEngineProvider(application) + ?.takeIf { it.voices.isNotEmpty() } + ?: return@run null + ReadAloudNavigatorFactory( application = application, publication = publication, audioEngineProvider = audioEngineProvider, - ttsEngineProvider = NullTtsEngineProvider + ttsEngineProvider = ttsEngineProvider )?.let { SelectNavigatorItem.ReadAloud(it) } + } val factories = listOfNotNull( reflowableFactory, diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt index 5c87d934ec..e238560b62 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt @@ -53,9 +53,11 @@ fun ReadAloudRendition( Text("Play When Ready: ${playbackState.value.playWhenReady}") - Text("Resource Href: ${playbackState.value.utteranceLocation.href}") + Text("Resource Href: ${playbackState.value.utteranceLocation?.href}") - Text("Utterance Selector ${playbackState.value.utteranceLocation.cssSelector?.value}") + Text("Utterance Css Selector ${playbackState.value.utteranceLocation?.cssSelector?.value}") + + Text("Utterance ${playbackState.value.utteranceLocation?.textQuote?.text}") } Toolbar(readerState) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index 45423f988d..3f0b91b04f 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -26,6 +26,7 @@ import org.readium.navigator.common.DecorationLocation import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings import org.readium.navigator.common.SettingsController +import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory import org.readium.navigator.media.readaloud.ReadAloudSettings import org.readium.navigator.web.fixedlayout.FixedWebGoLocation @@ -197,7 +198,7 @@ class ReaderOpener( private suspend fun createReadAloudReader( url: AbsoluteUrl, publication: Publication, - navigatorFactory: ReadAloudNavigatorFactory, + navigatorFactory: ReadAloudNavigatorFactory, initialLocator: Locator?, ): Try { val initialSettings = ReadAloudSettings( diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt index afb2d971ea..a8a2846b7e 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt @@ -18,6 +18,7 @@ import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.RenditionState import org.readium.navigator.common.SelectionController import org.readium.navigator.common.SelectionLocation +import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication @@ -48,7 +49,7 @@ data class VisualReaderState, ) : ReaderState { override fun close() { diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt index edd5cf295e..88e7970c3b 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt @@ -13,6 +13,7 @@ import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.ui.window.PopupProperties +import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory @@ -48,7 +49,7 @@ sealed class SelectNavigatorItem( ) : SelectNavigatorItem("Fixed Web Rendition") data class ReadAloud( - override val factory: ReadAloudNavigatorFactory, + override val factory: ReadAloudNavigatorFactory, ) : SelectNavigatorItem("Read Aloud Navigator") } diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index 75c78f98c7..bae0b3a588 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -37,6 +37,7 @@ import org.readium.r2.shared.util.data.ReadException @androidx.annotation.OptIn(UnstableApi::class) public class ExoPlayerEngine private constructor( private val exoPlayer: ExoPlayer, + override val playlist: List, private val listener: AudioEngine.Listener, ) : AudioEngine { @@ -45,6 +46,7 @@ public class ExoPlayerEngine private constructor( public operator fun invoke( application: Application, dataSourceFactory: DataSource.Factory, + playlist: List, listener: AudioEngine.Listener, ): ExoPlayerEngine { val exoPlayer = ExoPlayer.Builder(application) @@ -61,7 +63,7 @@ public class ExoPlayerEngine private constructor( exoPlayer.preloadConfiguration = ExoPlayer.PreloadConfiguration(10_000_000L) - return ExoPlayerEngine(exoPlayer, listener) + return ExoPlayerEngine(exoPlayer, playlist, listener) } } @@ -102,16 +104,9 @@ public class ExoPlayerEngine private constructor( private val coroutineScope: CoroutineScope = MainScope() - private var prepareCalled: Boolean = false - init { exoPlayer.addListener(Listener()) - } - - override fun setPlaylist( - items: List, - ) { - val mediaItems = items.map { item -> + val mediaItems = playlist.map { item -> val clippingConfig = MediaItem.ClippingConfiguration.Builder() .apply { item.interval?.start?.let { setStartPositionMs(it.inWholeMilliseconds) } @@ -123,11 +118,7 @@ public class ExoPlayerEngine private constructor( .build() } exoPlayer.setMediaItems(mediaItems) - - if (!prepareCalled) { - exoPlayer.prepare() - prepareCalled = true - } + exoPlayer.prepare() } override fun seekTo(index: Int) { @@ -140,7 +131,7 @@ public class ExoPlayerEngine private constructor( exoPlayer.playWhenReady = value } - public fun close() { + public override fun release() { coroutineScope.cancel() exoPlayer.release() } diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt index da6ad0c66b..5b9f49cd31 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt @@ -22,9 +22,10 @@ public class ExoPlayerEngineProvider( override fun createEngine( publication: Publication, + playlist: List, listener: AudioEngine.Listener, ): ExoPlayerEngine { val dataSourceFactory = ExoPlayerDataSource.Factory(publication) - return ExoPlayerEngine(application, dataSourceFactory, listener) + return ExoPlayerEngine(application, dataSourceFactory, playlist, listener) } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt new file mode 100644 index 0000000000..78d0b47759 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt @@ -0,0 +1,435 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import android.annotation.SuppressLint +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import android.speech.tts.TextToSpeech +import android.speech.tts.TextToSpeech.* +import android.speech.tts.UtteranceProgressListener +import android.speech.tts.Voice as AndroidVoice +import android.speech.tts.Voice.* +import java.util.UUID +import kotlin.collections.orEmpty +import kotlinx.coroutines.* +import org.readium.navigator.media.readaloud.AndroidTtsEngine.Companion.initializeTextToSpeech +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.tryOrNull +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +public class AndroidTtsEngineProvider private constructor( + private val context: Context, + private val textToSpeech: TextToSpeech, +) : TtsEngineProvider { + + override val voices: Set = + tryOrNull { textToSpeech.voices } // throws on Nexus 4 + ?.map { it.toTtsEngineVoice() } + ?.toSet() + .orEmpty() + + override fun createEngine( + voice: AndroidTtsEngine.Voice, + utterances: List, + listener: TtsEngine.Listener, + ): TtsEngine { + val voice = voices.firstOrNull { it.name == voice.name } + checkNotNull(voice) + + return AndroidTtsEngine( + context = context, + engine = textToSpeech, + listener = listener, + voice = voice, + utterances = utterances + ) + } + + public companion object { + + public suspend operator fun invoke( + context: Context, + ): AndroidTtsEngineProvider? { + val textToSpeech = initializeTextToSpeech(context) + ?: return null + + return AndroidTtsEngineProvider(context, textToSpeech) + } + + private fun AndroidVoice.toTtsEngineVoice() = + AndroidTtsEngine.Voice( + name = name, + language = Language(locale), + quality = when (quality) { + QUALITY_VERY_HIGH -> AndroidTtsEngine.Voice.Quality.Highest + QUALITY_HIGH -> AndroidTtsEngine.Voice.Quality.High + QUALITY_NORMAL -> AndroidTtsEngine.Voice.Quality.Normal + QUALITY_LOW -> AndroidTtsEngine.Voice.Quality.Low + QUALITY_VERY_LOW -> AndroidTtsEngine.Voice.Quality.Lowest + else -> throw IllegalStateException("Unexpected voice quality.") + }, + requiresNetwork = isNetworkConnectionRequired + ) + } +} + +/* + * On some Android implementations (i.e. on Oppo A9 2020 running Android 11), + * the TextToSpeech instance is often disconnected from the underlying service when the playback + * is paused and the app moves to the background. So we try to reset the TextToSpeech before + * actually returning an error. In the meantime, requests to the engine are queued + * into [pendingRequests]. + */ + +/** + * Default [TtsEngine] implementation using Android's native text to speech engine. + */ +@ExperimentalReadiumApi +public class AndroidTtsEngine internal constructor( + private val context: Context, + engine: TextToSpeech, + private val listener: TtsEngine.Listener, + public val voice: Voice, + override val utterances: List, +) : TtsEngine { + + public companion object { + + /** + * Starts the activity to install additional voice data. + */ + @SuppressLint("QueryPermissionsNeeded") + public fun requestInstallVoice(context: Context) { + val intent = Intent() + .setAction(Engine.ACTION_INSTALL_TTS_DATA) + .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + + val availableActivities = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + context.packageManager.queryIntentActivities( + intent, + PackageManager.ResolveInfoFlags.of(0) + ) + } else { + context.packageManager.queryIntentActivities(intent, 0) + } + + if (availableActivities.isNotEmpty()) { + context.startActivity(intent) + } + } + + internal suspend fun initializeTextToSpeech( + context: Context, + ): TextToSpeech? { + val init = CompletableDeferred() + + val initListener = OnInitListener { status -> + init.complete(status == SUCCESS) + } + val engine = TextToSpeech(context, initListener) + return if (init.await()) engine else null + } + } + + public sealed class Error( + override val message: String, + override val cause: org.readium.r2.shared.util.Error? = null, + ) : org.readium.r2.shared.util.Error { + + /** Denotes a generic operation failure. */ + public data object Unknown : Error("An unknown error occurred.") + + /** Denotes a failure caused by an invalid request. */ + public data object InvalidRequest : Error("Invalid request") + + /** Denotes a failure caused by a network connectivity problems. */ + public data object Network : Error("A network error occurred.") + + /** Denotes a failure caused by network timeout. */ + public data object NetworkTimeout : Error("Network timeout") + + /** Denotes a failure caused by an unfinished download of the voice data. */ + public data object NotInstalledYet : Error("Voice not installed yet.") + + /** Denotes a failure related to the output (audio device or a file). */ + public data object Output : Error("An error related to the output occurred.") + + /** Denotes a failure of a TTS service. */ + public data object Service : Error("An error occurred with the TTS service.") + + /** Denotes a failure of a TTS engine to synthesize the given input. */ + public data object Synthesis : Error("Synthesis failed.") + + /** + * Denotes the language data is missing. + * + * You can open the Android settings to install the missing data with: + * AndroidTtsEngine.requestInstallVoice(context) + */ + public data class LanguageMissingData(val language: Language) : + Error("Language data is missing.") + + /** + * Android's TTS error code. + * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR + */ + public companion object { + internal fun fromNativeError(code: Int): Error = + when (code) { + ERROR_INVALID_REQUEST -> InvalidRequest + ERROR_NETWORK -> Network + ERROR_NETWORK_TIMEOUT -> NetworkTimeout + ERROR_NOT_INSTALLED_YET -> NotInstalledYet + ERROR_OUTPUT -> Output + ERROR_SERVICE -> Service + ERROR_SYNTHESIS -> Synthesis + else -> Unknown + } + } + } + + /** + * Represents a voice provided by the TTS engine which can speak an utterance. + * + * @param name Voice name + * @param language Language (and region) this voice belongs to. + * @param quality Voice quality. + * @param requiresNetwork Indicates whether using this voice requires an Internet connection. + */ + public data class Voice( + val name: String, + val language: Language, + val quality: Quality = Quality.Normal, + val requiresNetwork: Boolean = false, + ) : TtsVoice { + + override val id: TtsVoice.Id = + TtsVoice.Id("${AndroidTtsEngine::class.qualifiedName}-$name}") + + override val languages: Set = + setOf(language) + + public enum class Quality { + Lowest, + Low, + Normal, + High, + Highest, + } + } + + private data class Request( + val id: Id, + val text: String, + ) { + + @JvmInline + value class Id(val value: String) + } + + private sealed class State { + + data class EngineAvailable( + val engine: TextToSpeech, + ) : State() + + data class WaitingForService( + var pendingRequest: Request?, + ) : State() + + data class Failure( + val error: Error, + ) : State() + } + + private val coroutineScope: CoroutineScope = + MainScope() + + private var state: State = + State.EngineAvailable(engine) + + private var isPrepared: Boolean = + false + + private var isClosed: Boolean = + false + + private var speed: Double = 1.0 + + private var pitch: Double = 1.0 + + override fun prepare() { + if (isPrepared) { + return + } + + isPrepared = true + + (state as? State.EngineAvailable) + ?.let { setupListener(it.engine) } + listener.onReady() + } + + override fun speak( + utteranceIndex: Int, + ) { + check(isPrepared) { "Engine has not been prepared." } + check(!isClosed) { "Engine is closed." } + + val id = Request.Id(UUID.randomUUID().toString()) + val text = checkNotNull(utterances)[utteranceIndex] + val request = Request(id, text) + when (val stateNow = state) { + is State.WaitingForService -> { + stateNow.pendingRequest = request + } + is State.Failure -> { + tryReconnect(request) + } + is State.EngineAvailable -> { + if (!doSpeak(stateNow.engine, request)) { + cleanEngine(stateNow.engine) + tryReconnect(request) + } + } + } + } + + public override fun stop() { + when (val stateNow = state) { + is State.EngineAvailable -> { + stateNow.engine.stop() + } + is State.Failure -> { + // Do nothing + } + is State.WaitingForService -> { + stateNow.pendingRequest = null + } + } + } + + public override fun release() { + if (isClosed) { + return + } + + isClosed = true + coroutineScope.cancel() + + when (val stateNow = state) { + is State.EngineAvailable -> { + cleanEngine(stateNow.engine) + } + is State.Failure -> { + // Do nothing + } + is State.WaitingForService -> { + // Do nothing + } + } + } + + private fun doSpeak( + engine: TextToSpeech, + request: Request, + ): Boolean { + engine.setupPitchAndSpeed() + return engine.setupVoice() && + (engine.speak(request.text, QUEUE_ADD, null, request.id.value) == SUCCESS) + } + + private fun setupListener(engine: TextToSpeech) { + engine.setOnUtteranceProgressListener(UtteranceListener(listener)) + } + + private fun onReconnectionSucceeded(engine: TextToSpeech) { + val previousState = state as State.WaitingForService + setupListener(engine) + engine.setupPitchAndSpeed() + state = State.EngineAvailable(engine) + if (isClosed) { + engine.shutdown() + } else { + previousState.pendingRequest?.let { doSpeak(engine, it) } + } + } + + private fun onReconnectionFailed() { + val error = Error.Service + state = State.Failure(error) + listener.onError(error) + } + + private fun tryReconnect(request: Request) { + state = State.WaitingForService(request) + coroutineScope.launch { + initializeTextToSpeech(context) + ?.let { onReconnectionSucceeded(it) } + ?: onReconnectionFailed() + } + } + + private fun cleanEngine(engine: TextToSpeech) { + engine.setOnUtteranceProgressListener(null) + engine.shutdown() + } + + private fun TextToSpeech.setupPitchAndSpeed() { + setSpeechRate(speed.toFloat()) + setPitch(pitch.toFloat()) + } + + private fun TextToSpeech.setupVoice(): Boolean { + val voice = voiceForName(voice.name) + setVoice(voice) + return true + } + + private fun TextToSpeech.voiceForName(name: String) = + voices.firstOrNull { it.name == name } + + private class UtteranceListener( + private val listener: TtsEngine.Listener, + ) : UtteranceProgressListener() { + override fun onStart(utteranceId: String) { + } + + override fun onStop(utteranceId: String, interrupted: Boolean) { + listener.onInterrupted() + } + + override fun onDone(utteranceId: String) { + listener.onDone() + } + + @Deprecated( + "Deprecated in the interface", + ReplaceWith("onError(utteranceId, -1)"), + level = DeprecationLevel.ERROR + ) + override fun onError(utteranceId: String) { + onError(utteranceId, -1) + } + + override fun onError(utteranceId: String, errorCode: Int) { + listener.onError(Error.fromNativeError(errorCode)) + } + + override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { + listener.onRange(start until end) + } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt new file mode 100644 index 0000000000..97900daca2 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +@ExperimentalReadiumApi +public interface AudioEngine { + + public interface Listener { + + public fun onItemChanged(engine: AudioEngine, index: Int) + + public fun onStateChanged(engine: AudioEngine, state: State) + } + + public enum class State { + Ready, + Starved, + Ended, + } + + public data class Item( + val href: Url, + val interval: TimeInterval?, + ) + + public var playWhenReady: Boolean + + public val playlist: List + + public fun seekTo(index: Int) + + public fun release() +} + +@ExperimentalReadiumApi +public interface AudioEngineProvider { + + public fun createEngine( + publication: Publication, + playlist: List, + listener: AudioEngine.Listener, + ): AudioEngine +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt deleted file mode 100644 index 5c2495f07a..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentIteratorGuidedNavigationService.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(ExperimentalReadiumApi::class) - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.services.GuidedNavigationIterator -import org.readium.r2.shared.publication.services.GuidedNavigationService - -internal class ContentIteratorGuidedNavigationService : GuidedNavigationService { - - override fun iterator(): GuidedNavigationIterator { - TODO("Not yet implemented") - } -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt index b030877289..8b2879a8f3 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt @@ -10,20 +10,16 @@ package org.readium.navigator.media.readaloud import kotlin.properties.Delegates import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Error -internal class ReadAloudDataLoader( - private val segmentFactory: ReadAloudSegmentFactory, +internal class ReadAloudDataLoader( + private val segmentFactory: ReadAloudSegmentFactory, initialSettings: ReadAloudSettings, ) { - - sealed interface NodeInfo - data class ItemRef( val segment: ReadAloudSegment, val nodeIndex: Int?, - ) : NodeInfo - - data object EmptyNode : NodeInfo + ) var settings by Delegates.observable(initialSettings) { property, oldValue, newValue -> preloadedRefs.clear() @@ -35,7 +31,7 @@ internal class ReadAloudDataLoader( val nextNode = node.next() ?: return if (nextNode !in preloadedRefs) { - loadSegmentForNode(nextNode) + loadSegmentForNode(nextNode)?.player?.prepare() } } @@ -44,16 +40,18 @@ internal class ReadAloudDataLoader( return preloadedRefs[node] } - private fun loadSegmentForNode(node: ReadAloudNode) { + private fun loadSegmentForNode(node: ReadAloudNode): ReadAloudSegment? { if (node in preloadedRefs) { - return + return null } val segment = segmentFactory.createSegmentFromNode(node) - ?: return // Ended + ?: return null // Ended val refs = computeRefsForSegment(segment) preloadedRefs.putAll(refs) + + return segment } private fun computeRefsForSegment(segment: ReadAloudSegment): Map { diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt deleted file mode 100644 index 131e03dbb0..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudEngine.kt +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.util.Language -import org.readium.r2.shared.util.TimeInterval -import org.readium.r2.shared.util.Url - -@ExperimentalReadiumApi -public sealed interface ReadAloudEngine { - - public var playWhenReady: Boolean -} - -@ExperimentalReadiumApi -public interface TtsEngine : ReadAloudEngine { - - public interface Listener { - - public fun onUtterancesReady() - - public fun onPlaybackCompleted() - } - - public fun feed(utterances: List) - - public fun speak(utteranceIndex: Int) -} - -@ExperimentalReadiumApi -public interface PausableTtsEngine : TtsEngine { - - public fun pause() - - public fun resume() -} - -@ExperimentalReadiumApi -public interface TtsVoice { - - @JvmInline - public value class Id(public val value: String) - - public val id: Id - - /** - * The languages supported by the voice. - */ - public val languages: Set -} - -@ExperimentalReadiumApi -public interface TtsEngineProvider { - - /** - * Sets of voices available with this [TtsEngineProvider]. - */ - public val voices: Set - - public fun createEngine(voice: TtsVoice): TtsEngine -} - -@ExperimentalReadiumApi -public object NullTtsEngineProvider : TtsEngineProvider { - - override val voices: Set = emptySet() - - override fun createEngine(voice: TtsVoice): TtsEngine { - throw IllegalStateException() - } -} - -@ExperimentalReadiumApi -public interface AudioEngine : ReadAloudEngine { - - public interface Listener { - - public fun onItemChanged(engine: AudioEngine, index: Int) - - public fun onStateChanged(engine: AudioEngine, state: State) - } - - public enum class State { - Ready, - Starved, - Ended, - } - - public data class Item( - val href: Url, - val interval: TimeInterval?, - ) - - public fun setPlaylist(items: List) - - public fun seekTo(index: Int) -} - -@ExperimentalReadiumApi -public interface AudioEngineProvider { - - public fun createEngine( - publication: Publication, - listener: AudioEngine.Listener, - ): AudioEngine -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt index 081e429c75..e98cad7cef 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt @@ -4,6 +4,8 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(ExperimentalReadiumApi::class) + package org.readium.navigator.media.readaloud import org.readium.navigator.common.CssSelector @@ -13,6 +15,8 @@ import org.readium.navigator.common.GoLocation import org.readium.navigator.common.Location import org.readium.navigator.common.TextAnchor import org.readium.navigator.common.TextAnchorLocation +import org.readium.navigator.common.TextQuote +import org.readium.navigator.common.TextQuoteLocation import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Locator.Locations @@ -74,11 +78,32 @@ internal data class TtsLocation( } @ExperimentalReadiumApi -public data class UtteranceLocation( +public sealed interface UtteranceLocation : ExportableLocation { + override val href: Url + + public val textQuote: TextQuote? + public val cssSelector: CssSelector? +} + +internal data class MediaOverlaysUtteranceLocation( + override val href: Url, + private val mediaType: MediaType?, + override val cssSelector: CssSelector?, +) : UtteranceLocation, CssSelectorLocation { + + override val textQuote: TextQuote? = null + + override fun toLocator(): Locator { + TODO("Not yet implemented") + } +} + +internal data class TtsUtteranceLocation( override val href: Url, private val mediaType: MediaType?, override val cssSelector: CssSelector?, -) : ExportableLocation, CssSelectorLocation { + override val textQuote: TextQuote, +) : UtteranceLocation, CssSelectorLocation, TextQuoteLocation { override fun toLocator(): Locator { TODO("Not yet implemented") diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index 9c9d6495c4..0ea92dc0c3 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -16,33 +16,36 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.withContext import org.readium.navigator.common.CssSelector +import org.readium.navigator.common.TextQuote import org.readium.navigator.media.common.MediaNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.extensions.addPrefix import org.readium.r2.shared.extensions.mapStateIn import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.Error as BaseError +import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.data.ReadError @ExperimentalReadiumApi -public class ReadAloudNavigator private constructor( +public class ReadAloudNavigator private constructor( private val guidedNavigationTree: ReadAloudNode, private val resources: List, - audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, - ttsEngineFactory: () -> TtsEngine, + audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, + ttsEngineProvider: TtsEngineProvider, initialSettings: ReadAloudSettings, initialLocation: ReadAloudGoLocation?, ) { public companion object { - internal suspend operator fun invoke( + internal suspend operator fun invoke( initialLocation: ReadAloudGoLocation?, initialSettings: ReadAloudSettings, publication: ReadAloudPublication, - audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, - ttsEngineFactory: () -> TtsEngine, - ): ReadAloudNavigator { + audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, + ttsEngineProvider: TtsEngineProvider, + ): ReadAloudNavigator { val tree = withContext(Dispatchers.Default) { ReadAloudNode.fromGuidedNavigationObject(publication.guidedNavigationTree) } @@ -51,7 +54,7 @@ public class ReadAloudNavigator private constructor( guidedNavigationTree = tree, resources = publication.resources, audioEngineFactory = audioEngineFactory, - ttsEngineFactory = ttsEngineFactory, + ttsEngineProvider = ttsEngineProvider, initialSettings = initialSettings, initialLocation = initialLocation ) @@ -62,7 +65,7 @@ public class ReadAloudNavigator private constructor( val state: State, val playWhenReady: Boolean, val node: ReadAloudNode, - val utteranceLocation: UtteranceLocation, + val utteranceLocation: UtteranceLocation?, ) public sealed interface State { @@ -78,10 +81,10 @@ public class ReadAloudNavigator private constructor( public sealed class Error( override val message: String, - override val cause: org.readium.r2.shared.util.Error?, - ) : org.readium.r2.shared.util.Error { + override val cause: BaseError?, + ) : BaseError { - public data class EngineError(override val cause: org.readium.r2.shared.util.Error) : + public data class EngineError(override val cause: BaseError) : Error("An error occurred in the playback engine.", cause) public data class ContentError(override val cause: ReadError) : @@ -92,20 +95,91 @@ public class ReadAloudNavigator private constructor( override fun onItemChanged(engine: AudioEngine, index: Int) { with(stateMachine) { - stateMutable.value = stateMutable.value.onAudioEngineItemChanged(engine, index) + if (engine == stateMutable.value.segment.player.player) { + stateMutable.value = stateMutable.value.onAudioEngineItemChanged(index) + } } } override fun onStateChanged(engine: AudioEngine, state: AudioEngine.State) { with(stateMachine) { - stateMutable.value = stateMutable.value.onAudioEngineStateChanged(engine, state) + if (engine == stateMutable.value.segment.player.player) { + stateMutable.value = stateMutable.value.onAudioEngineStateChanged(state) + } + } + } + } + + private inner class TtsPlayerListener : TtsPlayer.Listener { + + override fun onItemChanged( + player: TtsPlayer, + index: Int, + ) { + with(stateMachine) { + if (player == stateMutable.value.segment.player.player) { + stateMutable.value = stateMutable.value.onTtsPlayerItemChanged(index) + } + } + } + + override fun onStateChanged( + player: TtsPlayer, + state: TtsPlayer.State, + ) { + with(stateMachine) { + if (player == stateMutable.value.segment.player.player) { + stateMutable.value = stateMutable.value.onTtsPlayerStateChanged(state) + } } } } private val segmentFactory = ReadAloudSegmentFactory( - audioEngineFactory = { audioEngineFactory(AudioEngineListener()) }, - ttsEngineFactory = ttsEngineFactory + audioEngineFactory = { playlist: List -> + audioEngineFactory(playlist, AudioEngineListener()) + }, + ttsPlayerFactory = { language: Language?, utterances: List -> + val engineFactory = { engineListener: TtsEngine.Listener -> + // FIXME support engine provider with no voice + /*val preferredVoiceWithRegion = + settings.voices[language] + ?.let { voiceForName(it.value) } + + val preferredVoiceWithoutRegion = + settings.voices[language.removeRegion()] + ?.let { voiceForName(it.value) } + + + + val voice = preferredVoiceWithRegion + ?: preferredVoiceWithoutRegion + ?: run { + voiceSelector + .voice(language, voices) + ?.let { voiceForName(it.id.value) } + }*/ + + val voice = ttsEngineProvider.voices + .firstOrNull { voice -> + language in voice.languages + } ?: ttsEngineProvider.voices + .firstOrNull { voice -> + language?.removeRegion() in voice.languages.map { it.removeRegion() } + } + ?: ttsEngineProvider.voices.first() + val engine = ttsEngineProvider.createEngine( + voice = checkNotNull(voice), + utterances = utterances, + listener = engineListener + ) + engine as? PausableTtsEngine ?: PauseDecorator(engine) + } + TtsPlayer( + engineFactory = engineFactory, + listener = TtsPlayerListener() + ) + } ) private val dataLoader = ReadAloudDataLoader(segmentFactory, initialSettings) @@ -121,41 +195,83 @@ public class ReadAloudNavigator private constructor( nodeFromLocation ?: guidedNavigationTree.firstContentNode() ?: guidedNavigationTree } - private val stateMutable: MutableStateFlow = + private val stateMutable: MutableStateFlow = run { + val itemRef = dataLoader.getItemRef(initialNode)!! + MutableStateFlow( stateMachine.play( - segment = dataLoader.getItemRef(initialNode)!!.segment, - playWhenReady = true, + segment = itemRef.segment, + index = itemRef.nodeIndex ?: 0, + playWhenReady = false, settings = initialSettings ) ) + } private val coroutineScope: CoroutineScope = MainScope() + init { + stateMutable.value.segment.player.prepare() + play() + } + public val playback: StateFlow = stateMutable.mapStateIn(coroutineScope) { state -> Playback( playWhenReady = state.playWhenReady, state = state.playbackState.toState(), node = state.node, - utteranceLocation = state.node.utteranceLocation + utteranceLocation = state.utteranceLocation ) } - private val ReadAloudNode.utteranceLocation: UtteranceLocation get() { + private val ReadAloudStateMachine.State.utteranceLocation: UtteranceLocation? get() = + when (segment) { + is AudioSegment -> { + val textref = segment.textRefs[index] + val href = textref.removeFragment() + val cssSelector = textref.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + MediaOverlaysUtteranceLocation( + href = href, + mediaType = resources.first { item -> item.href == href }.mediaType, + cssSelector = cssSelector + ) + } + is TtsSegment<*> -> null + } + + private val ReadAloudNode.mediaOverlaysUtteranceLocation: UtteranceLocation? get() { val textref = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } - checkNotNull(textref) + ?: return null val href = textref.url.removeFragment() val cssSelector = textref.url.fragment ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } - return UtteranceLocation( + return MediaOverlaysUtteranceLocation( href = href, mediaType = resources.first { item -> item.href == href }.mediaType, cssSelector = cssSelector ) } + private val ReadAloudNode.ttsUtteranceLocation: UtteranceLocation? get() { + val textref = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + ?: return null + val href = textref.url.removeFragment() + val cssSelector = textref.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + val text = text + ?: return null + return TtsUtteranceLocation( + href = href, + mediaType = resources.first { item -> item.href == href }.mediaType, + cssSelector = cssSelector, + // FIXME: prefix and suffix + textQuote = TextQuote(text.plain!!, prefix = "", suffix = "") + ) + } + private fun ReadAloudStateMachine.PlaybackState.toState(): State = when (this) { is ReadAloudStateMachine.PlaybackState.Ready -> diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index 93beee7306..0840c53068 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -12,27 +12,29 @@ import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.epub.MediaOverlaysService import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.util.Error as BaseError import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.data.ReadError import org.readium.r2.shared.util.getOrElse @ExperimentalReadiumApi -public class ReadAloudNavigatorFactory private constructor( +public class ReadAloudNavigatorFactory private constructor( private val guidedNavigationService: GuidedNavigationService, private val resources: List, - private val audioEngineFactory: (AudioEngine.Listener) -> AudioEngine, - private val ttsEngineFactory: () -> TtsEngine, + private val audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, + private val ttsEngineProvider: TtsEngineProvider, ) { public companion object { - public operator fun invoke( + public operator fun invoke( application: Application, publication: Publication, audioEngineProvider: AudioEngineProvider, - ttsEngineProvider: TtsEngineProvider, + ttsEngineProvider: TtsEngineProvider, usePrerecordedVoicesWhenAvailable: Boolean = true, - ): ReadAloudNavigatorFactory? { + ): ReadAloudNavigatorFactory? { var guidedNavService: GuidedNavigationService? = null if (usePrerecordedVoicesWhenAvailable) { @@ -43,16 +45,16 @@ public class ReadAloudNavigatorFactory private constructor( } if (guidedNavService == null) { - return null + publication.findService(ContentService::class) + ?.let { guidedNavService = TtsGuidedNavigationService(it) } } - val audioEngineFactory = { listener: AudioEngine.Listener -> - audioEngineProvider.createEngine(publication, listener) + if (guidedNavService == null) { + return null } - val ttsEngineFactory = { - val voice = ttsEngineProvider.voices.first() - ttsEngineProvider.createEngine(voice) + val audioEngineFactory = { playlist: List, listener: AudioEngine.Listener -> + audioEngineProvider.createEngine(publication, playlist, listener) } val resources = (publication.readingOrder + publication.resources).map { @@ -66,18 +68,18 @@ public class ReadAloudNavigatorFactory private constructor( guidedNavigationService = guidedNavService, resources = resources, audioEngineFactory = audioEngineFactory, - ttsEngineFactory = ttsEngineFactory + ttsEngineProvider = ttsEngineProvider ) } } public sealed class Error( override val message: String, - override val cause: org.readium.r2.shared.util.Error?, - ) : org.readium.r2.shared.util.Error { + override val cause: BaseError?, + ) : BaseError { public class UnsupportedPublication( - cause: org.readium.r2.shared.util.Error? = null, + cause: BaseError? = null, ) : Error("Publication is not supported.", cause) public class GuidedNavigationService( @@ -88,7 +90,7 @@ public class ReadAloudNavigatorFactory private constructor( public suspend fun createNavigator( initialSettings: ReadAloudSettings, initialLocation: ReadAloudGoLocation? = null, - ): Try { + ): Try, Error> { val guidedDocs = buildList { val iterator = guidedNavigationService.iterator() while (iterator.hasNext()) { @@ -118,7 +120,7 @@ public class ReadAloudNavigatorFactory private constructor( initialLocation = initialLocation, publication = navigatorPublication, audioEngineFactory = audioEngineFactory, - ttsEngineFactory = ttsEngineFactory + ttsEngineProvider = ttsEngineProvider ) return Try.success(navigator) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt index b90066596d..50b85bf334 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt @@ -11,6 +11,9 @@ package org.readium.navigator.media.readaloud import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationAudioRef import org.readium.r2.shared.guided.GuidedNavigationText +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.Error +import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.TemporalFragmentParser import org.readium.r2.shared.util.TimeInterval import org.readium.r2.shared.util.Url @@ -19,28 +22,29 @@ internal sealed interface ReadAloudSegment { val nodes: List - val engine: ReadAloudEngine + val player: SegmentPlayer val emptyNodes: Set } internal data class AudioSegment( - override val engine: AudioEngine, + override val player: AudioSegmentPlayer, val items: List, + val textRefs: List, override val nodes: List, override val emptyNodes: Set, ) : ReadAloudSegment -internal data class TtsSegment( - override val engine: TtsEngine, +internal data class TtsSegment( + override val player: TtsSegmentPlayer, val items: List, override val nodes: List, override val emptyNodes: Set, ) : ReadAloudSegment -internal class ReadAloudSegmentFactory( - private val audioEngineFactory: () -> AudioEngine, - private val ttsEngineFactory: () -> TtsEngine, +internal class ReadAloudSegmentFactory( + private val audioEngineFactory: (List) -> AudioEngine, + private val ttsPlayerFactory: (Language?, List) -> TtsPlayer, ) { fun createSegmentFromNode(node: ReadAloudNode): ReadAloudSegment? = @@ -54,15 +58,17 @@ internal class ReadAloudSegmentFactory( ): AudioSegment { var nextNode: ReadAloudNode? = firstNode val audioItems = mutableListOf() + val textRefs = mutableListOf() val nodes = mutableListOf() val emptyNodes = mutableSetOf() while (nextNode != null && nextNode.content !is TextContent) { - val audioItem = (nextNode.content as? AudioContent)?.toAudioItem() + val audioContent = (nextNode.content as? AudioContent) - if (audioItem != null) { - audioItems.add(audioItem) + if (audioContent != null) { + audioItems.add(audioContent.toAudioItem()) nodes.add(nextNode) + textRefs.add(audioContent.textRef) } else { emptyNodes.add(nextNode) } @@ -70,13 +76,13 @@ internal class ReadAloudSegmentFactory( nextNode = nextNode.next() } - val audioEngine = audioEngineFactory() + val audioEngine = audioEngineFactory(audioItems) audioEngine.playWhenReady = false - audioEngine.setPlaylist(audioItems) return AudioSegment( - engine = audioEngine, + player = AudioSegmentPlayer(audioEngine), items = audioItems, + textRefs = textRefs, nodes = nodes, emptyNodes = emptyNodes ) @@ -84,13 +90,18 @@ internal class ReadAloudSegmentFactory( private fun createTtsSegmentFromNode( firstNode: ReadAloudNode, - ): TtsSegment { + ): TtsSegment { var nextNode: ReadAloudNode? = firstNode + val segmentLanguage = firstNode.text?.language val textItems = mutableListOf() val nodes = mutableListOf() val emptyNodes = mutableSetOf() - while (nextNode != null && nextNode.content !is AudioContent) { + while ( + nextNode != null && + nextNode.content !is AudioContent && + nextNode.language == segmentLanguage + ) { val textContent = (nextNode.content as? TextContent) if (textContent != null) { @@ -103,22 +114,38 @@ internal class ReadAloudSegmentFactory( nextNode = nextNode.next() } + val utterances = textItems.map { it.plain!! } + + val ttsPlayer = ttsPlayerFactory(segmentLanguage, utterances) + return TtsSegment( - engine = ttsEngineFactory(), + player = TtsSegmentPlayer(ttsPlayer), items = textItems, nodes = nodes, emptyNodes = emptyNodes ) } + private val ReadAloudNode.language: Language? get() = when (content) { + is TextContent -> text?.language + else -> null + } + private val ReadAloudNode.content: NodeContent? get() { refs .firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } - ?.let { - return AudioContent( - href = it.url.removeFragment(), - interval = it.url.timeInterval - ) + ?.let { audioRef -> + + val textRef = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + + // Ignore audio nodes without textref. Might be changed later. + textRef?.let { + return AudioContent( + href = audioRef.url.removeFragment(), + interval = audioRef.url.timeInterval, + textRef = it.url + ) + } } text @@ -135,6 +162,7 @@ private sealed interface NodeContent private data class AudioContent( val href: Url, val interval: TimeInterval?, + val textRef: Url, ) : NodeContent private data class TextContent( diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 8f516ee141..e64340b47a 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -11,8 +11,8 @@ package org.readium.navigator.media.readaloud import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Error -internal class ReadAloudStateMachine( - private val dataLoader: ReadAloudDataLoader, +internal class ReadAloudStateMachine( + private val dataLoader: ReadAloudDataLoader, private val navigationHelper: ReadAloudNavigationHelper, ) { @@ -31,42 +31,38 @@ internal class ReadAloudStateMachine( val playbackState: PlaybackState, val playWhenReady: Boolean, val node: ReadAloudNode, + val index: Int, val segment: ReadAloudSegment, val settings: ReadAloudSettings, ) - sealed interface Event { - - data class AudioEngineStateChanged( - val engine: AudioEngine, - val state: AudioEngine.State, - ) : Event - - data class AudioEngineItemChanged( - val engine: AudioEngine, - val index: Int, - ) : Event - } + fun play( + segment: ReadAloudSegment, + index: Int, + playWhenReady: Boolean, + settings: ReadAloudSettings, + ): State { + segment.player.seekTo(index) - fun play(segment: ReadAloudSegment, playWhenReady: Boolean, settings: ReadAloudSettings): State { - segment.engine.playWhenReady = playWhenReady + segment.player.playWhenReady = playWhenReady return State( playbackState = PlaybackState.Starved, playWhenReady = playWhenReady, - node = segment.nodes[0], + node = segment.nodes[index], + index = index, segment = segment, settings = settings ) } fun State.pause(): State { - segment.engine.playWhenReady = false + segment.player.playWhenReady = false return copy(playWhenReady = false) } fun State.resume(): State { - segment.engine.playWhenReady = true + segment.player.playWhenReady = true return copy(playWhenReady = true) } @@ -77,14 +73,9 @@ internal class ReadAloudStateMachine( val itemRef = dataLoader.getItemRef(firstContentNode) ?: return copy(playbackState = PlaybackState.Ended) - val engineFood = itemRef.segment + val index = itemRef.nodeIndex!! - when (engineFood) { - is AudioSegment -> engineFood.engine.seekTo(itemRef.nodeIndex!!) - is TtsSegment -> TODO() - } - - return play(engineFood, playWhenReady, settings) + return play(itemRef.segment, index, playWhenReady, settings) } fun State.updateSettings( @@ -96,45 +87,48 @@ internal class ReadAloudStateMachine( return copy(settings = newSettings) } - fun State.onEvent(event: Event): State = when (event) { - is Event.AudioEngineStateChanged -> - onAudioEngineStateChanged(event.engine, event.state) - is Event.AudioEngineItemChanged -> - onAudioEngineItemChanged(event.engine, event.index) - } - - fun State.onAudioEngineStateChanged(engine: AudioEngine, audioEngineState: AudioEngine.State): State { - val currentEngine = (segment as? AudioSegment)?.engine - if (currentEngine != engine) { - return this - } - + fun State.onAudioEngineStateChanged(audioEngineState: AudioEngine.State): State { return when (audioEngineState) { AudioEngine.State.Ready -> copy(playbackState = PlaybackState.Ready) AudioEngine.State.Starved -> copy(playbackState = PlaybackState.Starved) - AudioEngine.State.Ended -> onAudioEngineEnded() + AudioEngine.State.Ended -> onSegmentPlaybackEnded() } } - private fun State.onAudioEngineEnded(): State { + fun State.onAudioEngineItemChanged(item: Int): State { + dataLoader.onPlaybackProgressed(segment.nodes[item]) + return copy(node = segment.nodes[item], index = item) + } + + fun State.onTtsPlayerStateChanged(state: TtsPlayer.State): State { + return when (state) { + TtsPlayer.State.Ready -> copy(playbackState = PlaybackState.Ready) + TtsPlayer.State.Starved -> copy(playbackState = PlaybackState.Starved) + TtsPlayer.State.Ended -> onSegmentPlaybackEnded() + } + } + + fun State.onTtsPlayerItemChanged(item: Int): State { + dataLoader.onPlaybackProgressed(segment.nodes[item]) + return copy(node = segment.nodes[item], index = item) + } + + private fun State.onSegmentPlaybackEnded(): State { + segment.player.release() + val nextNode = node.next() ?: return copy(playbackState = PlaybackState.Ended) val newSegment = dataLoader.getItemRef(nextNode)?.segment ?: return copy(playbackState = PlaybackState.Ended) - newSegment.engine.playWhenReady = playWhenReady + newSegment.player.playWhenReady = playWhenReady - return copy(segment = newSegment, playbackState = PlaybackState.Starved, node = newSegment.nodes[0]) - } - - fun State.onAudioEngineItemChanged(engine: AudioEngine, item: Int): State { - val currentEngine = (segment as? AudioSegment)?.engine - if (currentEngine != engine) { - return this - } - - dataLoader.onPlaybackProgressed(segment.nodes[item]) - return copy(node = segment.nodes[item]) + return copy( + segment = newSegment, + playbackState = PlaybackState.Starved, + node = newSegment.nodes[0], + index = 0 + ) } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt new file mode 100644 index 0000000000..ff3f8fbee2 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt @@ -0,0 +1,70 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Error + +internal sealed interface SegmentPlayer { + + val player: Any + + var playWhenReady: Boolean + + fun prepare() + + fun seekTo(index: Int) + + fun release() +} + +internal class TtsSegmentPlayer( + override val player: TtsPlayer, +) : SegmentPlayer { + + override var playWhenReady: Boolean + get() = player.playWhenReady + set(value) { + player.playWhenReady = value + } + + override fun prepare() { + player.prepare() + } + + override fun seekTo(index: Int) { + player.seekTo(index) + } + + override fun release() { + player.release() + } +} + +internal class AudioSegmentPlayer( + override val player: AudioEngine, +) : SegmentPlayer { + + override var playWhenReady: Boolean + get() = player.playWhenReady + set(value) { + player.playWhenReady = value + } + + override fun prepare() { + } + + override fun seekTo(index: Int) { + player.seekTo(index) + } + + override fun release() { + player.release() + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt new file mode 100644 index 0000000000..a8ce681e6f --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Error +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +public interface TtsEngine { + + public interface Listener { + + public fun onReady() + + public fun onDone() + + public fun onInterrupted() + + /** + * Called when the [TtsEngine] is about to speak the specified [range] of the utterance with + * the given id. + * + * This callback may not be called if the [TtsEngine] does not provide range information. + */ + public fun onRange(range: IntRange) + + /** + * Called when an error has occurred during processing of the utterance with the given id. + */ + public fun onError(error: E) + } + + public val utterances: List + + public fun prepare() + + public fun speak(utteranceIndex: Int) + + public fun stop() + + public fun release() +} + +@ExperimentalReadiumApi +public interface TtsVoice { + + @JvmInline + public value class Id(public val value: String) + + public val id: Id + + /** + * The languages supported by the voice. + */ + public val languages: Set +} + +@ExperimentalReadiumApi +public interface TtsEngineProvider { + + /** + * Sets of voices available with this [TtsEngineProvider]. + */ + public val voices: Set + + public fun createEngine( + voice: V, + utterances: List, + listener: TtsEngine.Listener, + ): TtsEngine +} + +@ExperimentalReadiumApi +public interface PausableTtsEngine : TtsEngine { + + public fun pause() + + public fun resume() +} + +internal class PauseDecorator( + private val engine: TtsEngine, +) : PausableTtsEngine, TtsEngine by engine { + + private var currentIndex: Int = 0 + + override fun pause() { + engine.stop() + } + + override fun resume() { + engine.speak(currentIndex) + } + + override fun speak(utteranceIndex: Int) { + currentIndex = utteranceIndex + engine.speak(utteranceIndex) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt new file mode 100644 index 0000000000..af73ebb7ee --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt @@ -0,0 +1,108 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationText +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.data.ReadError + +internal class TtsGuidedNavigationService( + private val contentService: ContentService, +) : GuidedNavigationService { + + override fun iterator(): GuidedNavigationIterator { + val contentIterator = contentService.content(start = null).iterator() + return Iterator(contentIterator) + } + + private class Iterator( + private val contentIterator: Content.Iterator, + ) : GuidedNavigationIterator { + + private var guidedNavigationDocument: GuidedNavigationDocument? = null + + private var ended: Boolean = false + + override suspend fun hasNext(): Boolean { + if (ended) { + return false + } + + guidedNavigationDocument = createGuidedNavigationDocument() + ended = true + return true + } + + override suspend fun next(): Try { + val res = checkNotNull(guidedNavigationDocument) + return Try.success(res) + } + + private suspend fun createGuidedNavigationDocument(): GuidedNavigationDocument { + val tree = mutableListOf() + + while (contentIterator.hasNext()) { + val nodes = when (val element = contentIterator.next()) { + is Content.TextElement -> { + element.segments.mapNotNull { segment -> + if (segment.text.isEmpty()) { + return@mapNotNull null + } + GuidedNavigationObject( + refs = setOf( + GuidedNavigationTextRef(segment.locator.href) + ), + text = GuidedNavigationText( + plain = segment.text, + ssml = null, + language = segment.language + ), + ) + } + } + + is Content.TextualElement -> { + listOfNotNull( + element.text + ?.takeIf { it.isNotBlank() } + ?.let { + GuidedNavigationObject( + refs = setOf( + GuidedNavigationTextRef(element.locator.href) + ), + text = GuidedNavigationText(it) + ) + } + ) + } + + else -> emptyList() + } + + if (nodes.isNotEmpty()) { + tree.add( + GuidedNavigationObject( + children = nodes + ) + ) + } + } + + return GuidedNavigationDocument(guided = tree) + } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt new file mode 100644 index 0000000000..412b51de92 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt @@ -0,0 +1,162 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import kotlin.properties.Delegates +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Error + +internal class TtsPlayer( + engineFactory: (TtsEngine.Listener) -> PausableTtsEngine, + private val listener: Listener, +) { + + interface Listener { + + fun onItemChanged(player: TtsPlayer, index: Int) + + fun onStateChanged(player: TtsPlayer, state: State) + } + + internal enum class State { + Ready, + Starved, + Ended, + } + + private inner class EngineListener : TtsEngine.Listener { + override fun onReady() { + with(stateMachine) { + state = state.onEngineReady() + } + } + + override fun onDone() { + with(stateMachine) { + state = state.onUtteranceCompleted() + } + } + + override fun onInterrupted() { + } + + override fun onRange(range: IntRange) { + } + + override fun onError(error: E) { + } + } + + private val ttsEngine = engineFactory(EngineListener()) + + private val stateMachine = StateMachine(ttsEngine) + + private var state by Delegates.observable( + StateMachine.start(playWhenReady = false) + ) { property, oldValue, newValue -> + if (oldValue.playbackState != newValue.playbackState) { + val state = when (newValue.playbackState) { + StateMachine.PlaybackState.Ended -> State.Ended + StateMachine.PlaybackState.Ready -> State.Ready + StateMachine.PlaybackState.Starved -> State.Starved + } + listener.onStateChanged(this, state) + } + + if (oldValue.index != newValue.index) { + listener.onItemChanged(this, newValue.index) + } + } + + var playWhenReady: Boolean + get() = state.playWhenReady + set(value) { + with(stateMachine) { + state = if (value) state.resume() else state.pause() + } + } + + fun prepare() { + ttsEngine.prepare() + } + + fun seekTo(index: Int) { + with(stateMachine) { + state = state.seekTo(index) + } + } + + fun release() { + ttsEngine.release() + } +} + +private class StateMachine( + private val engine: PausableTtsEngine, +) { + sealed interface PlaybackState { + + data object Ready : PlaybackState + + data object Starved : PlaybackState + + data object Ended : PlaybackState + } + + data class State( + val playbackState: PlaybackState, + val playWhenReady: Boolean, + val index: Int, + ) + + companion object { + + fun start(playWhenReady: Boolean): State { + return State( + playWhenReady = playWhenReady, + playbackState = PlaybackState.Starved, + index = 0 + ) + } + } + + fun State.pause(): State { + engine.pause() + return copy(playWhenReady = false) + } + + fun State.resume(): State { + engine.resume() + return copy(playWhenReady = true) + } + + fun State.seekTo(index: Int): State { + if (playWhenReady) { + engine.stop() + engine.speak(index) + } + return copy(index = index) + } + + fun State.onUtteranceCompleted(): State { + if (index < engine.utterances.size - 1) { + engine.speak(index + 1) + return copy(index = index + 1) + } else { + return copy(playbackState = PlaybackState.Ended) + } + } + + fun State.onEngineReady(): State { + if (playWhenReady) { + engine.speak(index) + } + return copy(playbackState = PlaybackState.Ready) + } +} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt index 6173aa60d2..e05c38fdb9 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt @@ -16,7 +16,7 @@ import org.readium.r2.shared.util.Url */ @ExperimentalReadiumApi public data class GuidedNavigationDocument( - val links: List, + val links: List = emptyList(), val guided: List, ) @@ -25,10 +25,10 @@ public data class GuidedNavigationDocument( */ @ExperimentalReadiumApi public data class GuidedNavigationObject( - val children: List, - val roles: Set, - val refs: Set, - val text: GuidedNavigationText?, + val children: List = emptyList(), + val roles: Set = emptySet(), + val refs: Set = emptySet(), + val text: GuidedNavigationText? = null, ) /** @@ -42,11 +42,10 @@ public value class SsmlString(public val value: String) * Text holder for a guided navigation object. */ @ExperimentalReadiumApi -@ConsistentCopyVisibility -public data class GuidedNavigationText private constructor( +public data class GuidedNavigationText( val plain: String?, - val ssml: SsmlString?, - val language: Language?, + val ssml: SsmlString? = null, + val language: Language? = null, ) { init { require(plain != null || ssml != null) diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt index 85b263e6b3..fb52bc27e4 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt @@ -113,7 +113,11 @@ public class EpubParser( HtmlResourceContentIterator.Factory() ) ) - ).also { it[MediaOverlaysService::class] = SmilBasedMediaOverlaysService.createFactory(smils) } + ).also { + if (smils.isNotEmpty()) { + it[MediaOverlaysService::class] = SmilBasedMediaOverlaysService.createFactory(smils) + } + } ) return Try.success(builder) From 852ba8c2823b578aa30660c2fe79f0cfaecbd7c4 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Tue, 9 Sep 2025 18:10:52 +0200 Subject: [PATCH 11/13] Implement Preferences API --- .../readium/demo/navigator/DemoViewModel.kt | 28 +++-- .../preferences/PreferencesManager.kt | 5 + .../navigator/preferences/UserPreferences.kt | 47 +++++++ .../navigator/reader/ReadAloudRendition.kt | 57 ++++++++- .../demo/navigator/reader/ReaderOpener.kt | 53 +++++--- .../demo/navigator/reader/ReaderState.kt | 4 + .../navigator/reader/SelectNavigatorMenu.kt | 2 + .../demo/navigator/reader/VisualRendition.kt | 48 +------- .../exoplayer/readaloud/ExoPlayerEngine.kt | 34 +++-- .../media/readaloud/AndroidTtsEngine.kt | 106 ++++++++++------ .../navigator/media/readaloud/AudioEngine.kt | 12 +- .../media/readaloud/ReadAloudDataLoader.kt | 13 +- .../readaloud/ReadAloudNavigationHelper.kt | 1 + .../media/readaloud/ReadAloudNavigator.kt | 116 ++++++++++-------- .../readaloud/ReadAloudNavigatorFactory.kt | 17 +++ .../media/readaloud/ReadAloudSegment.kt | 21 +++- .../media/readaloud/ReadAloudStateMachine.kt | 99 +++++++++------ .../media/readaloud/SegmentPlayer.kt | 49 ++++++++ .../navigator/media/readaloud/TtsEngine.kt | 5 + .../navigator/media/readaloud/TtsPlayer.kt | 86 ++++++++----- .../{ => preferences}/ReadAloudSettings.kt | 7 +- .../r2/shared/guided/GuidedNavigationRole.kt | 12 +- 22 files changed, 555 insertions(+), 267 deletions(-) rename readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/{ => preferences}/ReadAloudSettings.kt (79%) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index a53dd78afb..c8e6743bac 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -124,19 +124,21 @@ class DemoViewModel( configuration = fixedConfig )?.let { SelectNavigatorItem.FixedWeb(it) } - val readAloudFactory = run { - val ttsEngineProvider = - AndroidTtsEngineProvider(application) - ?.takeIf { it.voices.isNotEmpty() } - ?: return@run null - - ReadAloudNavigatorFactory( - application = application, - publication = publication, - audioEngineProvider = audioEngineProvider, - ttsEngineProvider = ttsEngineProvider - )?.let { SelectNavigatorItem.ReadAloud(it) } - } + val ttsEngineProvider = + AndroidTtsEngineProvider(application) + ?.takeIf { it.voices.isNotEmpty() } + + val readAloudFactory = ttsEngineProvider + ?.let { + ReadAloudNavigatorFactory( + application = application, + publication = publication, + audioEngineProvider = audioEngineProvider, + ttsEngineProvider = ttsEngineProvider + ) + }?.let { + SelectNavigatorItem.ReadAloud(factory = it, ttsEngineProvider = ttsEngineProvider) + } val factories = listOfNotNull( reflowableFactory, diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt index 25070d40b4..21ff0adc18 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt @@ -9,6 +9,8 @@ package org.readium.demo.navigator.preferences import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow import org.readium.navigator.common.Preferences import org.readium.r2.shared.ExperimentalReadiumApi @@ -21,6 +23,9 @@ class PreferencesManager

>( private val preferencesMutable: MutableStateFlow

= MutableStateFlow(initialPreferences) + val preferences: StateFlow

= + preferencesMutable.asStateFlow() + fun setPreferences(preferences: P) { preferencesMutable.value = preferences } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt index 6d458a9072..3aa8788198 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt @@ -28,6 +28,7 @@ import org.readium.demo.navigator.reader.LITERATA import org.readium.navigator.common.Preferences import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings +import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.web.fixedlayout.preferences.FixedWebPreferencesEditor import org.readium.navigator.web.reflowable.preferences.ReflowableWebPreferencesEditor import org.readium.r2.navigator.preferences.Axis @@ -122,6 +123,15 @@ fun

, S : Settings, E : PreferencesEditor> UserPreferen visitedColor = editor.visitedColor, wordSpacing = editor.wordSpacing ) + + is ReadAloudPreferencesEditor -> { + MediaUserPreferences( + language = editor.language, + voice = editor.voice, + speed = editor.speed, + pitch = editor.pitch + ) + } } } } @@ -486,6 +496,43 @@ private fun ReflowableUserPreferences( } } +@Composable +private fun MediaUserPreferences( + language: Preference? = null, + voice: EnumPreference? = null, + speed: RangePreference? = null, + pitch: RangePreference? = null, +) { + Column { + if (speed != null) { + StepperItem( + title = "Speed", + preference = speed, + ) + } + + if (pitch != null) { + StepperItem( + title = "Pitch", + preference = pitch, + ) + } + if (language != null) { + LanguageItem( + preference = language + ) + } + + if (voice != null) { + MenuItem( + title = "Voice", + preference = voice, + formatValue = { it?.name ?: "Default" }, + ) + } + } +} + @Composable private fun Divider() { HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp)) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt index e238560b62..2e9a969bea 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -@file:OptIn(ExperimentalReadiumApi::class) +@file:OptIn(ExperimentalReadiumApi::class, ExperimentalMaterial3Api::class) package org.readium.demo.navigator.reader @@ -20,23 +20,76 @@ import androidx.compose.material.icons.filled.Pause import androidx.compose.material.icons.filled.PlayArrow import androidx.compose.material.icons.filled.SkipNext import androidx.compose.material.icons.filled.SkipPrevious +import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.ModalBottomSheet import androidx.compose.material3.Scaffold import androidx.compose.material3.Text +import androidx.compose.material3.rememberModalBottomSheetState import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp +import androidx.compose.ui.zIndex +import org.readium.demo.navigator.preferences.UserPreferences import org.readium.r2.shared.ExperimentalReadiumApi @Composable fun ReadAloudRendition( readerState: ReadAloudReaderState, ) { + val showPreferences = remember { mutableStateOf(false) } + val preferencesSheetState = rememberModalBottomSheetState() + + if (showPreferences.value) { + ModalBottomSheet( + sheetState = preferencesSheetState, + onDismissRequest = { + showPreferences.value = false + } + ) { + val preferencesEditor = readerState.preferencesEditor.collectAsState() + + UserPreferences( + editor = preferencesEditor.value, + title = "Preferences" + ) + } + } + + val showOutline = rememberSaveable { mutableStateOf(false) } + + if (showOutline.value) { + Outline( + modifier = Modifier + .zIndex(1f) + .fillMaxSize(), + publication = readerState.publication, + onBackActivated = { + showOutline.value = false + }, + onTocItemActivated = { + readerState.navigator.goTo(it) + showOutline.value = false + } + ) + } + Scaffold( - modifier = Modifier.fillMaxSize() + modifier = Modifier.fillMaxSize(), + topBar = { + RenditionTopBar( + modifier = Modifier.zIndex(10f), + visible = true, + onPreferencesActivated = { showPreferences.value = !showPreferences.value }, + onOutlineActivated = { showOutline.value = !showOutline.value } + ) + } ) { contentPadding -> Column( modifier = Modifier diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index 3f0b91b04f..6d33cbe49c 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -4,7 +4,11 @@ * available in the top-level LICENSE file of the project. */ -@file:OptIn(ExperimentalReadiumApi::class) +@file:OptIn( + ExperimentalReadiumApi::class, + InternalReadiumApi::class, + ExperimentalCoroutinesApi::class +) package org.readium.demo.navigator.reader @@ -12,7 +16,9 @@ import android.app.Application import androidx.compose.runtime.snapshotFlow import kotlinx.collections.immutable.plus import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import org.readium.demo.navigator.decorations.FixedWebHighlightsManager @@ -21,6 +27,7 @@ import org.readium.demo.navigator.decorations.ReflowableWebHighlightsManager import org.readium.demo.navigator.decorations.pageNumberDecorations import org.readium.demo.navigator.persistence.LocatorRepository import org.readium.demo.navigator.preferences.PreferencesManager +import org.readium.demo.navigator.preferences.ReadAloudPreferencesEditor import org.readium.navigator.common.DecorationController import org.readium.navigator.common.DecorationLocation import org.readium.navigator.common.PreferencesEditor @@ -28,7 +35,8 @@ import org.readium.navigator.common.Settings import org.readium.navigator.common.SettingsController import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory -import org.readium.navigator.media.readaloud.ReadAloudSettings +import org.readium.navigator.media.readaloud.TtsEngineProvider +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences import org.readium.navigator.web.fixedlayout.FixedWebGoLocation import org.readium.navigator.web.fixedlayout.FixedWebLocation import org.readium.navigator.web.fixedlayout.FixedWebRenditionController @@ -42,12 +50,12 @@ import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebSelectionLocation import org.readium.navigator.web.reflowable.preferences.ReflowableWebPreferences import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl import org.readium.r2.shared.util.Error -import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.Try import org.readium.r2.shared.util.getOrElse @@ -81,6 +89,7 @@ class ReaderOpener( url, publication, selectedNavigator.factory, + selectedNavigator.ttsEngineProvider, initialLocator ) }.getOrElse { error -> @@ -199,26 +208,38 @@ class ReaderOpener( url: AbsoluteUrl, publication: Publication, navigatorFactory: ReadAloudNavigatorFactory, + ttsEngineProvider: TtsEngineProvider, initialLocator: Locator?, ): Try { - val initialSettings = ReadAloudSettings( - language = Language("en"), - overrideContentLanguage = true, - preferRecordedVoices = true, - pitch = 1.0, - speed = 1.0, - voices = emptyMap(), - escapableRoles = GuidedNavigationRole.ESCAPABLE_ROLES.toSet(), - skippableRoles = GuidedNavigationRole.SKIPPABLE_ROLES.toSet() - ) + val coroutineScope = MainScope() + + val initialPreferences = ReadAloudPreferences() - val navigator = navigatorFactory.createNavigator(initialSettings) + val preferencesManager = PreferencesManager(initialPreferences) + + val preferencesEditor = preferencesManager.preferences.mapStateIn(coroutineScope) { + ReadAloudPreferencesEditor( + editor = navigatorFactory.createPreferencesEditor(it), + availableVoices = ttsEngineProvider.voices + ) + } + + val navigator = navigatorFactory.createNavigator(preferencesEditor.value.settings) .getOrElse { return Try.failure(it) } + preferencesEditor + .flatMapLatest { it.preferencesState } + .onEach { + navigator.settings = preferencesEditor.value.settings + preferencesManager.setPreferences(it) + }.launchIn(coroutineScope) + val readerState = ReadAloudReaderState( url = url, + coroutineScope = coroutineScope, publication = publication, - navigator = navigator + navigator = navigator, + preferencesEditor = preferencesEditor ) return Try.success(readerState) } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt index a8a2846b7e..631a8c5aed 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt @@ -10,7 +10,9 @@ package org.readium.demo.navigator.reader import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.StateFlow import org.readium.demo.navigator.decorations.HighlightsManager +import org.readium.demo.navigator.preferences.ReadAloudPreferencesEditor import org.readium.navigator.common.ExportableLocation import org.readium.navigator.common.GoLocation import org.readium.navigator.common.NavigationController @@ -48,8 +50,10 @@ data class VisualReaderState, + val preferencesEditor: StateFlow, ) : ReaderState { override fun close() { diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt index 88e7970c3b..76469d121f 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt @@ -15,6 +15,7 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.window.PopupProperties import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.media.readaloud.TtsEngineProvider import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.r2.shared.ExperimentalReadiumApi @@ -50,6 +51,7 @@ sealed class SelectNavigatorItem( data class ReadAloud( override val factory: ReadAloudNavigatorFactory, + val ttsEngineProvider: TtsEngineProvider, ) : SelectNavigatorItem("Read Aloud Navigator") } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt index 05b2302d1c..dd2622b3f2 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt @@ -9,16 +9,10 @@ package org.readium.demo.navigator.reader import androidx.activity.compose.BackHandler -import androidx.compose.animation.AnimatedVisibility -import androidx.compose.animation.fadeIn -import androidx.compose.animation.fadeOut import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.ModalBottomSheet -import androidx.compose.material3.TopAppBar import androidx.compose.material3.rememberModalBottomSheetState import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect @@ -30,12 +24,10 @@ import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.res.painterResource import androidx.compose.ui.zIndex import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch -import org.readium.demo.navigator.R import org.readium.demo.navigator.decorations.DecorationStyleAnnotationMark import org.readium.demo.navigator.decorations.EditAnnotationDialog import org.readium.demo.navigator.decorations.EditAnnotationViewModel @@ -116,7 +108,7 @@ fun VisualRen } Box { - TopBar( + RenditionTopBar( modifier = Modifier.zIndex(10f), visible = !fullScreenState.value, onPreferencesActivated = { showPreferences.value = !showPreferences.value }, @@ -270,41 +262,3 @@ fun VisualRen } } } - -@OptIn(ExperimentalMaterial3Api::class) -@Composable -private fun TopBar( - modifier: Modifier, - visible: Boolean, - onPreferencesActivated: () -> Unit, - onOutlineActivated: () -> Unit, -) { - AnimatedVisibility( - modifier = modifier, - visible = visible, - enter = fadeIn(), - exit = fadeOut() - ) { - TopAppBar( - title = { }, - actions = { - IconButton( - onClick = onPreferencesActivated - ) { - Icon( - painterResource(R.drawable.ic_preferences_24), - contentDescription = "Preferences", - ) - } - IconButton( - onClick = onOutlineActivated - ) { - Icon( - painterResource(R.drawable.ic_outline_24), - contentDescription = "Outline" - ) - } - } - ) - } -} diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index bae0b3a588..c42fe80ad8 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -74,13 +74,8 @@ public class ExoPlayerEngine private constructor( override fun onEvents(player: Player, events: Player.Events) { if (events.contains(Player.EVENT_PLAYBACK_STATE_CHANGED)) { - val newState = when (player.playbackState) { - Player.STATE_READY -> AudioEngine.State.Ready - Player.STATE_BUFFERING -> AudioEngine.State.Starved - Player.STATE_ENDED -> AudioEngine.State.Ended - else -> null - } - newState?.let { this@ExoPlayerEngine.listener.onStateChanged(this@ExoPlayerEngine, it) } + val newState = player.getEnginePlaybackState() + newState.let { this@ExoPlayerEngine.listener.onStateChanged(this@ExoPlayerEngine, it) } } if (events.contains(Player.EVENT_MEDIA_ITEM_TRANSITION)) { @@ -131,6 +126,23 @@ public class ExoPlayerEngine private constructor( exoPlayer.playWhenReady = value } + override val playbackState: AudioEngine.PlaybackState + get() = exoPlayer.getEnginePlaybackState() + + override var pitch: Double + get() = exoPlayer.playbackParameters.pitch.toDouble() + set(value) { + exoPlayer.playbackParameters = + exoPlayer.playbackParameters.withPitch(value.toFloat()) + } + + override var speed: Double + get() = exoPlayer.playbackParameters.speed.toDouble() + set(value) { + exoPlayer.playbackParameters = + exoPlayer.playbackParameters.withSpeed(value.toFloat()) + } + public override fun release() { coroutineScope.cancel() exoPlayer.release() @@ -151,4 +163,12 @@ public class ExoPlayerEngine private constructor( Error.Source(readError) } } + + private fun Player.getEnginePlaybackState(): AudioEngine.PlaybackState = + when (playbackState) { + Player.STATE_READY -> AudioEngine.PlaybackState.Ready + Player.STATE_BUFFERING, Player.STATE_IDLE -> AudioEngine.PlaybackState.Starved + Player.STATE_ENDED -> AudioEngine.PlaybackState.Ended + else -> throw IllegalStateException("Unexpected ExoPlayer state $playbackState") + } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt index 78d0b47759..89313cfb47 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt @@ -14,13 +14,30 @@ import android.content.Intent import android.content.pm.PackageManager import android.os.Build import android.speech.tts.TextToSpeech -import android.speech.tts.TextToSpeech.* +import android.speech.tts.TextToSpeech.ERROR_INVALID_REQUEST +import android.speech.tts.TextToSpeech.ERROR_NETWORK +import android.speech.tts.TextToSpeech.ERROR_NETWORK_TIMEOUT +import android.speech.tts.TextToSpeech.ERROR_NOT_INSTALLED_YET +import android.speech.tts.TextToSpeech.ERROR_OUTPUT +import android.speech.tts.TextToSpeech.ERROR_SERVICE +import android.speech.tts.TextToSpeech.ERROR_SYNTHESIS +import android.speech.tts.TextToSpeech.Engine +import android.speech.tts.TextToSpeech.OnInitListener +import android.speech.tts.TextToSpeech.QUEUE_ADD +import android.speech.tts.TextToSpeech.SUCCESS import android.speech.tts.UtteranceProgressListener import android.speech.tts.Voice as AndroidVoice -import android.speech.tts.Voice.* +import android.speech.tts.Voice.QUALITY_HIGH +import android.speech.tts.Voice.QUALITY_LOW +import android.speech.tts.Voice.QUALITY_NORMAL +import android.speech.tts.Voice.QUALITY_VERY_HIGH +import android.speech.tts.Voice.QUALITY_VERY_LOW import java.util.UUID -import kotlin.collections.orEmpty -import kotlinx.coroutines.* +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch import org.readium.navigator.media.readaloud.AndroidTtsEngine.Companion.initializeTextToSpeech import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi @@ -31,27 +48,25 @@ import org.readium.r2.shared.util.Language public class AndroidTtsEngineProvider private constructor( private val context: Context, private val textToSpeech: TextToSpeech, + private val fullVoices: Map, ) : TtsEngineProvider { override val voices: Set = - tryOrNull { textToSpeech.voices } // throws on Nexus 4 - ?.map { it.toTtsEngineVoice() } - ?.toSet() - .orEmpty() + fullVoices.keys override fun createEngine( voice: AndroidTtsEngine.Voice, utterances: List, listener: TtsEngine.Listener, ): TtsEngine { - val voice = voices.firstOrNull { it.name == voice.name } + val voice = fullVoices[voice] checkNotNull(voice) return AndroidTtsEngine( context = context, engine = textToSpeech, listener = listener, - voice = voice, + androidVoice = voice, utterances = utterances ) } @@ -60,11 +75,28 @@ public class AndroidTtsEngineProvider private constructor( public suspend operator fun invoke( context: Context, + maxRetries: Int = 2, ): AndroidTtsEngineProvider? { + require(maxRetries >= 0) + + suspend fun onFailure(): AndroidTtsEngineProvider? = + if (maxRetries == 0) { + null + } else { + invoke(context, maxRetries - 1) + } + val textToSpeech = initializeTextToSpeech(context) - ?: return null + ?: return onFailure() - return AndroidTtsEngineProvider(context, textToSpeech) + // Listing voices is not reliable. + val voices = tryOrNull { textToSpeech.voices } // throws on Nexus 4 + ?.filterNotNull() + ?.takeUnless { it.isEmpty() } + ?.associate { it.toTtsEngineVoice() to it } + ?: return onFailure() + + return AndroidTtsEngineProvider(context, textToSpeech, voices) } private fun AndroidVoice.toTtsEngineVoice() = @@ -88,8 +120,8 @@ public class AndroidTtsEngineProvider private constructor( * On some Android implementations (i.e. on Oppo A9 2020 running Android 11), * the TextToSpeech instance is often disconnected from the underlying service when the playback * is paused and the app moves to the background. So we try to reset the TextToSpeech before - * actually returning an error. In the meantime, requests to the engine are queued - * into [pendingRequests]. + * actually returning an error. In the meantime, requests to the engine are stored in the adapter + * state. */ /** @@ -100,7 +132,7 @@ public class AndroidTtsEngine internal constructor( private val context: Context, engine: TextToSpeech, private val listener: TtsEngine.Listener, - public val voice: Voice, + private val androidVoice: AndroidVoice, override val utterances: List, ) : TtsEngine { @@ -254,21 +286,17 @@ public class AndroidTtsEngine internal constructor( ) : State() } - private val coroutineScope: CoroutineScope = - MainScope() + private val coroutineScope: CoroutineScope = MainScope() - private var state: State = - State.EngineAvailable(engine) + private var state: State = State.EngineAvailable(engine) - private var isPrepared: Boolean = - false + private var isPrepared: Boolean = false - private var isClosed: Boolean = - false + private var isClosed: Boolean = false - private var speed: Double = 1.0 + override var speed: Double = 1.0 - private var pitch: Double = 1.0 + override var pitch: Double = 1.0 override fun prepare() { if (isPrepared) { @@ -279,6 +307,7 @@ public class AndroidTtsEngine internal constructor( (state as? State.EngineAvailable) ?.let { setupListener(it.engine) } + listener.onReady() } @@ -346,9 +375,9 @@ public class AndroidTtsEngine internal constructor( engine: TextToSpeech, request: Request, ): Boolean { - engine.setupPitchAndSpeed() - return engine.setupVoice() && - (engine.speak(request.text, QUEUE_ADD, null, request.id.value) == SUCCESS) + return engine.setupPitchAndSpeed() && + engine.setupVoice() && + engine.speak(request.text, QUEUE_ADD, null, request.id.value) == SUCCESS } private fun setupListener(engine: TextToSpeech) { @@ -358,7 +387,6 @@ public class AndroidTtsEngine internal constructor( private fun onReconnectionSucceeded(engine: TextToSpeech) { val previousState = state as State.WaitingForService setupListener(engine) - engine.setupPitchAndSpeed() state = State.EngineAvailable(engine) if (isClosed) { engine.shutdown() @@ -387,19 +415,21 @@ public class AndroidTtsEngine internal constructor( engine.shutdown() } - private fun TextToSpeech.setupPitchAndSpeed() { - setSpeechRate(speed.toFloat()) - setPitch(pitch.toFloat()) - } + private fun TextToSpeech.setupPitchAndSpeed(): Boolean { + if (setSpeechRate(speed.toFloat()) != SUCCESS) { + return false + } + + if (setPitch(pitch.toFloat()) != SUCCESS) { + return false + } - private fun TextToSpeech.setupVoice(): Boolean { - val voice = voiceForName(voice.name) - setVoice(voice) return true } - private fun TextToSpeech.voiceForName(name: String) = - voices.firstOrNull { it.name == name } + private fun TextToSpeech.setupVoice(): Boolean { + return setVoice(androidVoice) == SUCCESS + } private class UtteranceListener( private val listener: TtsEngine.Listener, diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt index 97900daca2..6297c8fb1c 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt @@ -20,10 +20,10 @@ public interface AudioEngine { public fun onItemChanged(engine: AudioEngine, index: Int) - public fun onStateChanged(engine: AudioEngine, state: State) + public fun onStateChanged(engine: AudioEngine, state: PlaybackState) } - public enum class State { + public enum class PlaybackState { Ready, Starved, Ended, @@ -34,9 +34,15 @@ public interface AudioEngine { val interval: TimeInterval?, ) + public val playlist: List + public var playWhenReady: Boolean - public val playlist: List + public var pitch: Double + + public var speed: Double + + public val playbackState: PlaybackState public fun seekTo(index: Int) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt index 8b2879a8f3..51c45c3a7c 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt @@ -9,6 +9,7 @@ package org.readium.navigator.media.readaloud import kotlin.properties.Delegates +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Error @@ -31,16 +32,16 @@ internal class ReadAloudDataLoader( val nextNode = node.next() ?: return if (nextNode !in preloadedRefs) { - loadSegmentForNode(nextNode)?.player?.prepare() + // loadSegmentForNode(nextNode, true) } } - fun getItemRef(node: ReadAloudNode): ItemRef? { - loadSegmentForNode(node) + fun getItemRef(node: ReadAloudNode, prepare: Boolean = true): ItemRef? { + loadSegmentForNode(node, prepare) return preloadedRefs[node] } - private fun loadSegmentForNode(node: ReadAloudNode): ReadAloudSegment? { + private fun loadSegmentForNode(node: ReadAloudNode, prepare: Boolean): ReadAloudSegment? { if (node in preloadedRefs) { return null } @@ -48,6 +49,10 @@ internal class ReadAloudDataLoader( val segment = segmentFactory.createSegmentFromNode(node) ?: return null // Ended + if (prepare) { + segment.player.prepare() + } + val refs = computeRefsForSegment(segment) preloadedRefs.putAll(refs) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt index aa85026a82..fc964a5c9f 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt @@ -8,6 +8,7 @@ package org.readium.navigator.media.readaloud +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationAudioRef diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index 0ea92dc0c3..b055829fff 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -18,6 +18,7 @@ import kotlinx.coroutines.withContext import org.readium.navigator.common.CssSelector import org.readium.navigator.common.TextQuote import org.readium.navigator.media.common.MediaNavigator +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.extensions.addPrefix @@ -101,7 +102,7 @@ public class ReadAloudNavigator private constructor } } - override fun onStateChanged(engine: AudioEngine, state: AudioEngine.State) { + override fun onStateChanged(engine: AudioEngine, state: AudioEngine.PlaybackState) { with(stateMachine) { if (engine == stateMutable.value.segment.player.player) { stateMutable.value = stateMutable.value.onAudioEngineStateChanged(state) @@ -125,7 +126,7 @@ public class ReadAloudNavigator private constructor override fun onStateChanged( player: TtsPlayer, - state: TtsPlayer.State, + state: TtsPlayer.PlaybackState, ) { with(stateMachine) { if (player == stateMutable.value.segment.player.player) { @@ -135,50 +136,66 @@ public class ReadAloudNavigator private constructor } } + public var settings: ReadAloudSettings by Delegates.observable(initialSettings) { + property, oldValue, newValue -> + with(stateMachine) { + stateMutable.value = stateMutable.value.updateSettings(oldValue, newValue) + } + } + private val segmentFactory = ReadAloudSegmentFactory( audioEngineFactory = { playlist: List -> audioEngineFactory(playlist, AudioEngineListener()) }, ttsPlayerFactory = { language: Language?, utterances: List -> - val engineFactory = { engineListener: TtsEngine.Listener -> - // FIXME support engine provider with no voice - /*val preferredVoiceWithRegion = - settings.voices[language] - ?.let { voiceForName(it.value) } - - val preferredVoiceWithoutRegion = - settings.voices[language.removeRegion()] - ?.let { voiceForName(it.value) } - - - - val voice = preferredVoiceWithRegion - ?: preferredVoiceWithoutRegion - ?: run { - voiceSelector - .voice(language, voices) - ?.let { voiceForName(it.id.value) } - }*/ - - val voice = ttsEngineProvider.voices - .firstOrNull { voice -> - language in voice.languages - } ?: ttsEngineProvider.voices - .firstOrNull { voice -> - language?.removeRegion() in voice.languages.map { it.removeRegion() } + val language = + settings.language + .takeIf { settings.overrideContentLanguage } + ?: language + ?: settings.language + + val preferredVoiceWithRegion = + settings.voices[language] + ?.let { voiceId -> ttsEngineProvider.voices.firstOrNull { it.id == voiceId } } + + val preferredVoiceWithoutRegion = + language + .let { settings.voices[it.removeRegion()] } + ?.let { voiceId -> ttsEngineProvider.voices.firstOrNull { it.id == voiceId } } + + val fallbackVoiceWithRegion = ttsEngineProvider.voices + .firstOrNull { language in it.languages } + + val fallbackVoiceWithoutRegion = ttsEngineProvider.voices + .firstOrNull { voice -> language.removeRegion() in voice.languages.map { it.removeRegion() } } + + val voice = preferredVoiceWithRegion + ?: preferredVoiceWithoutRegion + ?: fallbackVoiceWithRegion + ?: fallbackVoiceWithoutRegion + ?: ttsEngineProvider.voices.firstOrNull() + + val engineFactory = voice?.let { voice -> + { engineListener: TtsEngine.Listener -> + val engine = ttsEngineProvider.createEngine( + voice = voice, + utterances = utterances, + listener = engineListener + ) + + when (engine) { + is PausableTtsEngine -> engine + else -> PauseDecorator(engine) } - ?: ttsEngineProvider.voices.first() - val engine = ttsEngineProvider.createEngine( - voice = checkNotNull(voice), - utterances = utterances, - listener = engineListener + } + } + + engineFactory?.let { + TtsPlayer( + engineFactory = engineFactory, + listener = TtsPlayerListener() ) - engine as? PausableTtsEngine ?: PauseDecorator(engine) } - TtsPlayer( - engineFactory = engineFactory, - listener = TtsPlayerListener() - ) } ) @@ -196,12 +213,12 @@ public class ReadAloudNavigator private constructor } private val stateMutable: MutableStateFlow = run { - val itemRef = dataLoader.getItemRef(initialNode)!! + val itemRef = dataLoader.getItemRef(initialNode, prepare = false)!! MutableStateFlow( - stateMachine.play( + stateMachine.init( segment = itemRef.segment, - index = itemRef.nodeIndex ?: 0, + itemIndex = itemRef.nodeIndex ?: 0, playWhenReady = false, settings = initialSettings ) @@ -211,11 +228,6 @@ public class ReadAloudNavigator private constructor private val coroutineScope: CoroutineScope = MainScope() - init { - stateMutable.value.segment.player.prepare() - play() - } - public val playback: StateFlow = stateMutable.mapStateIn(coroutineScope) { state -> Playback( @@ -284,6 +296,11 @@ public class ReadAloudNavigator private constructor State.Failure(Error.EngineError(error)) } + init { + stateMutable.value.segment.player.prepare() + play() + } + public fun play() { with(stateMachine) { stateMutable.value = stateMutable.value.resume() @@ -376,11 +393,4 @@ public class ReadAloudNavigator private constructor ) goTo(location) } - - public var settings: ReadAloudSettings by Delegates.observable(initialSettings) { - property, oldValue, newValue -> - with(stateMachine) { - stateMutable.value = stateMutable.value.updateSettings(oldValue, newValue) - } - } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index 0840c53068..4ac3e7ac34 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -7,8 +7,13 @@ package org.readium.navigator.media.readaloud import android.app.Application +import org.readium.navigator.media.readaloud.preferences.ReadAloudDefaults +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferencesEditor +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.publication.Metadata import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.epub.MediaOverlaysService import org.readium.r2.shared.publication.services.GuidedNavigationService @@ -20,6 +25,7 @@ import org.readium.r2.shared.util.getOrElse @ExperimentalReadiumApi public class ReadAloudNavigatorFactory private constructor( + private val publicationMetadata: Metadata, private val guidedNavigationService: GuidedNavigationService, private val resources: List, private val audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, @@ -65,6 +71,7 @@ public class ReadAloudNavigatorFactory private cons } return ReadAloudNavigatorFactory( + publicationMetadata = publication.metadata, guidedNavigationService = guidedNavService, resources = resources, audioEngineFactory = audioEngineFactory, @@ -125,4 +132,14 @@ public class ReadAloudNavigatorFactory private cons return Try.success(navigator) } + + public fun createPreferencesEditor( + initialPreferences: ReadAloudPreferences, + defaults: ReadAloudDefaults = ReadAloudDefaults(), + ): ReadAloudPreferencesEditor = + ReadAloudPreferencesEditor( + initialPreferences, + publicationMetadata, + defaults, + ) } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt index 50b85bf334..7206367b28 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt @@ -43,19 +43,17 @@ internal data class TtsSegment( ) : ReadAloudSegment internal class ReadAloudSegmentFactory( - private val audioEngineFactory: (List) -> AudioEngine, - private val ttsPlayerFactory: (Language?, List) -> TtsPlayer, + private val audioEngineFactory: (List) -> AudioEngine?, + private val ttsPlayerFactory: (Language?, List) -> TtsPlayer?, ) { fun createSegmentFromNode(node: ReadAloudNode): ReadAloudSegment? = createAudioSegmentFromNode(node) - .takeUnless { it.items.isEmpty() } ?: createTtsSegmentFromNode(node) - .takeUnless { it.items.isEmpty() } private fun createAudioSegmentFromNode( firstNode: ReadAloudNode, - ): AudioSegment { + ): AudioSegment? { var nextNode: ReadAloudNode? = firstNode val audioItems = mutableListOf() val textRefs = mutableListOf() @@ -76,7 +74,13 @@ internal class ReadAloudSegmentFactory( nextNode = nextNode.next() } + if (audioItems.isEmpty()) { + return null + } + val audioEngine = audioEngineFactory(audioItems) + ?: return null + audioEngine.playWhenReady = false return AudioSegment( @@ -90,7 +94,7 @@ internal class ReadAloudSegmentFactory( private fun createTtsSegmentFromNode( firstNode: ReadAloudNode, - ): TtsSegment { + ): TtsSegment? { var nextNode: ReadAloudNode? = firstNode val segmentLanguage = firstNode.text?.language val textItems = mutableListOf() @@ -114,9 +118,14 @@ internal class ReadAloudSegmentFactory( nextNode = nextNode.next() } + if (textItems.isEmpty()) { + return null + } + val utterances = textItems.map { it.plain!! } val ttsPlayer = ttsPlayerFactory(segmentLanguage, utterances) + ?: return null return TtsSegment( player = TtsSegmentPlayer(ttsPlayer), diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index e64340b47a..1e4fd66992 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -8,6 +8,7 @@ package org.readium.navigator.media.readaloud +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Error @@ -36,36 +37,27 @@ internal class ReadAloudStateMachine( val settings: ReadAloudSettings, ) - fun play( + fun init( segment: ReadAloudSegment, - index: Int, + itemIndex: Int, playWhenReady: Boolean, settings: ReadAloudSettings, ): State { - segment.player.seekTo(index) - + segment.player.seekTo(itemIndex) segment.player.playWhenReady = playWhenReady + segment.player.speed = settings.speed + segment.player.pitch = settings.pitch return State( - playbackState = PlaybackState.Starved, + playbackState = segment.player.playbackState.toStateMachinePlaybackState(), playWhenReady = playWhenReady, - node = segment.nodes[index], - index = index, + node = segment.nodes[itemIndex], + index = itemIndex, segment = segment, settings = settings ) } - fun State.pause(): State { - segment.player.playWhenReady = false - return copy(playWhenReady = false) - } - - fun State.resume(): State { - segment.player.playWhenReady = true - return copy(playWhenReady = true) - } - fun State.jump(node: ReadAloudNode): State { val firstContentNode = with(navigationHelper) { node.firstContentNode() } ?: return copy(playbackState = PlaybackState.Ended) @@ -73,9 +65,33 @@ internal class ReadAloudStateMachine( val itemRef = dataLoader.getItemRef(firstContentNode) ?: return copy(playbackState = PlaybackState.Ended) - val index = itemRef.nodeIndex!! + val currentSegment = segment - return play(itemRef.segment, index, playWhenReady, settings) + if (currentSegment != itemRef.segment) { + currentSegment.player.release() + } + + val index = itemRef.nodeIndex!! // This is a content node + return init(itemRef.segment, index, playWhenReady, settings) + } + + fun State.restart(): State { + segment.player.release() + + val itemRef = dataLoader.getItemRef(node) + ?: return copy(playbackState = PlaybackState.Ended) + + return init(itemRef.segment, index, playWhenReady, settings) + } + + fun State.pause(): State { + segment.player.playWhenReady = false + return copy(playWhenReady = false) + } + + fun State.resume(): State { + segment.player.playWhenReady = true + return copy(playWhenReady = true) } fun State.updateSettings( @@ -84,14 +100,26 @@ internal class ReadAloudStateMachine( ): State { navigationHelper.settings = newSettings dataLoader.settings = newSettings - return copy(settings = newSettings) + + segment.player.pitch = settings.pitch + segment.player.speed = settings.speed + + return copy(settings = newSettings).restart() } - fun State.onAudioEngineStateChanged(audioEngineState: AudioEngine.State): State { + fun State.onAudioEngineStateChanged(audioEngineState: AudioEngine.PlaybackState): State { return when (audioEngineState) { - AudioEngine.State.Ready -> copy(playbackState = PlaybackState.Ready) - AudioEngine.State.Starved -> copy(playbackState = PlaybackState.Starved) - AudioEngine.State.Ended -> onSegmentPlaybackEnded() + AudioEngine.PlaybackState.Ready -> copy(playbackState = PlaybackState.Ready) + AudioEngine.PlaybackState.Starved -> copy(playbackState = PlaybackState.Starved) + AudioEngine.PlaybackState.Ended -> onSegmentPlaybackEnded() + } + } + + fun State.onTtsPlayerStateChanged(state: TtsPlayer.PlaybackState): State { + return when (state) { + TtsPlayer.PlaybackState.Ready -> copy(playbackState = PlaybackState.Ready) + TtsPlayer.PlaybackState.Starved -> copy(playbackState = PlaybackState.Starved) + TtsPlayer.PlaybackState.Ended -> onSegmentPlaybackEnded() } } @@ -100,35 +128,36 @@ internal class ReadAloudStateMachine( return copy(node = segment.nodes[item], index = item) } - fun State.onTtsPlayerStateChanged(state: TtsPlayer.State): State { - return when (state) { - TtsPlayer.State.Ready -> copy(playbackState = PlaybackState.Ready) - TtsPlayer.State.Starved -> copy(playbackState = PlaybackState.Starved) - TtsPlayer.State.Ended -> onSegmentPlaybackEnded() - } - } - fun State.onTtsPlayerItemChanged(item: Int): State { dataLoader.onPlaybackProgressed(segment.nodes[item]) return copy(node = segment.nodes[item], index = item) } private fun State.onSegmentPlaybackEnded(): State { - segment.player.release() - val nextNode = node.next() ?: return copy(playbackState = PlaybackState.Ended) val newSegment = dataLoader.getItemRef(nextNode)?.segment ?: return copy(playbackState = PlaybackState.Ended) + segment.player.release() + newSegment.player.playWhenReady = playWhenReady + newSegment.player.speed = settings.speed + newSegment.player.pitch = settings.pitch return copy( segment = newSegment, - playbackState = PlaybackState.Starved, + playbackState = segment.player.playbackState.toStateMachinePlaybackState(), node = newSegment.nodes[0], index = 0 ) } } + +private fun SegmentPlayer.PlaybackState.toStateMachinePlaybackState() = + when (this) { + SegmentPlayer.PlaybackState.Ready -> ReadAloudStateMachine.PlaybackState.Ready + SegmentPlayer.PlaybackState.Starved -> ReadAloudStateMachine.PlaybackState.Starved + SegmentPlayer.PlaybackState.Ended -> ReadAloudStateMachine.PlaybackState.Ended + } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt index ff3f8fbee2..3f1f5c57d2 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt @@ -13,10 +13,22 @@ import org.readium.r2.shared.util.Error internal sealed interface SegmentPlayer { + enum class PlaybackState { + Ready, + Starved, + Ended, + } + val player: Any var playWhenReady: Boolean + var pitch: Double + + var speed: Double + + val playbackState: PlaybackState + fun prepare() fun seekTo(index: Int) @@ -33,6 +45,24 @@ internal class TtsSegmentPlayer( set(value) { player.playWhenReady = value } + override var pitch: Double + get() = player.pitch + set(value) { + player.pitch = value + } + + override var speed: Double + get() = player.speed + set(value) { + player.speed = value + } + + override val playbackState: SegmentPlayer.PlaybackState + get() = when (player.playbackState) { + TtsPlayer.PlaybackState.Ready -> SegmentPlayer.PlaybackState.Ready + TtsPlayer.PlaybackState.Starved -> SegmentPlayer.PlaybackState.Starved + TtsPlayer.PlaybackState.Ended -> SegmentPlayer.PlaybackState.Ended + } override fun prepare() { player.prepare() @@ -57,6 +87,25 @@ internal class AudioSegmentPlayer( player.playWhenReady = value } + override var pitch: Double + get() = player.pitch + set(value) { + player.pitch = value + } + + override var speed: Double + get() = player.speed + set(value) { + player.speed = value + } + + override val playbackState: SegmentPlayer.PlaybackState + get() = when (player.playbackState) { + AudioEngine.PlaybackState.Ready -> SegmentPlayer.PlaybackState.Ready + AudioEngine.PlaybackState.Starved -> SegmentPlayer.PlaybackState.Starved + AudioEngine.PlaybackState.Ended -> SegmentPlayer.PlaybackState.Ended + } + override fun prepare() { } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt index a8ce681e6f..5a5d27abc5 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt @@ -39,6 +39,10 @@ public interface TtsEngine { public val utterances: List + public var pitch: Double + + public var speed: Double + public fun prepare() public fun speak(utteranceIndex: Int) @@ -51,6 +55,7 @@ public interface TtsEngine { @ExperimentalReadiumApi public interface TtsVoice { + @kotlinx.serialization.Serializable @JvmInline public value class Id(public val value: String) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt index 412b51de92..b4c1c31ecf 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt @@ -9,6 +9,8 @@ package org.readium.navigator.media.readaloud import kotlin.properties.Delegates +import org.readium.navigator.media.readaloud.StateMachine.PlaybackState +import org.readium.navigator.media.readaloud.StateMachine.State import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Error @@ -21,10 +23,10 @@ internal class TtsPlayer( fun onItemChanged(player: TtsPlayer, index: Int) - fun onStateChanged(player: TtsPlayer, state: State) + fun onStateChanged(player: TtsPlayer, state: TtsPlayer.PlaybackState) } - internal enum class State { + internal enum class PlaybackState { Ready, Starved, Ended, @@ -58,19 +60,20 @@ internal class TtsPlayer( private val stateMachine = StateMachine(ttsEngine) private var state by Delegates.observable( - StateMachine.start(playWhenReady = false) + State( + playWhenReady = false, + playbackState = StateMachine.PlaybackState.Starved, + indexToPlay = 0, + lastSubmittedIndex = null + ) ) { property, oldValue, newValue -> if (oldValue.playbackState != newValue.playbackState) { - val state = when (newValue.playbackState) { - StateMachine.PlaybackState.Ended -> State.Ended - StateMachine.PlaybackState.Ready -> State.Ready - StateMachine.PlaybackState.Starved -> State.Starved - } - listener.onStateChanged(this, state) + val playerState = newValue.playbackState.toPlayerPlaybackState() + listener.onStateChanged(this, playerState) } - if (oldValue.index != newValue.index) { - listener.onItemChanged(this, newValue.index) + if (oldValue.indexToPlay != newValue.indexToPlay) { + listener.onItemChanged(this, newValue.indexToPlay) } } @@ -82,6 +85,21 @@ internal class TtsPlayer( } } + var pitch: Double + get() = ttsEngine.pitch + set(value) { + ttsEngine.pitch = value + } + + var speed: Double + get() = ttsEngine.speed + set(value) { + ttsEngine.speed = value + } + + val playbackState: TtsPlayer.PlaybackState get() = + state.playbackState.toPlayerPlaybackState() + fun prepare() { ttsEngine.prepare() } @@ -97,6 +115,13 @@ internal class TtsPlayer( } } +private fun StateMachine.PlaybackState.toPlayerPlaybackState() = + when (this) { + PlaybackState.Ended -> TtsPlayer.PlaybackState.Ended + PlaybackState.Ready -> TtsPlayer.PlaybackState.Ready + PlaybackState.Starved -> TtsPlayer.PlaybackState.Starved + } + private class StateMachine( private val engine: PausableTtsEngine, ) { @@ -112,42 +137,41 @@ private class StateMachine( data class State( val playbackState: PlaybackState, val playWhenReady: Boolean, - val index: Int, + val indexToPlay: Int, + val lastSubmittedIndex: Int?, ) - companion object { - - fun start(playWhenReady: Boolean): State { - return State( - playWhenReady = playWhenReady, - playbackState = PlaybackState.Starved, - index = 0 - ) - } - } - fun State.pause(): State { engine.pause() return copy(playWhenReady = false) } fun State.resume(): State { - engine.resume() - return copy(playWhenReady = true) + if (lastSubmittedIndex == indexToPlay) { + engine.resume() + } else { + engine.stop() + engine.speak(indexToPlay) + } + + return copy(playWhenReady = true, lastSubmittedIndex = indexToPlay) } fun State.seekTo(index: Int): State { if (playWhenReady) { engine.stop() engine.speak(index) + return copy(indexToPlay = index, lastSubmittedIndex = index) + } else { + return copy(indexToPlay = index) } - return copy(index = index) } fun State.onUtteranceCompleted(): State { - if (index < engine.utterances.size - 1) { - engine.speak(index + 1) - return copy(index = index + 1) + if (indexToPlay < engine.utterances.size - 1) { + val newIndexToPlay = indexToPlay + 1 + engine.speak(newIndexToPlay) + return copy(indexToPlay = newIndexToPlay, lastSubmittedIndex = newIndexToPlay) } else { return copy(playbackState = PlaybackState.Ended) } @@ -155,8 +179,8 @@ private class StateMachine( fun State.onEngineReady(): State { if (playWhenReady) { - engine.speak(index) + engine.speak(indexToPlay) } - return copy(playbackState = PlaybackState.Ready) + return copy(playbackState = PlaybackState.Ready, lastSubmittedIndex = indexToPlay) } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt similarity index 79% rename from readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt rename to readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt index 4034f39c5c..995088adbf 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSettings.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt @@ -4,8 +4,10 @@ * available in the top-level LICENSE file of the project. */ -package org.readium.navigator.media.readaloud +package org.readium.navigator.media.readaloud.preferences +import org.readium.navigator.common.Settings +import org.readium.navigator.media.readaloud.TtsVoice import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationRole import org.readium.r2.shared.util.Language @@ -14,10 +16,9 @@ import org.readium.r2.shared.util.Language public data class ReadAloudSettings( val language: Language, val overrideContentLanguage: Boolean, - val preferRecordedVoices: Boolean, val pitch: Double, val speed: Double, val voices: Map, val escapableRoles: Set, val skippableRoles: Set, -) +) : Settings diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt index 7c31c6cb9a..eac467dedb 100644 --- a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt @@ -6,9 +6,12 @@ package org.readium.r2.shared.guided +import kotlinx.serialization.Serializable + /** * A role usable in a guided navigation object. */ +@Serializable @JvmInline public value class GuidedNavigationRole(public val value: String) { @@ -78,14 +81,5 @@ public value class GuidedNavigationRole(public val value: String) { public val LOI: GuidedNavigationRole = GuidedNavigationRole("loi") public val LOT: GuidedNavigationRole = GuidedNavigationRole("lot") public val LOV: GuidedNavigationRole = GuidedNavigationRole("lov") - - public val SKIPPABLE_ROLES: List = - listOf( - ASIDE, BIBLIOGRAPHY, ENDNOTES, FOOTNOTE, NOTEREF, PULLQUOTE, - LANDMARKS, LOA, LOI, LOT, LOV, PAGEBREAK, TOC - ) - - public val ESCAPABLE_ROLES: List = - listOf(ASIDE, FIGURE, LIST, LIST_ITEM, TABLE, ROW, CELL) } } From 6059339b26aee3e776d1f21b32ab9dead8c4d6f5 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Thu, 11 Sep 2025 15:12:46 +0200 Subject: [PATCH 12/13] Merge audio and tts engines --- .../readium/demo/navigator/DemoViewModel.kt | 24 +- .../preferences/ReadAloudPreferencesEditor.kt | 86 +++++++ .../navigator/preferences/UserPreferences.kt | 15 +- .../demo/navigator/reader/ReaderOpener.kt | 19 +- .../demo/navigator/reader/ReaderState.kt | 4 +- .../demo/navigator/reader/RenditionTopBar.kt | 52 +++++ .../navigator/reader/SelectNavigatorMenu.kt | 6 +- .../exoplayer/readaloud/ExoPlayerEngine.kt | 188 ++++++++++----- .../readaloud/ExoPlayerEngineProvider.kt | 29 ++- .../navigator/media/readaloud/AudioEngine.kt | 60 ----- .../media/readaloud/PlaybackEngine.kt | 142 ++++++++++++ .../media/readaloud/ReadAloudDataLoader.kt | 10 +- .../media/readaloud/ReadAloudNavigator.kt | 145 ++++++------ .../readaloud/ReadAloudNavigatorFactory.kt | 10 +- .../media/readaloud/ReadAloudSegment.kt | 34 ++- .../media/readaloud/ReadAloudStateMachine.kt | 101 ++++---- .../media/readaloud/SegmentPlayer.kt | 119 ---------- ...AndroidTtsEngine.kt => SystemTtsEngine.kt} | 215 ++++++++++-------- .../navigator/media/readaloud/TtsEngine.kt | 111 --------- .../navigator/media/readaloud/TtsPlayer.kt | 186 --------------- .../preferences/ReadAloudDefaults.kt | 28 +++ .../preferences/ReadAloudPreferences.kt | 54 +++++ .../preferences/ReadAloudPreferencesEditor.kt | 126 ++++++++++ .../preferences/ReadAloudSettings.kt | 1 + .../preferences/ReadAloudSettingsResolver.kt | 66 ++++++ 25 files changed, 999 insertions(+), 832 deletions(-) create mode 100644 demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt create mode 100644 demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt rename readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/{AndroidTtsEngine.kt => SystemTtsEngine.kt} (72%) delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt delete mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt create mode 100644 readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index c8e6743bac..c868b972f0 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -23,8 +23,8 @@ import org.readium.demo.navigator.reader.SelectNavigatorItem import org.readium.demo.navigator.reader.SelectNavigatorViewModel import org.readium.demo.navigator.reader.fixedConfig import org.readium.demo.navigator.reader.reflowableConfig -import org.readium.navigator.media.readaloud.AndroidTtsEngineProvider import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.media.readaloud.SystemTtsEngineProvider import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.r2.shared.ExperimentalReadiumApi @@ -125,20 +125,14 @@ class DemoViewModel( )?.let { SelectNavigatorItem.FixedWeb(it) } val ttsEngineProvider = - AndroidTtsEngineProvider(application) - ?.takeIf { it.voices.isNotEmpty() } - - val readAloudFactory = ttsEngineProvider - ?.let { - ReadAloudNavigatorFactory( - application = application, - publication = publication, - audioEngineProvider = audioEngineProvider, - ttsEngineProvider = ttsEngineProvider - ) - }?.let { - SelectNavigatorItem.ReadAloud(factory = it, ttsEngineProvider = ttsEngineProvider) - } + SystemTtsEngineProvider(application) + + val readAloudFactory = ReadAloudNavigatorFactory( + application = application, + publication = publication, + audioEngineProvider = audioEngineProvider, + ttsEngineProvider = ttsEngineProvider + )?.let { SelectNavigatorItem.ReadAloud(it) } val factories = listOfNotNull( reflowableFactory, diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt new file mode 100644 index 0000000000..c7ab367e07 --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.demo.navigator.preferences + +import kotlinx.coroutines.flow.StateFlow +import org.readium.navigator.common.PreferencesEditor +import org.readium.navigator.media.readaloud.SystemTtsEngine +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.navigator.preferences.EnumPreference +import org.readium.r2.navigator.preferences.Preference +import org.readium.r2.navigator.preferences.RangePreference +import org.readium.r2.navigator.preferences.map +import org.readium.r2.navigator.preferences.withSupportedValues +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@OptIn(ExperimentalReadiumApi::class) +class ReadAloudPreferencesEditor( + private val editor: org.readium.navigator.media.readaloud.preferences.ReadAloudPreferencesEditor, + private val availableVoices: Set, +) : PreferencesEditor { + + override val preferences: ReadAloudPreferences + get() = editor.preferences + + val preferencesState: StateFlow = + editor.preferencesState + + override val settings: ReadAloudSettings + get() = editor.settings + + override fun clear() { + editor.clear() + } + + val language: Preference = + editor.language + + /** + * [ReadAloudPreferencesEditor] supports choosing voices for any language or region. + * For this test app, we've chosen to present to the user only the voice for the + * TTS default language and to ignore regions. + */ + val voice: EnumPreference = run { + val currentLanguage = language.effectiveValue?.removeRegion() + + editor.voices.map( + from = { voiceIds -> + currentLanguage + ?.let { voiceIds[it] } + ?.let { voiceId -> availableVoices.firstOrNull { it.id == voiceId } } + }, + to = { voice -> + currentLanguage + ?.let { editor.voices.value.orEmpty().update(it, voice?.id) } + ?: editor.voices.value.orEmpty() + } + ).withSupportedValues( + availableVoices + .filter { voice -> currentLanguage in voice.languages.map { it.removeRegion() } } + ) + } + val pitch: RangePreference = + editor.pitch + + val speed: RangePreference = + editor.speed + + val readContinuously: Preference = + editor.readContinuously + + private fun Map.update(key: K, value: V?): Map = + buildMap { + putAll(this@update) + if (value == null) { + remove(key) + } else { + put(key, value) + } + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt index 3aa8788198..e332e0dedc 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt @@ -28,7 +28,7 @@ import org.readium.demo.navigator.reader.LITERATA import org.readium.navigator.common.Preferences import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings -import org.readium.navigator.media.readaloud.AndroidTtsEngine +import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.navigator.web.fixedlayout.preferences.FixedWebPreferencesEditor import org.readium.navigator.web.reflowable.preferences.ReflowableWebPreferencesEditor import org.readium.r2.navigator.preferences.Axis @@ -129,7 +129,8 @@ fun

, S : Settings, E : PreferencesEditor> UserPreferen language = editor.language, voice = editor.voice, speed = editor.speed, - pitch = editor.pitch + pitch = editor.pitch, + readContinuously = editor.readContinuously ) } } @@ -499,9 +500,10 @@ private fun ReflowableUserPreferences( @Composable private fun MediaUserPreferences( language: Preference? = null, - voice: EnumPreference? = null, + voice: EnumPreference? = null, speed: RangePreference? = null, pitch: RangePreference? = null, + readContinuously: Preference? = null, ) { Column { if (speed != null) { @@ -530,6 +532,13 @@ private fun MediaUserPreferences( formatValue = { it?.name ?: "Default" }, ) } + + if (readContinuously != null) { + SwitchItem( + title = "Read Continuously", + preference = readContinuously + ) + } } } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index 6d33cbe49c..e03c4b3aad 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -33,9 +33,8 @@ import org.readium.navigator.common.DecorationLocation import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings import org.readium.navigator.common.SettingsController -import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory -import org.readium.navigator.media.readaloud.TtsEngineProvider +import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences import org.readium.navigator.web.fixedlayout.FixedWebGoLocation import org.readium.navigator.web.fixedlayout.FixedWebLocation @@ -89,7 +88,6 @@ class ReaderOpener( url, publication, selectedNavigator.factory, - selectedNavigator.ttsEngineProvider, initialLocator ) }.getOrElse { error -> @@ -207,8 +205,7 @@ class ReaderOpener( private suspend fun createReadAloudReader( url: AbsoluteUrl, publication: Publication, - navigatorFactory: ReadAloudNavigatorFactory, - ttsEngineProvider: TtsEngineProvider, + navigatorFactory: ReadAloudNavigatorFactory, initialLocator: Locator?, ): Try { val coroutineScope = MainScope() @@ -217,16 +214,20 @@ class ReaderOpener( val preferencesManager = PreferencesManager(initialPreferences) + val initialSettings = navigatorFactory + .createPreferencesEditor(preferencesManager.preferences.value) + .settings + + val navigator = navigatorFactory.createNavigator(initialSettings) + .getOrElse { return Try.failure(it) } + val preferencesEditor = preferencesManager.preferences.mapStateIn(coroutineScope) { ReadAloudPreferencesEditor( editor = navigatorFactory.createPreferencesEditor(it), - availableVoices = ttsEngineProvider.voices + availableVoices = navigator.voices ) } - val navigator = navigatorFactory.createNavigator(preferencesEditor.value.settings) - .getOrElse { return Try.failure(it) } - preferencesEditor .flatMapLatest { it.preferencesState } .onEach { diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt index 631a8c5aed..65802d49ba 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt @@ -20,8 +20,8 @@ import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.RenditionState import org.readium.navigator.common.SelectionController import org.readium.navigator.common.SelectionLocation -import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigator +import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl @@ -52,7 +52,7 @@ data class ReadAloudReaderState( val url: AbsoluteUrl, val coroutineScope: CoroutineScope, val publication: Publication, - val navigator: ReadAloudNavigator, + val navigator: ReadAloudNavigator, val preferencesEditor: StateFlow, ) : ReaderState { diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt new file mode 100644 index 0000000000..772394888e --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt @@ -0,0 +1,52 @@ +@file:OptIn(ExperimentalMaterial3Api::class) + +package org.readium.demo.navigator.reader + +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.TopAppBar +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.painterResource +import org.readium.demo.navigator.R + +@Composable +fun RenditionTopBar( + modifier: Modifier, + visible: Boolean, + onPreferencesActivated: () -> Unit, + onOutlineActivated: () -> Unit, +) { + AnimatedVisibility( + modifier = modifier, + visible = visible, + enter = fadeIn(), + exit = fadeOut() + ) { + TopAppBar( + title = { }, + actions = { + IconButton( + onClick = onPreferencesActivated + ) { + Icon( + painterResource(R.drawable.ic_preferences_24), + contentDescription = "Preferences", + ) + } + IconButton( + onClick = onOutlineActivated + ) { + Icon( + painterResource(R.drawable.ic_outline_24), + contentDescription = "Outline" + ) + } + } + ) + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt index 76469d121f..d4b815ad0f 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt @@ -13,9 +13,8 @@ import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.ui.window.PopupProperties -import org.readium.navigator.media.readaloud.AndroidTtsEngine import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory -import org.readium.navigator.media.readaloud.TtsEngineProvider +import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.r2.shared.ExperimentalReadiumApi @@ -50,8 +49,7 @@ sealed class SelectNavigatorItem( ) : SelectNavigatorItem("Fixed Web Rendition") data class ReadAloud( - override val factory: ReadAloudNavigatorFactory, - val ttsEngineProvider: TtsEngineProvider, + override val factory: ReadAloudNavigatorFactory, ) : SelectNavigatorItem("Read Aloud Navigator") } diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index c42fe80ad8..d78bd402f1 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -17,11 +17,10 @@ import androidx.media3.datasource.DataSource import androidx.media3.exoplayer.ExoPlaybackException import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.source.DefaultMediaSourceFactory -import kotlinx.coroutines.CoroutineScope +import kotlin.properties.Delegates import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.MainScope -import kotlinx.coroutines.cancel -import org.readium.navigator.media.readaloud.AudioEngine +import org.readium.navigator.media.readaloud.AudioChunk +import org.readium.navigator.media.readaloud.PlaybackEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.extensions.findInstance @@ -30,24 +29,23 @@ import org.readium.r2.shared.util.data.ReadError import org.readium.r2.shared.util.data.ReadException /** - * An [AudioEngine] based on Media3 ExoPlayer. + * A [PlaybackEngine] based on Media3 ExoPlayer. */ @ExperimentalReadiumApi @OptIn(ExperimentalCoroutinesApi::class) @androidx.annotation.OptIn(UnstableApi::class) public class ExoPlayerEngine private constructor( private val exoPlayer: ExoPlayer, - override val playlist: List, - private val listener: AudioEngine.Listener, -) : AudioEngine { + private val listener: PlaybackEngine.Listener, +) : PlaybackEngine { public companion object { public operator fun invoke( application: Application, dataSourceFactory: DataSource.Factory, - playlist: List, - listener: AudioEngine.Listener, + chunks: List, + listener: PlaybackEngine.Listener, ): ExoPlayerEngine { val exoPlayer = ExoPlayer.Builder(application) .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) @@ -62,8 +60,23 @@ public class ExoPlayerEngine private constructor( .build() exoPlayer.preloadConfiguration = ExoPlayer.PreloadConfiguration(10_000_000L) + exoPlayer.pauseAtEndOfMediaItems = true + + val mediaItems = chunks.map { item -> + val clippingConfig = MediaItem.ClippingConfiguration.Builder() + .apply { + item.interval?.start?.let { setStartPositionMs(it.inWholeMilliseconds) } + item.interval?.end?.let { setEndPositionMs(it.inWholeMilliseconds) } + }.build() + MediaItem.Builder() + .setUri(item.href.toString()) + .setClippingConfiguration(clippingConfig) + .build() + } + exoPlayer.setMediaItems(mediaItems) + exoPlayer.prepare() - return ExoPlayerEngine(exoPlayer, playlist, listener) + return ExoPlayerEngine(exoPlayer, listener) } } @@ -72,14 +85,31 @@ public class ExoPlayerEngine private constructor( override fun onPlaybackParametersChanged(playbackParameters: PlaybackParameters) { } - override fun onEvents(player: Player, events: Player.Events) { - if (events.contains(Player.EVENT_PLAYBACK_STATE_CHANGED)) { - val newState = player.getEnginePlaybackState() - newState.let { this@ExoPlayerEngine.listener.onStateChanged(this@ExoPlayerEngine, it) } + override fun onPlayWhenReadyChanged(playWhenReady: Boolean, reason: Int) { + if (reason == Player.PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM) { + state = State.Ended } + } - if (events.contains(Player.EVENT_MEDIA_ITEM_TRANSITION)) { - this@ExoPlayerEngine.listener.onItemChanged(this@ExoPlayerEngine, player.currentMediaItemIndex) + override fun onPlaybackStateChanged(playbackState: Int) { + state = when (val stateNow = state) { + State.Idle -> { + stateNow + } + State.Ended -> { + stateNow + } + is State.Running -> { + when (playbackState) { + Player.STATE_READY -> + stateNow.copy(playbackState = PlaybackEngine.PlaybackState.Playing) + Player.STATE_BUFFERING -> + stateNow.copy(playbackState = PlaybackEngine.PlaybackState.Starved) + Player.STATE_ENDED -> + State.Idle + else -> stateNow + } + } } } } @@ -96,38 +126,17 @@ public class ExoPlayerEngine private constructor( Error("An error occurred while trying to read publication content.", cause) } - private val coroutineScope: CoroutineScope = - MainScope() + private sealed interface State { - init { - exoPlayer.addListener(Listener()) - val mediaItems = playlist.map { item -> - val clippingConfig = MediaItem.ClippingConfiguration.Builder() - .apply { - item.interval?.start?.let { setStartPositionMs(it.inWholeMilliseconds) } - item.interval?.end?.let { setEndPositionMs(it.inWholeMilliseconds) } - }.build() - MediaItem.Builder() - .setUri(item.href.toString()) - .setClippingConfiguration(clippingConfig) - .build() - } - exoPlayer.setMediaItems(mediaItems) - exoPlayer.prepare() - } - - override fun seekTo(index: Int) { - exoPlayer.seekTo(index, 0L) - } + object Idle : State - override var playWhenReady: Boolean - get() = exoPlayer.playWhenReady - set(value) { - exoPlayer.playWhenReady = value - } + object Ended : State - override val playbackState: AudioEngine.PlaybackState - get() = exoPlayer.getEnginePlaybackState() + data class Running( + val playbackState: PlaybackEngine.PlaybackState, + val paused: Boolean, + ) : State + } override var pitch: Double get() = exoPlayer.playbackParameters.pitch.toDouble() @@ -143,8 +152,85 @@ public class ExoPlayerEngine private constructor( exoPlayer.playbackParameters.withSpeed(value.toFloat()) } + override var itemToPlay: Int by Delegates.observable(0) { property, oldValue, newValue -> + when (state) { + State.Ended -> { + if (newValue != exoPlayer.currentMediaItemIndex + 1) { + exoPlayer.seekTo(newValue, 0) + } + } + State.Idle -> { + if (newValue != exoPlayer.currentMediaItemIndex) { + exoPlayer.seekTo(newValue, 0) + } + } + is State.Running -> { + } + } + } + + private var state: State by Delegates.observable(State.Idle) { property, oldValue, newValue -> + if (newValue != oldValue) { + when { + newValue is State.Ended -> { + listener.onPlaybackCompleted() + } + newValue is State.Running && oldValue is State.Running && + newValue.playbackState != oldValue.playbackState -> { + listener.onPlaybackStateChanged(newValue.playbackState) + } + } + } + } + + init { + exoPlayer.addListener(Listener()) + } + + override fun start() { + val playbackState = when (exoPlayer.playbackState) { + Player.STATE_READY -> PlaybackEngine.PlaybackState.Playing + Player.STATE_BUFFERING -> PlaybackEngine.PlaybackState.Starved + else -> throw IllegalStateException("Unexpected ExoPlayer state ${exoPlayer.playbackState}") + } + + listener.onStartRequested(playbackState) + exoPlayer.playWhenReady = true + state = State.Running(playbackState = playbackState, paused = false) + } + + override fun stop() { + exoPlayer.playWhenReady = false + exoPlayer.seekTo(0) + state = State.Idle + listener.onStopRequested() + } + + override fun resume() { + state = when (val stateNow = state) { + State.Idle, State.Ended -> { + stateNow + } + is State.Running -> { + exoPlayer.playWhenReady = true + stateNow.copy(paused = false) + } + } + } + + override fun pause() { + state = when (val stateNow = state) { + State.Idle, State.Ended -> { + stateNow + } + is State.Running -> { + exoPlayer.playWhenReady = false + stateNow.copy(paused = true) + } + } + } + public override fun release() { - coroutineScope.cancel() exoPlayer.release() } @@ -163,12 +249,4 @@ public class ExoPlayerEngine private constructor( Error.Source(readError) } } - - private fun Player.getEnginePlaybackState(): AudioEngine.PlaybackState = - when (playbackState) { - Player.STATE_READY -> AudioEngine.PlaybackState.Ready - Player.STATE_BUFFERING, Player.STATE_IDLE -> AudioEngine.PlaybackState.Starved - Player.STATE_ENDED -> AudioEngine.PlaybackState.Ended - else -> throw IllegalStateException("Unexpected ExoPlayer state $playbackState") - } } diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt index 5b9f49cd31..f2be8e1721 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt @@ -4,28 +4,41 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(InternalReadiumApi::class) + package org.readium.adapter.exoplayer.readaloud import android.app.Application import org.readium.adapter.exoplayer.audio.ExoPlayerDataSource -import org.readium.navigator.media.readaloud.AudioEngine +import org.readium.navigator.media.readaloud.AudioChunk +import org.readium.navigator.media.readaloud.AudioEngineFactory import org.readium.navigator.media.readaloud.AudioEngineProvider +import org.readium.navigator.media.readaloud.PlaybackEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.publication.Publication -@OptIn(InternalReadiumApi::class) @ExperimentalReadiumApi public class ExoPlayerEngineProvider( private val application: Application, ) : AudioEngineProvider { - override fun createEngine( - publication: Publication, - playlist: List, - listener: AudioEngine.Listener, - ): ExoPlayerEngine { + override fun createEngineFactory(publication: Publication): AudioEngineFactory { val dataSourceFactory = ExoPlayerDataSource.Factory(publication) - return ExoPlayerEngine(application, dataSourceFactory, playlist, listener) + return ExoPlayerEngineFactory(application, dataSourceFactory) + } +} + +@ExperimentalReadiumApi +public class ExoPlayerEngineFactory internal constructor( + private val application: Application, + private val dataSourceFactory: ExoPlayerDataSource.Factory, +) : AudioEngineFactory { + + override fun createPlaybackEngine( + chunks: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine { + return ExoPlayerEngine(application, dataSourceFactory, chunks, listener) } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt deleted file mode 100644 index 6297c8fb1c..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AudioEngine.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(ExperimentalReadiumApi::class) - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Publication -import org.readium.r2.shared.util.TimeInterval -import org.readium.r2.shared.util.Url - -@ExperimentalReadiumApi -public interface AudioEngine { - - public interface Listener { - - public fun onItemChanged(engine: AudioEngine, index: Int) - - public fun onStateChanged(engine: AudioEngine, state: PlaybackState) - } - - public enum class PlaybackState { - Ready, - Starved, - Ended, - } - - public data class Item( - val href: Url, - val interval: TimeInterval?, - ) - - public val playlist: List - - public var playWhenReady: Boolean - - public var pitch: Double - - public var speed: Double - - public val playbackState: PlaybackState - - public fun seekTo(index: Int) - - public fun release() -} - -@ExperimentalReadiumApi -public interface AudioEngineProvider { - - public fun createEngine( - publication: Publication, - playlist: List, - listener: AudioEngine.Listener, - ): AudioEngine -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt new file mode 100644 index 0000000000..8da4c78790 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt @@ -0,0 +1,142 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.util.Error +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +@ExperimentalReadiumApi +public interface AudioEngineProvider { + + public fun createEngineFactory( + publication: Publication, + ): AudioEngineFactory +} + +@ExperimentalReadiumApi +public interface AudioEngineFactory { + + public fun createPlaybackEngine( + chunks: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine +} + +@ExperimentalReadiumApi +public data class AudioChunk( + val href: Url, + val interval: TimeInterval?, +) + +@ExperimentalReadiumApi +public interface TtsEngineProvider { + + public suspend fun createEngineFactory(): TtsEngineFactory +} + +@ExperimentalReadiumApi +public interface TtsEngineFactory { + + /** + * Sets of voices available with this [TtsEngineFactory]. + */ + public val voices: Set + + public fun createPlaybackEngine( + voice: V, + utterances: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine +} + +internal class NullPlaybackEngineFactory() : TtsEngineFactory { + + override val voices: Set = emptySet() + + override fun createPlaybackEngine( + voice: V, + utterances: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine { + throw IllegalArgumentException("Unknown voice.") + } +} + +@ExperimentalReadiumApi +public interface TtsVoice { + + @kotlinx.serialization.Serializable + @JvmInline + public value class Id(public val value: String) + + public val id: Id + + /** + * The languages supported by the voice. + */ + public val languages: Set +} + +@ExperimentalReadiumApi +public interface PlaybackEngine { + + public var pitch: Double + + public var speed: Double + + /** + * Sets the index of the item to play on the next call to [start]. + */ + public var itemToPlay: Int + + public enum class PlaybackState { + Playing, + Starved, + } + + public interface Listener { + + public fun onStartRequested(initialState: PlaybackState) + + public fun onStopRequested() + + public fun onPlaybackCompleted() + + public fun onPlaybackStateChanged(state: PlaybackState) + + public fun onRangeStarted(range: IntRange) + } + + /** + * Starts playing the [itemToPlay]-th item. + * + * The state will become either [PlaybackState.Playing] or [PlaybackState.Starved]. + */ + public fun start() + + /** + * Stops ongoing playback. + * + * Makes the state become [PlaybackState.Idle] + */ + public fun stop() + + public fun pause() + + public fun resume() + + /** + * Free all used resources. + */ + public fun release() +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt index 51c45c3a7c..2f378a3eed 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt @@ -36,12 +36,12 @@ internal class ReadAloudDataLoader( } } - fun getItemRef(node: ReadAloudNode, prepare: Boolean = true): ItemRef? { - loadSegmentForNode(node, prepare) + fun getItemRef(node: ReadAloudNode): ItemRef? { + loadSegmentForNode(node) return preloadedRefs[node] } - private fun loadSegmentForNode(node: ReadAloudNode, prepare: Boolean): ReadAloudSegment? { + private fun loadSegmentForNode(node: ReadAloudNode): ReadAloudSegment? { if (node in preloadedRefs) { return null } @@ -49,10 +49,6 @@ internal class ReadAloudDataLoader( val segment = segmentFactory.createSegmentFromNode(node) ?: return null // Ended - if (prepare) { - segment.player.prepare() - } - val refs = computeRefsForSegment(segment) preloadedRefs.putAll(refs) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index b055829fff..e4708a5756 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -8,6 +8,8 @@ package org.readium.navigator.media.readaloud +import android.os.Handler +import android.os.Looper import kotlin.properties.Delegates import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -33,8 +35,8 @@ import org.readium.r2.shared.util.data.ReadError public class ReadAloudNavigator private constructor( private val guidedNavigationTree: ReadAloudNode, private val resources: List, - audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, - ttsEngineProvider: TtsEngineProvider, + private val audioEngineFactory: AudioEngineFactory, + private val ttsEngineFactory: TtsEngineFactory, initialSettings: ReadAloudSettings, initialLocation: ReadAloudGoLocation?, ) { @@ -44,8 +46,8 @@ public class ReadAloudNavigator private constructor initialLocation: ReadAloudGoLocation?, initialSettings: ReadAloudSettings, publication: ReadAloudPublication, - audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, - ttsEngineProvider: TtsEngineProvider, + audioEngineFactory: AudioEngineFactory, + ttsEngineFactory: TtsEngineFactory, ): ReadAloudNavigator { val tree = withContext(Dispatchers.Default) { ReadAloudNode.fromGuidedNavigationObject(publication.guidedNavigationTree) @@ -55,7 +57,7 @@ public class ReadAloudNavigator private constructor guidedNavigationTree = tree, resources = publication.resources, audioEngineFactory = audioEngineFactory, - ttsEngineProvider = ttsEngineProvider, + ttsEngineFactory = ttsEngineFactory, initialSettings = initialSettings, initialLocation = initialLocation ) @@ -63,21 +65,21 @@ public class ReadAloudNavigator private constructor } public data class Playback( - val state: State, + val state: ReadAloudNavigator.PlaybackState, val playWhenReady: Boolean, val node: ReadAloudNode, val utteranceLocation: UtteranceLocation?, ) - public sealed interface State { + public sealed interface PlaybackState { - public data object Ready : State, MediaNavigator.State.Ready + public data object Ready : PlaybackState, MediaNavigator.State.Ready - public data object Starved : State, MediaNavigator.State.Buffering + public data object Starved : PlaybackState, MediaNavigator.State.Buffering - public data object Ended : State, MediaNavigator.State.Ended + public data object Ended : PlaybackState, MediaNavigator.State.Ended - public data class Failure(val error: Error) : State, MediaNavigator.State.Failure + public data class Failure(val error: Error) : PlaybackState, MediaNavigator.State.Failure } public sealed class Error( @@ -92,46 +94,39 @@ public class ReadAloudNavigator private constructor Error("An error occurred while trying to read publication content.", cause) } - private inner class AudioEngineListener : AudioEngine.Listener { + private inner class PlaybackEngineListener : PlaybackEngine.Listener { - override fun onItemChanged(engine: AudioEngine, index: Int) { - with(stateMachine) { - if (engine == stateMutable.value.segment.player.player) { - stateMutable.value = stateMutable.value.onAudioEngineItemChanged(index) + private val handler = Handler(Looper.getMainLooper()) + + override fun onPlaybackStateChanged(state: PlaybackEngine.PlaybackState) { + handler.post { + with(stateMachine) { + stateMutable.value = stateMutable.value.onPlaybackEngineStateChanged(state) } } } - override fun onStateChanged(engine: AudioEngine, state: AudioEngine.PlaybackState) { - with(stateMachine) { - if (engine == stateMutable.value.segment.player.player) { - stateMutable.value = stateMutable.value.onAudioEngineStateChanged(state) + override fun onStartRequested(initialState: PlaybackEngine.PlaybackState) { + handler.post { + with(stateMachine) { + stateMutable.value = stateMutable.value.onPlaybackEngineStateChanged(initialState) } } } - } - private inner class TtsPlayerListener : TtsPlayer.Listener { + override fun onStopRequested() { + } - override fun onItemChanged( - player: TtsPlayer, - index: Int, - ) { - with(stateMachine) { - if (player == stateMutable.value.segment.player.player) { - stateMutable.value = stateMutable.value.onTtsPlayerItemChanged(index) + override fun onPlaybackCompleted() { + handler.post { + with(stateMachine) { + stateMutable.value = stateMutable.value.onPlaybackCompleted() } } } - override fun onStateChanged( - player: TtsPlayer, - state: TtsPlayer.PlaybackState, - ) { - with(stateMachine) { - if (player == stateMutable.value.segment.player.player) { - stateMutable.value = stateMutable.value.onTtsPlayerStateChanged(state) - } + override fun onRangeStarted(range: IntRange) { + handler.post { } } } @@ -139,15 +134,23 @@ public class ReadAloudNavigator private constructor public var settings: ReadAloudSettings by Delegates.observable(initialSettings) { property, oldValue, newValue -> with(stateMachine) { - stateMutable.value = stateMutable.value.updateSettings(oldValue, newValue) + if (newValue != oldValue) { + stateMutable.value = stateMutable.value.onSettingsChanged(newValue) + } } } - private val segmentFactory = ReadAloudSegmentFactory( - audioEngineFactory = { playlist: List -> - audioEngineFactory(playlist, AudioEngineListener()) + public val voices: Set = + ttsEngineFactory.voices + + private val segmentFactory = ReadAloudSegmentFactory( + audioEngineFactory = { chunks: List -> + audioEngineFactory.createPlaybackEngine( + chunks = chunks, + listener = PlaybackEngineListener() + ) }, - ttsPlayerFactory = { language: Language?, utterances: List -> + ttsEngineFactory = { language: Language?, utterances: List -> val language = settings.language .takeIf { settings.overrideContentLanguage } @@ -156,44 +159,30 @@ public class ReadAloudNavigator private constructor val preferredVoiceWithRegion = settings.voices[language] - ?.let { voiceId -> ttsEngineProvider.voices.firstOrNull { it.id == voiceId } } + ?.let { voiceId -> ttsEngineFactory.voices.firstOrNull { it.id == voiceId } } val preferredVoiceWithoutRegion = language .let { settings.voices[it.removeRegion()] } - ?.let { voiceId -> ttsEngineProvider.voices.firstOrNull { it.id == voiceId } } + ?.let { voiceId -> ttsEngineFactory.voices.firstOrNull { it.id == voiceId } } - val fallbackVoiceWithRegion = ttsEngineProvider.voices + val fallbackVoiceWithRegion = ttsEngineFactory.voices .firstOrNull { language in it.languages } - val fallbackVoiceWithoutRegion = ttsEngineProvider.voices + val fallbackVoiceWithoutRegion = ttsEngineFactory.voices .firstOrNull { voice -> language.removeRegion() in voice.languages.map { it.removeRegion() } } val voice = preferredVoiceWithRegion ?: preferredVoiceWithoutRegion ?: fallbackVoiceWithRegion ?: fallbackVoiceWithoutRegion - ?: ttsEngineProvider.voices.firstOrNull() - - val engineFactory = voice?.let { voice -> - { engineListener: TtsEngine.Listener -> - val engine = ttsEngineProvider.createEngine( - voice = voice, - utterances = utterances, - listener = engineListener - ) - - when (engine) { - is PausableTtsEngine -> engine - else -> PauseDecorator(engine) - } - } - } + ?: ttsEngineFactory.voices.firstOrNull() - engineFactory?.let { - TtsPlayer( - engineFactory = engineFactory, - listener = TtsPlayerListener() + voice?.let { voice -> + ttsEngineFactory.createPlaybackEngine( + voice = voice, + utterances = utterances, + listener = PlaybackEngineListener() ) } } @@ -213,14 +202,15 @@ public class ReadAloudNavigator private constructor } private val stateMutable: MutableStateFlow = run { - val itemRef = dataLoader.getItemRef(initialNode, prepare = false)!! + val itemRef = dataLoader.getItemRef(initialNode)!! // there is at least one node to read + val nodeIndex = itemRef.nodeIndex!! // the node has content MutableStateFlow( - stateMachine.init( + stateMachine.play( segment = itemRef.segment, - itemIndex = itemRef.nodeIndex ?: 0, - playWhenReady = false, - settings = initialSettings + itemIndex = nodeIndex, + playWhenReady = true, + settings = settings ) ) } @@ -241,7 +231,7 @@ public class ReadAloudNavigator private constructor private val ReadAloudStateMachine.State.utteranceLocation: UtteranceLocation? get() = when (segment) { is AudioSegment -> { - val textref = segment.textRefs[index] + val textref = segment.textRefs[nodeIndex] val href = textref.removeFragment() val cssSelector = textref.fragment ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } @@ -284,20 +274,19 @@ public class ReadAloudNavigator private constructor ) } - private fun ReadAloudStateMachine.PlaybackState.toState(): State = + private fun ReadAloudStateMachine.PlaybackState.toState(): PlaybackState = when (this) { is ReadAloudStateMachine.PlaybackState.Ready -> - State.Ready + PlaybackState.Ready is ReadAloudStateMachine.PlaybackState.Starved -> - State.Starved + PlaybackState.Starved is ReadAloudStateMachine.PlaybackState.Ended -> - State.Ended + PlaybackState.Ended is ReadAloudStateMachine.PlaybackState.Failure -> - State.Failure(Error.EngineError(error)) + PlaybackState.Failure(Error.EngineError(error)) } init { - stateMutable.value.segment.player.prepare() play() } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index 4ac3e7ac34..7248796f1c 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -28,7 +28,7 @@ public class ReadAloudNavigatorFactory private cons private val publicationMetadata: Metadata, private val guidedNavigationService: GuidedNavigationService, private val resources: List, - private val audioEngineFactory: (List, AudioEngine.Listener) -> AudioEngine, + private val audioEngineFactory: AudioEngineFactory, private val ttsEngineProvider: TtsEngineProvider, ) { @@ -59,9 +59,7 @@ public class ReadAloudNavigatorFactory private cons return null } - val audioEngineFactory = { playlist: List, listener: AudioEngine.Listener -> - audioEngineProvider.createEngine(publication, playlist, listener) - } + val audioEngineFactory = audioEngineProvider.createEngineFactory(publication) val resources = (publication.readingOrder + publication.resources).map { ReadAloudPublication.Item( @@ -122,12 +120,14 @@ public class ReadAloudNavigatorFactory private cons resources = resources, ) + val ttsEngineFactory = ttsEngineProvider.createEngineFactory() + val navigator = ReadAloudNavigator( initialSettings = initialSettings, initialLocation = initialLocation, publication = navigatorPublication, audioEngineFactory = audioEngineFactory, - ttsEngineProvider = ttsEngineProvider + ttsEngineFactory = ttsEngineFactory ) return Try.success(navigator) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt index 7206367b28..6763ff785f 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt @@ -22,29 +22,29 @@ internal sealed interface ReadAloudSegment { val nodes: List - val player: SegmentPlayer + val player: PlaybackEngine val emptyNodes: Set } internal data class AudioSegment( - override val player: AudioSegmentPlayer, - val items: List, + override val player: PlaybackEngine, + val items: List, val textRefs: List, override val nodes: List, override val emptyNodes: Set, ) : ReadAloudSegment internal data class TtsSegment( - override val player: TtsSegmentPlayer, + override val player: PlaybackEngine, val items: List, override val nodes: List, override val emptyNodes: Set, ) : ReadAloudSegment internal class ReadAloudSegmentFactory( - private val audioEngineFactory: (List) -> AudioEngine?, - private val ttsPlayerFactory: (Language?, List) -> TtsPlayer?, + private val audioEngineFactory: (List) -> PlaybackEngine?, + private val ttsEngineFactory: (Language?, List) -> PlaybackEngine?, ) { fun createSegmentFromNode(node: ReadAloudNode): ReadAloudSegment? = @@ -55,7 +55,7 @@ internal class ReadAloudSegmentFactory( firstNode: ReadAloudNode, ): AudioSegment? { var nextNode: ReadAloudNode? = firstNode - val audioItems = mutableListOf() + val audioChunks = mutableListOf() val textRefs = mutableListOf() val nodes = mutableListOf() val emptyNodes = mutableSetOf() @@ -64,7 +64,7 @@ internal class ReadAloudSegmentFactory( val audioContent = (nextNode.content as? AudioContent) if (audioContent != null) { - audioItems.add(audioContent.toAudioItem()) + audioChunks.add(audioContent.toAudioItem()) nodes.add(nextNode) textRefs.add(audioContent.textRef) } else { @@ -74,18 +74,16 @@ internal class ReadAloudSegmentFactory( nextNode = nextNode.next() } - if (audioItems.isEmpty()) { + if (audioChunks.isEmpty()) { return null } - val audioEngine = audioEngineFactory(audioItems) + val audioEngine = audioEngineFactory(audioChunks) ?: return null - audioEngine.playWhenReady = false - return AudioSegment( - player = AudioSegmentPlayer(audioEngine), - items = audioItems, + player = audioEngine, + items = audioChunks, textRefs = textRefs, nodes = nodes, emptyNodes = emptyNodes @@ -124,11 +122,11 @@ internal class ReadAloudSegmentFactory( val utterances = textItems.map { it.plain!! } - val ttsPlayer = ttsPlayerFactory(segmentLanguage, utterances) + val ttsPlayer = ttsEngineFactory(segmentLanguage, utterances) ?: return null return TtsSegment( - player = TtsSegmentPlayer(ttsPlayer), + player = ttsPlayer, items = textItems, nodes = nodes, emptyNodes = emptyNodes @@ -178,8 +176,8 @@ private data class TextContent( val text: GuidedNavigationText, ) : NodeContent -private fun AudioContent.toAudioItem(): AudioEngine.Item { - return AudioEngine.Item( +private fun AudioContent.toAudioItem(): AudioChunk { + return AudioChunk( href = href, interval = interval ) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 1e4fd66992..82ffa0284c 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -32,27 +32,32 @@ internal class ReadAloudStateMachine( val playbackState: PlaybackState, val playWhenReady: Boolean, val node: ReadAloudNode, - val index: Int, + val nodeIndex: Int, + val playerPaused: Boolean, val segment: ReadAloudSegment, val settings: ReadAloudSettings, ) - fun init( + fun play( segment: ReadAloudSegment, itemIndex: Int, playWhenReady: Boolean, settings: ReadAloudSettings, ): State { - segment.player.seekTo(itemIndex) - segment.player.playWhenReady = playWhenReady - segment.player.speed = settings.speed - segment.player.pitch = settings.pitch + segment.player.stop() + segment.player.itemToPlay = itemIndex + segment.applySettings(settings) + + if (playWhenReady) { + segment.player.start() + } return State( - playbackState = segment.player.playbackState.toStateMachinePlaybackState(), + playbackState = PlaybackState.Ready, playWhenReady = playWhenReady, node = segment.nodes[itemIndex], - index = itemIndex, + nodeIndex = itemIndex, + playerPaused = false, segment = segment, settings = settings ) @@ -72,7 +77,7 @@ internal class ReadAloudStateMachine( } val index = itemRef.nodeIndex!! // This is a content node - return init(itemRef.segment, index, playWhenReady, settings) + return play(itemRef.segment, index, playWhenReady, settings) } fun State.restart(): State { @@ -81,59 +86,65 @@ internal class ReadAloudStateMachine( val itemRef = dataLoader.getItemRef(node) ?: return copy(playbackState = PlaybackState.Ended) - return init(itemRef.segment, index, playWhenReady, settings) + val nodeIndex = itemRef.nodeIndex!! + + return play(itemRef.segment, nodeIndex, playWhenReady, settings) } fun State.pause(): State { - segment.player.playWhenReady = false - return copy(playWhenReady = false) + segment.player.pause() + return copy(playWhenReady = false, playerPaused = true) } fun State.resume(): State { - segment.player.playWhenReady = true - return copy(playWhenReady = true) + if (playerPaused) { + segment.player.resume() + } else { + segment.player.start() + } + return copy(playWhenReady = true, playerPaused = false) } - fun State.updateSettings( - oldSettings: ReadAloudSettings, + fun State.onSettingsChanged( newSettings: ReadAloudSettings, ): State { navigationHelper.settings = newSettings dataLoader.settings = newSettings - segment.player.pitch = settings.pitch - segment.player.speed = settings.speed - return copy(settings = newSettings).restart() } - fun State.onAudioEngineStateChanged(audioEngineState: AudioEngine.PlaybackState): State { - return when (audioEngineState) { - AudioEngine.PlaybackState.Ready -> copy(playbackState = PlaybackState.Ready) - AudioEngine.PlaybackState.Starved -> copy(playbackState = PlaybackState.Starved) - AudioEngine.PlaybackState.Ended -> onSegmentPlaybackEnded() - } + private fun ReadAloudSegment.applySettings(settings: ReadAloudSettings) { + player.speed = settings.speed + player.pitch = settings.pitch } - fun State.onTtsPlayerStateChanged(state: TtsPlayer.PlaybackState): State { + fun State.onPlaybackEngineStateChanged(state: PlaybackEngine.PlaybackState): State { return when (state) { - TtsPlayer.PlaybackState.Ready -> copy(playbackState = PlaybackState.Ready) - TtsPlayer.PlaybackState.Starved -> copy(playbackState = PlaybackState.Starved) - TtsPlayer.PlaybackState.Ended -> onSegmentPlaybackEnded() + PlaybackEngine.PlaybackState.Playing -> + copy(playbackState = PlaybackState.Ready) + PlaybackEngine.PlaybackState.Starved -> + copy(playbackState = PlaybackState.Starved) } } - fun State.onAudioEngineItemChanged(item: Int): State { - dataLoader.onPlaybackProgressed(segment.nodes[item]) - return copy(node = segment.nodes[item], index = item) + fun State.onPlaybackCompleted(): State { + return if (nodeIndex + 1 < segment.nodes.size) { + setSegmentItem(index = nodeIndex + 1, playWhenReady = playWhenReady && settings.readContinuously) + } else { + onSegmentEnded() + } } - fun State.onTtsPlayerItemChanged(item: Int): State { - dataLoader.onPlaybackProgressed(segment.nodes[item]) - return copy(node = segment.nodes[item], index = item) + private fun State.setSegmentItem(index: Int, playWhenReady: Boolean): State { + segment.player.itemToPlay = index + if (playWhenReady) { + segment.player.start() + } + return copy(nodeIndex = index, node = segment.nodes[index], playWhenReady = playWhenReady) } - private fun State.onSegmentPlaybackEnded(): State { + private fun State.onSegmentEnded(): State { val nextNode = node.next() ?: return copy(playbackState = PlaybackState.Ended) @@ -142,22 +153,8 @@ internal class ReadAloudStateMachine( segment.player.release() - newSegment.player.playWhenReady = playWhenReady - newSegment.player.speed = settings.speed - newSegment.player.pitch = settings.pitch + val playWhenReady = settings.readContinuously - return copy( - segment = newSegment, - playbackState = segment.player.playbackState.toStateMachinePlaybackState(), - node = newSegment.nodes[0], - index = 0 - ) + return play(newSegment, 0, playWhenReady, settings) } } - -private fun SegmentPlayer.PlaybackState.toStateMachinePlaybackState() = - when (this) { - SegmentPlayer.PlaybackState.Ready -> ReadAloudStateMachine.PlaybackState.Ready - SegmentPlayer.PlaybackState.Starved -> ReadAloudStateMachine.PlaybackState.Starved - SegmentPlayer.PlaybackState.Ended -> ReadAloudStateMachine.PlaybackState.Ended - } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt deleted file mode 100644 index 3f1f5c57d2..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SegmentPlayer.kt +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(ExperimentalReadiumApi::class) - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Error - -internal sealed interface SegmentPlayer { - - enum class PlaybackState { - Ready, - Starved, - Ended, - } - - val player: Any - - var playWhenReady: Boolean - - var pitch: Double - - var speed: Double - - val playbackState: PlaybackState - - fun prepare() - - fun seekTo(index: Int) - - fun release() -} - -internal class TtsSegmentPlayer( - override val player: TtsPlayer, -) : SegmentPlayer { - - override var playWhenReady: Boolean - get() = player.playWhenReady - set(value) { - player.playWhenReady = value - } - override var pitch: Double - get() = player.pitch - set(value) { - player.pitch = value - } - - override var speed: Double - get() = player.speed - set(value) { - player.speed = value - } - - override val playbackState: SegmentPlayer.PlaybackState - get() = when (player.playbackState) { - TtsPlayer.PlaybackState.Ready -> SegmentPlayer.PlaybackState.Ready - TtsPlayer.PlaybackState.Starved -> SegmentPlayer.PlaybackState.Starved - TtsPlayer.PlaybackState.Ended -> SegmentPlayer.PlaybackState.Ended - } - - override fun prepare() { - player.prepare() - } - - override fun seekTo(index: Int) { - player.seekTo(index) - } - - override fun release() { - player.release() - } -} - -internal class AudioSegmentPlayer( - override val player: AudioEngine, -) : SegmentPlayer { - - override var playWhenReady: Boolean - get() = player.playWhenReady - set(value) { - player.playWhenReady = value - } - - override var pitch: Double - get() = player.pitch - set(value) { - player.pitch = value - } - - override var speed: Double - get() = player.speed - set(value) { - player.speed = value - } - - override val playbackState: SegmentPlayer.PlaybackState - get() = when (player.playbackState) { - AudioEngine.PlaybackState.Ready -> SegmentPlayer.PlaybackState.Ready - AudioEngine.PlaybackState.Starved -> SegmentPlayer.PlaybackState.Starved - AudioEngine.PlaybackState.Ended -> SegmentPlayer.PlaybackState.Ended - } - - override fun prepare() { - } - - override fun seekTo(index: Int) { - player.seekTo(index) - } - - override fun release() { - player.release() - } -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt similarity index 72% rename from readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt rename to readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt index 89313cfb47..9c1ff8f53b 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/AndroidTtsEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt @@ -4,7 +4,7 @@ * available in the top-level LICENSE file of the project. */ -@file:OptIn(InternalReadiumApi::class) +@file:OptIn(InternalReadiumApi::class, ExperimentalReadiumApi::class) package org.readium.navigator.media.readaloud @@ -38,82 +38,75 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.MainScope import kotlinx.coroutines.cancel import kotlinx.coroutines.launch -import org.readium.navigator.media.readaloud.AndroidTtsEngine.Companion.initializeTextToSpeech import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.extensions.tryOrNull import org.readium.r2.shared.util.Language @ExperimentalReadiumApi -public class AndroidTtsEngineProvider private constructor( +public class SystemTtsEngineProvider( + private val context: Context, + private val maxConnectionRetries: Int = 3, +) : TtsEngineProvider { + + override suspend fun createEngineFactory(): TtsEngineFactory = + tryCreateEngineFactory(maxConnectionRetries) ?: NullPlaybackEngineFactory() + + private suspend fun tryCreateEngineFactory(maxRetries: Int): SystemTtsEngineFactory? { + suspend fun onFailure(): SystemTtsEngineFactory? = + if (maxRetries == 0) { + null + } else { + tryCreateEngineFactory(maxRetries - 1) + } + + val textToSpeech = initializeTextToSpeech(context) + ?: return onFailure() + + // Listing voices is not reliable. + val voices = tryOrNull { textToSpeech.voices } // throws on Nexus 4 + ?.filterNotNull() + ?.takeUnless { it.isEmpty() } + ?.associate { it.toTtsEngineVoice() to it } + ?: return onFailure() + + return SystemTtsEngineFactory( + context = context, + textToSpeech = textToSpeech, + fullVoices = voices, + maxConnectionRetries = maxConnectionRetries + ) + } +} + +@ExperimentalReadiumApi +public class SystemTtsEngineFactory internal constructor( private val context: Context, private val textToSpeech: TextToSpeech, - private val fullVoices: Map, -) : TtsEngineProvider { + private val fullVoices: Map, + private val maxConnectionRetries: Int, +) : TtsEngineFactory { - override val voices: Set = + override val voices: Set = fullVoices.keys - override fun createEngine( - voice: AndroidTtsEngine.Voice, + override fun createPlaybackEngine( + voice: SystemTtsEngine.Voice, utterances: List, - listener: TtsEngine.Listener, - ): TtsEngine { + listener: PlaybackEngine.Listener, + ): PlaybackEngine { val voice = fullVoices[voice] checkNotNull(voice) - return AndroidTtsEngine( + return SystemTtsEngine( context = context, engine = textToSpeech, listener = listener, - androidVoice = voice, - utterances = utterances + systemVoice = voice, + utterances = utterances, + maxConnectionRetries = maxConnectionRetries ) } - - public companion object { - - public suspend operator fun invoke( - context: Context, - maxRetries: Int = 2, - ): AndroidTtsEngineProvider? { - require(maxRetries >= 0) - - suspend fun onFailure(): AndroidTtsEngineProvider? = - if (maxRetries == 0) { - null - } else { - invoke(context, maxRetries - 1) - } - - val textToSpeech = initializeTextToSpeech(context) - ?: return onFailure() - - // Listing voices is not reliable. - val voices = tryOrNull { textToSpeech.voices } // throws on Nexus 4 - ?.filterNotNull() - ?.takeUnless { it.isEmpty() } - ?.associate { it.toTtsEngineVoice() to it } - ?: return onFailure() - - return AndroidTtsEngineProvider(context, textToSpeech, voices) - } - - private fun AndroidVoice.toTtsEngineVoice() = - AndroidTtsEngine.Voice( - name = name, - language = Language(locale), - quality = when (quality) { - QUALITY_VERY_HIGH -> AndroidTtsEngine.Voice.Quality.Highest - QUALITY_HIGH -> AndroidTtsEngine.Voice.Quality.High - QUALITY_NORMAL -> AndroidTtsEngine.Voice.Quality.Normal - QUALITY_LOW -> AndroidTtsEngine.Voice.Quality.Low - QUALITY_VERY_LOW -> AndroidTtsEngine.Voice.Quality.Lowest - else -> throw IllegalStateException("Unexpected voice quality.") - }, - requiresNetwork = isNetworkConnectionRequired - ) - } } /* @@ -125,16 +118,17 @@ public class AndroidTtsEngineProvider private constructor( */ /** - * Default [TtsEngine] implementation using Android's native text to speech engine. + * Default [PlaybackEngine] implementation using Android's native text to speech engine. */ @ExperimentalReadiumApi -public class AndroidTtsEngine internal constructor( +public class SystemTtsEngine internal constructor( private val context: Context, engine: TextToSpeech, - private val listener: TtsEngine.Listener, - private val androidVoice: AndroidVoice, - override val utterances: List, -) : TtsEngine { + private val listener: PlaybackEngine.Listener, + private val systemVoice: AndroidVoice, + private val utterances: List, + private val maxConnectionRetries: Int, +) : PlaybackEngine { public companion object { @@ -161,18 +155,6 @@ public class AndroidTtsEngine internal constructor( context.startActivity(intent) } } - - internal suspend fun initializeTextToSpeech( - context: Context, - ): TextToSpeech? { - val init = CompletableDeferred() - - val initListener = OnInitListener { status -> - init.complete(status == SUCCESS) - } - val engine = TextToSpeech(context, initListener) - return if (init.await()) engine else null - } } public sealed class Error( @@ -248,7 +230,7 @@ public class AndroidTtsEngine internal constructor( ) : TtsVoice { override val id: TtsVoice.Id = - TtsVoice.Id("${AndroidTtsEngine::class.qualifiedName}-$name}") + TtsVoice.Id("${SystemTtsEngine::class.qualifiedName}-$name}") override val languages: Set = setOf(language) @@ -290,36 +272,34 @@ public class AndroidTtsEngine internal constructor( private var state: State = State.EngineAvailable(engine) - private var isPrepared: Boolean = false - private var isClosed: Boolean = false override var speed: Double = 1.0 override var pitch: Double = 1.0 - override fun prepare() { - if (isPrepared) { - return - } + override var itemToPlay: Int = 0 - isPrepared = true + init { + setupListener(engine) + } - (state as? State.EngineAvailable) - ?.let { setupListener(it.engine) } + override fun start() { + listener.onStartRequested(PlaybackEngine.PlaybackState.Playing) + doStart(itemToPlay) + } - listener.onReady() + override fun resume() { + doStart(itemToPlay) } - override fun speak( - utteranceIndex: Int, - ) { - check(isPrepared) { "Engine has not been prepared." } + private fun doStart(utteranceIndex: Int) { check(!isClosed) { "Engine is closed." } val id = Request.Id(UUID.randomUUID().toString()) val text = checkNotNull(utterances)[utteranceIndex] val request = Request(id, text) + when (val stateNow = state) { is State.WaitingForService -> { stateNow.pendingRequest = request @@ -328,7 +308,7 @@ public class AndroidTtsEngine internal constructor( tryReconnect(request) } is State.EngineAvailable -> { - if (!doSpeak(stateNow.engine, request)) { + if (!speak(stateNow.engine, request)) { cleanEngine(stateNow.engine) tryReconnect(request) } @@ -337,6 +317,15 @@ public class AndroidTtsEngine internal constructor( } public override fun stop() { + listener.onStopRequested() + doStop() + } + + override fun pause() { + doStop() + } + + private fun doStop() { when (val stateNow = state) { is State.EngineAvailable -> { stateNow.engine.stop() @@ -371,7 +360,7 @@ public class AndroidTtsEngine internal constructor( } } - private fun doSpeak( + private fun speak( engine: TextToSpeech, request: Request, ): Boolean { @@ -391,14 +380,14 @@ public class AndroidTtsEngine internal constructor( if (isClosed) { engine.shutdown() } else { - previousState.pendingRequest?.let { doSpeak(engine, it) } + previousState.pendingRequest?.let { speak(engine, it) } } } private fun onReconnectionFailed() { val error = Error.Service state = State.Failure(error) - listener.onError(error) + // listener.onError(error) } private fun tryReconnect(request: Request) { @@ -428,21 +417,20 @@ public class AndroidTtsEngine internal constructor( } private fun TextToSpeech.setupVoice(): Boolean { - return setVoice(androidVoice) == SUCCESS + return setVoice(systemVoice) == SUCCESS } private class UtteranceListener( - private val listener: TtsEngine.Listener, + private val listener: PlaybackEngine.Listener, ) : UtteranceProgressListener() { override fun onStart(utteranceId: String) { } override fun onStop(utteranceId: String, interrupted: Boolean) { - listener.onInterrupted() } override fun onDone(utteranceId: String) { - listener.onDone() + listener.onPlaybackCompleted() } @Deprecated( @@ -455,11 +443,38 @@ public class AndroidTtsEngine internal constructor( } override fun onError(utteranceId: String, errorCode: Int) { - listener.onError(Error.fromNativeError(errorCode)) + // listener.onError(Error.fromNativeError(errorCode)) } override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { - listener.onRange(start until end) + listener.onRangeStarted(start until end) } } } + +private fun AndroidVoice.toTtsEngineVoice() = + SystemTtsEngine.Voice( + name = name, + language = Language(locale), + quality = when (quality) { + QUALITY_VERY_HIGH -> SystemTtsEngine.Voice.Quality.Highest + QUALITY_HIGH -> SystemTtsEngine.Voice.Quality.High + QUALITY_NORMAL -> SystemTtsEngine.Voice.Quality.Normal + QUALITY_LOW -> SystemTtsEngine.Voice.Quality.Low + QUALITY_VERY_LOW -> SystemTtsEngine.Voice.Quality.Lowest + else -> throw IllegalStateException("Unexpected voice quality.") + }, + requiresNetwork = isNetworkConnectionRequired + ) + +private suspend fun initializeTextToSpeech( + context: Context, +): TextToSpeech? { + val init = CompletableDeferred() + + val initListener = OnInitListener { status -> + init.complete(status == SUCCESS) + } + val engine = TextToSpeech(context, initListener) + return if (init.await()) engine else null +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt deleted file mode 100644 index 5a5d27abc5..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsEngine.kt +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(ExperimentalReadiumApi::class) - -package org.readium.navigator.media.readaloud - -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Error -import org.readium.r2.shared.util.Language - -@ExperimentalReadiumApi -public interface TtsEngine { - - public interface Listener { - - public fun onReady() - - public fun onDone() - - public fun onInterrupted() - - /** - * Called when the [TtsEngine] is about to speak the specified [range] of the utterance with - * the given id. - * - * This callback may not be called if the [TtsEngine] does not provide range information. - */ - public fun onRange(range: IntRange) - - /** - * Called when an error has occurred during processing of the utterance with the given id. - */ - public fun onError(error: E) - } - - public val utterances: List - - public var pitch: Double - - public var speed: Double - - public fun prepare() - - public fun speak(utteranceIndex: Int) - - public fun stop() - - public fun release() -} - -@ExperimentalReadiumApi -public interface TtsVoice { - - @kotlinx.serialization.Serializable - @JvmInline - public value class Id(public val value: String) - - public val id: Id - - /** - * The languages supported by the voice. - */ - public val languages: Set -} - -@ExperimentalReadiumApi -public interface TtsEngineProvider { - - /** - * Sets of voices available with this [TtsEngineProvider]. - */ - public val voices: Set - - public fun createEngine( - voice: V, - utterances: List, - listener: TtsEngine.Listener, - ): TtsEngine -} - -@ExperimentalReadiumApi -public interface PausableTtsEngine : TtsEngine { - - public fun pause() - - public fun resume() -} - -internal class PauseDecorator( - private val engine: TtsEngine, -) : PausableTtsEngine, TtsEngine by engine { - - private var currentIndex: Int = 0 - - override fun pause() { - engine.stop() - } - - override fun resume() { - engine.speak(currentIndex) - } - - override fun speak(utteranceIndex: Int) { - currentIndex = utteranceIndex - engine.speak(utteranceIndex) - } -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt deleted file mode 100644 index b4c1c31ecf..0000000000 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsPlayer.kt +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2025 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(ExperimentalReadiumApi::class) - -package org.readium.navigator.media.readaloud - -import kotlin.properties.Delegates -import org.readium.navigator.media.readaloud.StateMachine.PlaybackState -import org.readium.navigator.media.readaloud.StateMachine.State -import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Error - -internal class TtsPlayer( - engineFactory: (TtsEngine.Listener) -> PausableTtsEngine, - private val listener: Listener, -) { - - interface Listener { - - fun onItemChanged(player: TtsPlayer, index: Int) - - fun onStateChanged(player: TtsPlayer, state: TtsPlayer.PlaybackState) - } - - internal enum class PlaybackState { - Ready, - Starved, - Ended, - } - - private inner class EngineListener : TtsEngine.Listener { - override fun onReady() { - with(stateMachine) { - state = state.onEngineReady() - } - } - - override fun onDone() { - with(stateMachine) { - state = state.onUtteranceCompleted() - } - } - - override fun onInterrupted() { - } - - override fun onRange(range: IntRange) { - } - - override fun onError(error: E) { - } - } - - private val ttsEngine = engineFactory(EngineListener()) - - private val stateMachine = StateMachine(ttsEngine) - - private var state by Delegates.observable( - State( - playWhenReady = false, - playbackState = StateMachine.PlaybackState.Starved, - indexToPlay = 0, - lastSubmittedIndex = null - ) - ) { property, oldValue, newValue -> - if (oldValue.playbackState != newValue.playbackState) { - val playerState = newValue.playbackState.toPlayerPlaybackState() - listener.onStateChanged(this, playerState) - } - - if (oldValue.indexToPlay != newValue.indexToPlay) { - listener.onItemChanged(this, newValue.indexToPlay) - } - } - - var playWhenReady: Boolean - get() = state.playWhenReady - set(value) { - with(stateMachine) { - state = if (value) state.resume() else state.pause() - } - } - - var pitch: Double - get() = ttsEngine.pitch - set(value) { - ttsEngine.pitch = value - } - - var speed: Double - get() = ttsEngine.speed - set(value) { - ttsEngine.speed = value - } - - val playbackState: TtsPlayer.PlaybackState get() = - state.playbackState.toPlayerPlaybackState() - - fun prepare() { - ttsEngine.prepare() - } - - fun seekTo(index: Int) { - with(stateMachine) { - state = state.seekTo(index) - } - } - - fun release() { - ttsEngine.release() - } -} - -private fun StateMachine.PlaybackState.toPlayerPlaybackState() = - when (this) { - PlaybackState.Ended -> TtsPlayer.PlaybackState.Ended - PlaybackState.Ready -> TtsPlayer.PlaybackState.Ready - PlaybackState.Starved -> TtsPlayer.PlaybackState.Starved - } - -private class StateMachine( - private val engine: PausableTtsEngine, -) { - sealed interface PlaybackState { - - data object Ready : PlaybackState - - data object Starved : PlaybackState - - data object Ended : PlaybackState - } - - data class State( - val playbackState: PlaybackState, - val playWhenReady: Boolean, - val indexToPlay: Int, - val lastSubmittedIndex: Int?, - ) - - fun State.pause(): State { - engine.pause() - return copy(playWhenReady = false) - } - - fun State.resume(): State { - if (lastSubmittedIndex == indexToPlay) { - engine.resume() - } else { - engine.stop() - engine.speak(indexToPlay) - } - - return copy(playWhenReady = true, lastSubmittedIndex = indexToPlay) - } - - fun State.seekTo(index: Int): State { - if (playWhenReady) { - engine.stop() - engine.speak(index) - return copy(indexToPlay = index, lastSubmittedIndex = index) - } else { - return copy(indexToPlay = index) - } - } - - fun State.onUtteranceCompleted(): State { - if (indexToPlay < engine.utterances.size - 1) { - val newIndexToPlay = indexToPlay + 1 - engine.speak(newIndexToPlay) - return copy(indexToPlay = newIndexToPlay, lastSubmittedIndex = newIndexToPlay) - } else { - return copy(playbackState = PlaybackState.Ended) - } - } - - fun State.onEngineReady(): State { - if (playWhenReady) { - engine.speak(indexToPlay) - } - return copy(playbackState = PlaybackState.Ready, lastSubmittedIndex = indexToPlay) - } -} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt new file mode 100644 index 0000000000..8fbc40514d --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import org.readium.r2.shared.util.Language + +/** + * Default values for the ReadAloudNavigator. + * + * These values will be used as a last resort when no user preference takes precedence. + * + * @see ReadAloudPreferences + */ +public data class ReadAloudDefaults( + val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null, + val readContinuously: Boolean? = null, +) { + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt new file mode 100644 index 0000000000..9d65c763b2 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import kotlinx.serialization.Serializable +import org.readium.navigator.common.Preferences +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.util.Language + +/** + * Preferences for the the ReadAloudNavigator. + * + * @param language Language of the publication content. + * @param pitch Playback pitch rate. + * @param speed Playback speed rate. + * @param voices Map of preferred voices for specific languages. + * @param escapableRoles Roles that will be considered as escapable. + * @param skippableRoles Roles that will be considered as skippable. + * @param readContinuously Do not pause after reading each content item. + */ +@ExperimentalReadiumApi +@Serializable +public data class ReadAloudPreferences( + val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null, + val voices: Map? = null, + val escapableRoles: Set? = null, + val skippableRoles: Set? = null, + val readContinuously: Boolean? = true, +) : Preferences { + + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } + + public override fun plus(other: ReadAloudPreferences): ReadAloudPreferences = + ReadAloudPreferences( + language = other.language ?: language, + pitch = other.pitch ?: pitch, + speed = other.speed ?: speed, + voices = other.voices ?: voices, + escapableRoles = other.escapableRoles ?: escapableRoles, + skippableRoles = other.skippableRoles ?: skippableRoles, + readContinuously = other.readContinuously ?: readContinuously + ) +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt new file mode 100644 index 0000000000..2483d3f1bf --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class) + +package org.readium.navigator.media.readaloud.preferences + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import org.readium.navigator.common.PreferencesEditor +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.navigator.extensions.format +import org.readium.r2.navigator.preferences.DoubleIncrement +import org.readium.r2.navigator.preferences.Preference +import org.readium.r2.navigator.preferences.PreferenceDelegate +import org.readium.r2.navigator.preferences.RangePreference +import org.readium.r2.navigator.preferences.RangePreferenceDelegate +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +/** + * Editor for a set of [ReadAloudPreferences]. + * + * Use [ReadAloudPreferencesEditor] to assist you in building a preferences user interface or modifying + * existing preferences. It includes rules for adjusting preferences, such as the supported values + * or ranges. + */ +@ExperimentalReadiumApi +public class ReadAloudPreferencesEditor( + initialPreferences: ReadAloudPreferences, + publicationMetadata: Metadata, + defaults: ReadAloudDefaults, +) : PreferencesEditor { + + private data class State( + val preferences: ReadAloudPreferences, + val settings: ReadAloudSettings, + ) + + private val coroutineScope: CoroutineScope = + MainScope() + private val settingsResolver: ReadAloudSettingsResolver = + ReadAloudSettingsResolver(publicationMetadata, defaults) + + private var state: MutableStateFlow = + MutableStateFlow(initialPreferences.toState()) + + override val preferences: ReadAloudPreferences + get() = state.value.preferences + + override val settings: ReadAloudSettings + get() = state.value.settings + + public val preferencesState: StateFlow = + state.mapStateIn(coroutineScope) { it.preferences } + + override fun clear() { + updateValues { ReadAloudPreferences() } + } + + public val language: Preference = + PreferenceDelegate( + getValue = { preferences.language }, + getEffectiveValue = { state.value.settings.language }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(language = value) } } + ) + + public val pitch: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.pitch }, + getEffectiveValue = { state.value.settings.pitch }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(pitch = value) } }, + supportedRange = 0.1..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(2)}x" } + ) + + public val speed: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.speed }, + getEffectiveValue = { state.value.settings.speed }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(speed = value) } }, + supportedRange = 0.1..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(2)}x" } + ) + + public val voices: Preference> = + PreferenceDelegate( + getValue = { preferences.voices }, + getEffectiveValue = { state.value.settings.voices }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(voices = value) } } + ) + + public val readContinuously: Preference = + PreferenceDelegate( + getValue = { preferences.readContinuously }, + getEffectiveValue = { state.value.settings.readContinuously }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(readContinuously = value) } } + ) + + private fun updateValues(updater: (ReadAloudPreferences) -> ReadAloudPreferences) { + val newPreferences = updater(preferences) + state.value = newPreferences.toState() + } + + private fun ReadAloudPreferences.toState(): State { + return State( + preferences = this, + settings = settingsResolver.settings(this), + ) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt index 995088adbf..e2131299a5 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt @@ -21,4 +21,5 @@ public data class ReadAloudSettings( val voices: Map, val escapableRoles: Set, val skippableRoles: Set, + val readContinuously: Boolean, ) : Settings diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt new file mode 100644 index 0000000000..05437314b1 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import java.util.Locale +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.ASIDE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.BIBLIOGRAPHY +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.CELL +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.ENDNOTES +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.FIGURE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.FOOTNOTE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LANDMARKS +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LIST +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LIST_ITEM +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOA +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOI +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOT +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOV +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.NOTEREF +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.PAGEBREAK +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.PULLQUOTE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.ROW +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.TABLE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.TOC +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +@OptIn(ExperimentalReadiumApi::class) +internal class ReadAloudSettingsResolver( + private val metadata: Metadata, + private val defaults: ReadAloudDefaults, +) { + + fun settings(preferences: ReadAloudPreferences): ReadAloudSettings { + val language = preferences.language + ?: metadata.language + ?: defaults.language + ?: Language(Locale.getDefault()) + + val defaultSkippableRoles: Set = + setOf( + ASIDE, BIBLIOGRAPHY, ENDNOTES, FOOTNOTE, NOTEREF, PULLQUOTE, + LANDMARKS, LOA, LOI, LOT, LOV, PAGEBREAK, TOC + ) + + val defaultEscapableRoles: Set = + setOf(ASIDE, FIGURE, LIST, LIST_ITEM, TABLE, ROW, CELL) + + return ReadAloudSettings( + language = language, + voices = preferences.voices ?: emptyMap(), + pitch = preferences.pitch ?: defaults.pitch ?: 1.0, + speed = preferences.speed ?: defaults.speed ?: 1.0, + overrideContentLanguage = preferences.language != null, + escapableRoles = preferences.escapableRoles ?: defaultEscapableRoles, + skippableRoles = preferences.skippableRoles ?: defaultSkippableRoles, + readContinuously = preferences.readContinuously ?: defaults.readContinuously ?: true + ) + } +} From 629f31d59d5df31d37d32759597ffc3e478120e2 Mon Sep 17 00:00:00 2001 From: Quentin Gliosca Date: Fri, 12 Sep 2025 12:44:53 +0200 Subject: [PATCH 13/13] Various improvements --- .../readium/demo/navigator/DemoViewModel.kt | 2 +- .../navigator/reader/ReadAloudRendition.kt | 12 +- .../demo/navigator/reader/ReaderOpener.kt | 5 +- .../demo/navigator/reader/ReaderState.kt | 6 +- .../navigator/reader/SelectNavigatorMenu.kt | 3 +- .../exoplayer/readaloud/ExoPlayerEngine.kt | 6 +- .../readaloud/ExoPlayerEngineProvider.kt | 9 +- .../navigator/common/LocationElements.kt | 7 + .../org/readium/navigator/common/Locations.kt | 6 + ...e.kt => ContentGuidedNavigationService.kt} | 24 +- .../media/readaloud/PlaybackEngine.kt | 143 ++++++++--- .../media/readaloud/ReadAloudDataLoader.kt | 7 +- .../media/readaloud/ReadAloudLocations.kt | 98 ++----- .../readaloud/ReadAloudNavigationHelper.kt | 21 +- .../media/readaloud/ReadAloudNavigator.kt | 243 ++++++++++-------- .../readaloud/ReadAloudNavigatorFactory.kt | 30 +-- .../media/readaloud/ReadAloudSegment.kt | 7 +- .../media/readaloud/ReadAloudStateMachine.kt | 9 +- .../media/readaloud/SystemTtsEngine.kt | 19 +- .../preferences/ReadAloudDefaults.kt | 7 + .../preferences/ReadAloudSettingsResolver.kt | 32 ++- 21 files changed, 387 insertions(+), 309 deletions(-) rename readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/{TtsGuidedNavigationService.kt => ContentGuidedNavigationService.kt} (83%) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index c868b972f0..e22f0350c6 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -127,7 +127,7 @@ class DemoViewModel( val ttsEngineProvider = SystemTtsEngineProvider(application) - val readAloudFactory = ReadAloudNavigatorFactory( + val readAloudFactory = ReadAloudNavigatorFactory.invoke( application = application, publication = publication, audioEngineProvider = audioEngineProvider, diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt index 2e9a969bea..ca007662e5 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt @@ -37,6 +37,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp import androidx.compose.ui.zIndex import org.readium.demo.navigator.preferences.UserPreferences +import org.readium.navigator.media.readaloud.ReadAloudTextHighlightLocation import org.readium.r2.shared.ExperimentalReadiumApi @Composable @@ -106,11 +107,16 @@ fun ReadAloudRendition( Text("Play When Ready: ${playbackState.value.playWhenReady}") - Text("Resource Href: ${playbackState.value.utteranceLocation?.href}") + when (val highlightLocation = playbackState.value.nodeHighlightLocation) { + is ReadAloudTextHighlightLocation -> { + Text("Node Highlight Href: ${highlightLocation.href}") - Text("Utterance Css Selector ${playbackState.value.utteranceLocation?.cssSelector?.value}") + Text("Node Highlight Css Selector ${highlightLocation.cssSelector?.value}") - Text("Utterance ${playbackState.value.utteranceLocation?.textQuote?.text}") + Text("Node Highlight Text ${highlightLocation.textQuote?.text}") + } + null -> {} + } } Toolbar(readerState) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index e03c4b3aad..c66cede80b 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -205,7 +205,7 @@ class ReaderOpener( private suspend fun createReadAloudReader( url: AbsoluteUrl, publication: Publication, - navigatorFactory: ReadAloudNavigatorFactory, + navigatorFactory: ReadAloudNavigatorFactory, initialLocator: Locator?, ): Try { val coroutineScope = MainScope() @@ -222,9 +222,10 @@ class ReaderOpener( .getOrElse { return Try.failure(it) } val preferencesEditor = preferencesManager.preferences.mapStateIn(coroutineScope) { + @Suppress("UNCHECKED_CAST") ReadAloudPreferencesEditor( editor = navigatorFactory.createPreferencesEditor(it), - availableVoices = navigator.voices + availableVoices = navigator.voices as Set ) } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt index 65802d49ba..3155c02a00 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt @@ -21,7 +21,6 @@ import org.readium.navigator.common.RenditionState import org.readium.navigator.common.SelectionController import org.readium.navigator.common.SelectionLocation import org.readium.navigator.media.readaloud.ReadAloudNavigator -import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl @@ -52,10 +51,13 @@ data class ReadAloudReaderState( val url: AbsoluteUrl, val coroutineScope: CoroutineScope, val publication: Publication, - val navigator: ReadAloudNavigator, + val navigator: ReadAloudNavigator, val preferencesEditor: StateFlow, ) : ReaderState { override fun close() { + navigator.release() + coroutineScope.cancel() + publication.close() } } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt index d4b815ad0f..edd5cf295e 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt @@ -14,7 +14,6 @@ import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.ui.window.PopupProperties import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory -import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.r2.shared.ExperimentalReadiumApi @@ -49,7 +48,7 @@ sealed class SelectNavigatorItem( ) : SelectNavigatorItem("Fixed Web Rendition") data class ReadAloud( - override val factory: ReadAloudNavigatorFactory, + override val factory: ReadAloudNavigatorFactory, ) : SelectNavigatorItem("Read Aloud Navigator") } diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt index d78bd402f1..fb1c28313d 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -20,6 +20,7 @@ import androidx.media3.exoplayer.source.DefaultMediaSourceFactory import kotlin.properties.Delegates import kotlinx.coroutines.ExperimentalCoroutinesApi import org.readium.navigator.media.readaloud.AudioChunk +import org.readium.navigator.media.readaloud.AudioEngineProgress import org.readium.navigator.media.readaloud.PlaybackEngine import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi @@ -36,7 +37,7 @@ import org.readium.r2.shared.util.data.ReadException @androidx.annotation.OptIn(UnstableApi::class) public class ExoPlayerEngine private constructor( private val exoPlayer: ExoPlayer, - private val listener: PlaybackEngine.Listener, + private val listener: PlaybackEngine.Listener, ) : PlaybackEngine { public companion object { @@ -45,7 +46,7 @@ public class ExoPlayerEngine private constructor( application: Application, dataSourceFactory: DataSource.Factory, chunks: List, - listener: PlaybackEngine.Listener, + listener: PlaybackEngine.Listener, ): ExoPlayerEngine { val exoPlayer = ExoPlayer.Builder(application) .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) @@ -203,7 +204,6 @@ public class ExoPlayerEngine private constructor( exoPlayer.playWhenReady = false exoPlayer.seekTo(0) state = State.Idle - listener.onStopRequested() } override fun resume() { diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt index f2be8e1721..b382993409 100644 --- a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt @@ -12,6 +12,7 @@ import android.app.Application import org.readium.adapter.exoplayer.audio.ExoPlayerDataSource import org.readium.navigator.media.readaloud.AudioChunk import org.readium.navigator.media.readaloud.AudioEngineFactory +import org.readium.navigator.media.readaloud.AudioEngineProgress import org.readium.navigator.media.readaloud.AudioEngineProvider import org.readium.navigator.media.readaloud.PlaybackEngine import org.readium.r2.shared.ExperimentalReadiumApi @@ -21,9 +22,9 @@ import org.readium.r2.shared.publication.Publication @ExperimentalReadiumApi public class ExoPlayerEngineProvider( private val application: Application, -) : AudioEngineProvider { +) : AudioEngineProvider { - override fun createEngineFactory(publication: Publication): AudioEngineFactory { + override fun createEngineFactory(publication: Publication): ExoPlayerEngineFactory { val dataSourceFactory = ExoPlayerDataSource.Factory(publication) return ExoPlayerEngineFactory(application, dataSourceFactory) } @@ -33,11 +34,11 @@ public class ExoPlayerEngineProvider( public class ExoPlayerEngineFactory internal constructor( private val application: Application, private val dataSourceFactory: ExoPlayerDataSource.Factory, -) : AudioEngineFactory { +) : AudioEngineFactory { override fun createPlaybackEngine( chunks: List, - listener: PlaybackEngine.Listener, + listener: PlaybackEngine.Listener, ): PlaybackEngine { return ExoPlayerEngine(application, dataSourceFactory, chunks, listener) } diff --git a/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt b/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt index 6da3d29421..58c7ac9cce 100644 --- a/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt +++ b/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt @@ -6,6 +6,7 @@ package org.readium.navigator.common +import kotlin.time.Duration import org.readium.r2.shared.ExperimentalReadiumApi /** @@ -66,3 +67,9 @@ public data class TextAnchor( val prefix: String, val suffix: String, ) + +@JvmInline +@ExperimentalReadiumApi +public value class TimeOffset( + public val value: Duration, +) diff --git a/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt b/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt index b1f1ec9145..bc3947ea36 100644 --- a/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt +++ b/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt @@ -72,3 +72,9 @@ public interface PositionLocation : Location { public val position: Position } + +@ExperimentalReadiumApi +public interface TimeOffsetLocation : Location { + + public val timeOffset: TimeOffset +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentGuidedNavigationService.kt similarity index 83% rename from readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt rename to readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentGuidedNavigationService.kt index af73ebb7ee..07b336fe30 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/TtsGuidedNavigationService.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentGuidedNavigationService.kt @@ -13,14 +13,17 @@ import org.readium.r2.shared.guided.GuidedNavigationDocument import org.readium.r2.shared.guided.GuidedNavigationObject import org.readium.r2.shared.guided.GuidedNavigationText import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.html.cssSelector import org.readium.r2.shared.publication.services.GuidedNavigationIterator import org.readium.r2.shared.publication.services.GuidedNavigationService import org.readium.r2.shared.publication.services.content.Content import org.readium.r2.shared.publication.services.content.ContentService import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.data.ReadError -internal class TtsGuidedNavigationService( +internal class ContentGuidedNavigationService( private val contentService: ContentService, ) : GuidedNavigationService { @@ -59,13 +62,12 @@ internal class TtsGuidedNavigationService( val nodes = when (val element = contentIterator.next()) { is Content.TextElement -> { element.segments.mapNotNull { segment -> - if (segment.text.isEmpty()) { + if (segment.text.isBlank()) { return@mapNotNull null } + GuidedNavigationObject( - refs = setOf( - GuidedNavigationTextRef(segment.locator.href) - ), + refs = setOfNotNull(segment.locator.toTextRef()), text = GuidedNavigationText( plain = segment.text, ssml = null, @@ -81,9 +83,7 @@ internal class TtsGuidedNavigationService( ?.takeIf { it.isNotBlank() } ?.let { GuidedNavigationObject( - refs = setOf( - GuidedNavigationTextRef(element.locator.href) - ), + refs = setOfNotNull(element.locator.toTextRef()), text = GuidedNavigationText(it) ) } @@ -104,5 +104,13 @@ internal class TtsGuidedNavigationService( return GuidedNavigationDocument(guided = tree) } + + private fun Locator.toTextRef(): GuidedNavigationTextRef? { + val htmlId = locations.cssSelector + ?.takeIf { it.startsWith("#") } + .orEmpty() + val url = Url("$href$htmlId") + return url?.let { GuidedNavigationTextRef(it) } + } } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt index 8da4c78790..4b91720628 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt @@ -8,6 +8,7 @@ package org.readium.navigator.media.readaloud +import kotlin.time.Duration import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.Error @@ -16,19 +17,19 @@ import org.readium.r2.shared.util.TimeInterval import org.readium.r2.shared.util.Url @ExperimentalReadiumApi -public interface AudioEngineProvider { +public interface AudioEngineProvider { public fun createEngineFactory( publication: Publication, - ): AudioEngineFactory + ): AudioEngineFactory } @ExperimentalReadiumApi -public interface AudioEngineFactory { +public interface AudioEngineFactory { public fun createPlaybackEngine( chunks: List, - listener: PlaybackEngine.Listener, + listener: PlaybackEngine.Listener, ): PlaybackEngine } @@ -39,100 +40,137 @@ public data class AudioChunk( ) @ExperimentalReadiumApi -public interface TtsEngineProvider { +public interface TtsEngineProvider { public suspend fun createEngineFactory(): TtsEngineFactory } @ExperimentalReadiumApi -public interface TtsEngineFactory { +public interface TtsEngineFactory { /** * Sets of voices available with this [TtsEngineFactory]. */ public val voices: Set + /** + * Creates a [PlaybackEngine] to read [utterances] using the given [voiceId]. + * + * Throws if the given [voiceId] matches no voice in [voices]. + */ public fun createPlaybackEngine( - voice: V, + voiceId: TtsVoice.Id, utterances: List, - listener: PlaybackEngine.Listener, + listener: PlaybackEngine.Listener, ): PlaybackEngine } -internal class NullPlaybackEngineFactory() : TtsEngineFactory { +internal class NullTtsEngineFactory() : TtsEngineFactory { override val voices: Set = emptySet() override fun createPlaybackEngine( - voice: V, + voiceId: TtsVoice.Id, utterances: List, - listener: PlaybackEngine.Listener, + listener: PlaybackEngine.Listener, ): PlaybackEngine { throw IllegalArgumentException("Unknown voice.") } } -@ExperimentalReadiumApi -public interface TtsVoice { - - @kotlinx.serialization.Serializable - @JvmInline - public value class Id(public val value: String) - - public val id: Id - - /** - * The languages supported by the voice. - */ - public val languages: Set -} - +/** + * Engine reading aloud a list of items. + */ @ExperimentalReadiumApi public interface PlaybackEngine { - public var pitch: Double - - public var speed: Double - /** - * Sets the index of the item to play on the next call to [start]. + * State of the playback. */ - public var itemToPlay: Int - public enum class PlaybackState { + /** + * The playback is ongoing. + */ Playing, + + /** + * The playback has been momentarily interrupted because of a lack of ready data. + */ Starved, } - public interface Listener { + /** + * Marker interface for playback progress information. + */ + public interface Progress + + /** + * Listener for a [PlaybackEngine] + */ + public interface Listener { + /** + * Called after [start] was invoked. [initialState] tells you if the engine has enough data + * to start playing right now or must still wait for data. + */ public fun onStartRequested(initialState: PlaybackState) - public fun onStopRequested() + /** + * Called when the playback state changes. + */ + public fun onPlaybackStateChanged(state: PlaybackState) + /** + * Called when the last playback request completed. + */ public fun onPlaybackCompleted() - public fun onPlaybackStateChanged(state: PlaybackState) + /** + * Called when an error occurred during playback. + */ + public fun onPlaybackError(error: E) - public fun onRangeStarted(range: IntRange) + /** + * Called regularly to report progress when this information is available. + */ + public fun onPlaybackProgressed(progress: P) } /** - * Starts playing the [itemToPlay]-th item. + * Sets the playback pitch. + */ + public var pitch: Double + + /** + * Sets the playback speed. + */ + public var speed: Double + + /** + * Sets the index of the item to play on the next call to [start]. * - * The state will become either [PlaybackState.Playing] or [PlaybackState.Starved]. + * The behavior is undefined if this property is set during playback. + */ + public var itemToPlay: Int + + /** + * Starts playing the [itemToPlay]-th item. */ public fun start() /** * Stops ongoing playback. - * - * Makes the state become [PlaybackState.Idle] */ public fun stop() + /** + * Pauses playback. + */ public fun pause() + /** + * Resumes playback where it was paused if possible, or starts again otherwise. + */ public fun resume() /** @@ -140,3 +178,26 @@ public interface PlaybackEngine { */ public fun release() } + +@ExperimentalReadiumApi +public interface TtsVoice { + + @kotlinx.serialization.Serializable + @JvmInline + public value class Id(public val value: String) + + public val id: Id + + /** + * The languages supported by the voice. + */ + public val languages: Set +} + +@ExperimentalReadiumApi +@JvmInline +public value class TtsEngineProgress(public val value: IntRange) : PlaybackEngine.Progress + +@ExperimentalReadiumApi +@JvmInline +public value class AudioEngineProgress(public val value: Duration) : PlaybackEngine.Progress diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt index 2f378a3eed..6382c220a3 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt @@ -11,10 +11,9 @@ package org.readium.navigator.media.readaloud import kotlin.properties.Delegates import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.util.Error -internal class ReadAloudDataLoader( - private val segmentFactory: ReadAloudSegmentFactory, +internal class ReadAloudDataLoader( + private val segmentFactory: ReadAloudSegmentFactory, initialSettings: ReadAloudSettings, ) { data class ItemRef( @@ -32,7 +31,7 @@ internal class ReadAloudDataLoader( val nextNode = node.next() ?: return if (nextNode !in preloadedRefs) { - // loadSegmentForNode(nextNode, true) + loadSegmentForNode(nextNode) } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt index e98cad7cef..04594a5b48 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt @@ -9,103 +9,35 @@ package org.readium.navigator.media.readaloud import org.readium.navigator.common.CssSelector -import org.readium.navigator.common.CssSelectorLocation -import org.readium.navigator.common.ExportableLocation -import org.readium.navigator.common.GoLocation import org.readium.navigator.common.Location import org.readium.navigator.common.TextAnchor -import org.readium.navigator.common.TextAnchorLocation import org.readium.navigator.common.TextQuote -import org.readium.navigator.common.TextQuoteLocation +import org.readium.navigator.common.TimeOffset import org.readium.r2.shared.ExperimentalReadiumApi -import org.readium.r2.shared.publication.Locator -import org.readium.r2.shared.publication.Locator.Locations import org.readium.r2.shared.util.Url -import org.readium.r2.shared.util.mediatype.MediaType @ExperimentalReadiumApi -public data class ReadAloudGoLocation( - override val href: Url, - val cssSelector: CssSelector?, - val textAnchor: TextAnchor?, -) : GoLocation { - - public constructor(location: Location) : this( - href = location.href, - cssSelector = (location as? CssSelectorLocation)?.cssSelector, - textAnchor = (location as? TextAnchorLocation)?.textAnchor - ) -} - -@ExperimentalReadiumApi -public sealed interface ReadAloudLocation : ExportableLocation { - override val href: Url - public val textAnchor: TextAnchor? - public val cssSelector: CssSelector? -} +public sealed interface ReadAloudLocation : Location @ExperimentalReadiumApi -internal data class MediaOverlaysLocation( +public data class ReadAloudTextLocation( override val href: Url, - private val mediaType: MediaType?, - override val cssSelector: CssSelector?, -) : ReadAloudLocation, CssSelectorLocation { - - override fun toLocator(): Locator = - Locator( - href = href, - mediaType = mediaType ?: MediaType.XHTML, - locations = Locations() // TODO - ) - - override val textAnchor: TextAnchor? = null -} + public val textAnchor: TextAnchor?, + public val cssSelector: CssSelector?, +) : ReadAloudLocation @ExperimentalReadiumApi -internal data class TtsLocation( +public data class ReadAloudAudioLocation( override val href: Url, - private val mediaType: MediaType?, - override val cssSelector: CssSelector?, - override val textAnchor: TextAnchor, -) : ReadAloudLocation, CssSelectorLocation, TextAnchorLocation { - - override fun toLocator(): Locator = - Locator( - href = href, - mediaType = mediaType ?: MediaType.XHTML, - locations = Locations() // TODO - ) -} + public val timeOffset: TimeOffset?, +) : ReadAloudLocation @ExperimentalReadiumApi -public sealed interface UtteranceLocation : ExportableLocation { - override val href: Url +public sealed interface ReadAloudHighlightLocation : Location - public val textQuote: TextQuote? - public val cssSelector: CssSelector? -} - -internal data class MediaOverlaysUtteranceLocation( - override val href: Url, - private val mediaType: MediaType?, - override val cssSelector: CssSelector?, -) : UtteranceLocation, CssSelectorLocation { - - override val textQuote: TextQuote? = null - - override fun toLocator(): Locator { - TODO("Not yet implemented") - } -} - -internal data class TtsUtteranceLocation( +@ExperimentalReadiumApi +public data class ReadAloudTextHighlightLocation( override val href: Url, - private val mediaType: MediaType?, - override val cssSelector: CssSelector?, - override val textQuote: TextQuote, -) : UtteranceLocation, CssSelectorLocation, TextQuoteLocation { - - override fun toLocator(): Locator { - TODO("Not yet implemented") - } -} + public val textQuote: TextQuote?, + public val cssSelector: CssSelector?, +) : ReadAloudHighlightLocation diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt index fc964a5c9f..c6f89f1f03 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt @@ -11,18 +11,29 @@ package org.readium.navigator.media.readaloud import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.util.TemporalFragmentParser internal class ReadAloudNavigationHelper( var settings: ReadAloudSettings, ) { - fun ReadAloudNode.firstMatchingLocation(location: ReadAloudGoLocation): ReadAloudNode? = + fun ReadAloudNode.firstMatchingLocation(location: ReadAloudLocation): ReadAloudNode? = firstDescendantOrNull { it.matchLocation(location) } - private fun ReadAloudNode.matchLocation(location: ReadAloudGoLocation): Boolean = - refs.any { ref -> - ref.url.removeFragment() == location.href && - ref.url.fragment == location.cssSelector?.value?.removePrefix("#") + private fun ReadAloudNode.matchLocation(location: ReadAloudLocation): Boolean = + when (location) { + is ReadAloudAudioLocation -> { + refs.any { ref -> + val refFragment = ref.url.fragment?.let { TemporalFragmentParser.parse(it) } + ref.url.removeFragment() == location.href && refFragment?.start == location.timeOffset?.value + } + } + is ReadAloudTextLocation -> { + refs.any { ref -> + ref.url.removeFragment() == location.href && + ref.url.fragment == location.cssSelector?.value?.removePrefix("#") + } + } } fun ReadAloudNode.isSkippable(): Boolean = diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt index e4708a5756..f6873b91c7 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -11,44 +11,54 @@ package org.readium.navigator.media.readaloud import android.os.Handler import android.os.Looper import kotlin.properties.Delegates +import kotlin.time.Duration import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.runningFold +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.readium.navigator.common.CssSelector -import org.readium.navigator.common.TextQuote +import org.readium.navigator.common.TimeOffset import org.readium.navigator.media.common.MediaNavigator import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.extensions.addPrefix import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.guided.GuidedNavigationAudioRef import org.readium.r2.shared.guided.GuidedNavigationTextRef import org.readium.r2.shared.util.Error as BaseError import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TemporalFragmentParser import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.mediatype.MediaType @ExperimentalReadiumApi -public class ReadAloudNavigator private constructor( +public class ReadAloudNavigator private constructor( private val guidedNavigationTree: ReadAloudNode, private val resources: List, - private val audioEngineFactory: AudioEngineFactory, - private val ttsEngineFactory: TtsEngineFactory, + private val audioEngineFactory: AudioEngineFactory, + private val ttsEngineFactory: TtsEngineFactory, initialSettings: ReadAloudSettings, - initialLocation: ReadAloudGoLocation?, + initialLocation: ReadAloudLocation?, ) { public companion object { - internal suspend operator fun invoke( - initialLocation: ReadAloudGoLocation?, + internal suspend operator fun invoke( + initialLocation: ReadAloudLocation?, initialSettings: ReadAloudSettings, publication: ReadAloudPublication, - audioEngineFactory: AudioEngineFactory, - ttsEngineFactory: TtsEngineFactory, - ): ReadAloudNavigator { + audioEngineFactory: AudioEngineFactory, + ttsEngineFactory: TtsEngineFactory, + ): ReadAloudNavigator { val tree = withContext(Dispatchers.Default) { ReadAloudNode.fromGuidedNavigationObject(publication.guidedNavigationTree) } @@ -65,11 +75,30 @@ public class ReadAloudNavigator private constructor } public data class Playback( - val state: ReadAloudNavigator.PlaybackState, + val state: PlaybackState, val playWhenReady: Boolean, val node: ReadAloudNode, - val utteranceLocation: UtteranceLocation?, - ) + private val textItemMediaType: MediaType?, + ) { + val nodeHighlightLocation: ReadAloudHighlightLocation? get() { + val textRef = node.refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + ?: return null + + val href = textRef.url.removeFragment() + val cssSelector = textRef.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + return ReadAloudTextHighlightLocation( + href = href, + textQuote = null, + // mediaType = textItemMediaType, + cssSelector = cssSelector + ) + } + + val utteranceHighlightLocation: ReadAloudHighlightLocation? get() { + return null + } + } public sealed interface PlaybackState { @@ -87,14 +116,14 @@ public class ReadAloudNavigator private constructor override val cause: BaseError?, ) : BaseError { - public data class EngineError(override val cause: BaseError) : - Error("An error occurred in the playback engine.", cause) + public data class AudioEngineError(override val cause: BaseError) : + Error("An error occurred in the audio engine.", cause) - public data class ContentError(override val cause: ReadError) : - Error("An error occurred while trying to read publication content.", cause) + public data class TtsEngineError(override val cause: ReadError) : + Error("An error occurred in the TTS engine.", cause) } - private inner class PlaybackEngineListener : PlaybackEngine.Listener { + private inner class PlaybackEngineListener : PlaybackEngine.Listener { private val handler = Handler(Looper.getMainLooper()) @@ -114,9 +143,6 @@ public class ReadAloudNavigator private constructor } } - override fun onStopRequested() { - } - override fun onPlaybackCompleted() { handler.post { with(stateMachine) { @@ -125,9 +151,10 @@ public class ReadAloudNavigator private constructor } } - override fun onRangeStarted(range: IntRange) { - handler.post { - } + override fun onPlaybackError(error: BaseError) { + } + + override fun onPlaybackProgressed(progress: PlaybackEngine.Progress) { } } @@ -140,10 +167,10 @@ public class ReadAloudNavigator private constructor } } - public val voices: Set = + public val voices: Set = ttsEngineFactory.voices - private val segmentFactory = ReadAloudSegmentFactory( + private val segmentFactory = ReadAloudSegmentFactory( audioEngineFactory = { chunks: List -> audioEngineFactory.createPlaybackEngine( chunks = chunks, @@ -180,7 +207,7 @@ public class ReadAloudNavigator private constructor voice?.let { voice -> ttsEngineFactory.createPlaybackEngine( - voice = voice, + voiceId = voice.id, utterances = utterances, listener = PlaybackEngineListener() ) @@ -220,60 +247,18 @@ public class ReadAloudNavigator private constructor public val playback: StateFlow = stateMutable.mapStateIn(coroutineScope) { state -> + val textRef = state.node.refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + val textHref = textRef?.url?.removeFragment() + val textItemMediaType = textHref?.let { + resources.firstOrNull { item -> item.href == textHref }?.mediaType + } Playback( playWhenReady = state.playWhenReady, state = state.playbackState.toState(), node = state.node, - utteranceLocation = state.utteranceLocation + textItemMediaType = textItemMediaType ) } - - private val ReadAloudStateMachine.State.utteranceLocation: UtteranceLocation? get() = - when (segment) { - is AudioSegment -> { - val textref = segment.textRefs[nodeIndex] - val href = textref.removeFragment() - val cssSelector = textref.fragment - ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } - MediaOverlaysUtteranceLocation( - href = href, - mediaType = resources.first { item -> item.href == href }.mediaType, - cssSelector = cssSelector - ) - } - is TtsSegment<*> -> null - } - - private val ReadAloudNode.mediaOverlaysUtteranceLocation: UtteranceLocation? get() { - val textref = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } - ?: return null - val href = textref.url.removeFragment() - val cssSelector = textref.url.fragment - ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } - return MediaOverlaysUtteranceLocation( - href = href, - mediaType = resources.first { item -> item.href == href }.mediaType, - cssSelector = cssSelector - ) - } - - private val ReadAloudNode.ttsUtteranceLocation: UtteranceLocation? get() { - val textref = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } - ?: return null - val href = textref.url.removeFragment() - val cssSelector = textref.url.fragment - ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } - val text = text - ?: return null - return TtsUtteranceLocation( - href = href, - mediaType = resources.first { item -> item.href == href }.mediaType, - cssSelector = cssSelector, - // FIXME: prefix and suffix - textQuote = TextQuote(text.plain!!, prefix = "", suffix = "") - ) - } - private fun ReadAloudStateMachine.PlaybackState.toState(): PlaybackState = when (this) { is ReadAloudStateMachine.PlaybackState.Ready -> @@ -283,11 +268,69 @@ public class ReadAloudNavigator private constructor is ReadAloudStateMachine.PlaybackState.Ended -> PlaybackState.Ended is ReadAloudStateMachine.PlaybackState.Failure -> - PlaybackState.Failure(Error.EngineError(error)) + PlaybackState.Failure(Error.AudioEngineError(error)) } - init { - play() + public val locations: StateFlow> = + stateMutable.runningFold( + emptyList() + ) { prevLocations: List, state -> + val textLocation = state.node.toReadAloudTextLocation() + val audioLocation = state.node.toReadAloudAudioLocation() + buildList { + when (state.segment) { + is AudioSegment -> { + audioLocation?.let { add(it) } + textLocation?.let { add(it) } + } + is TtsSegment -> { + textLocation?.let { add(it) } + audioLocation?.let { add(it) } + } + } + + if (none { it is ReadAloudTextLocation }) { + prevLocations.firstOrNull { it is ReadAloudTextLocation } + ?.let { add(it) } + } + + if (none { it is ReadAloudAudioLocation }) { + prevLocations.firstOrNull { it is ReadAloudAudioLocation } + ?.let { add(it) } + } + } + }.stateInFirst( + scope = coroutineScope, + started = SharingStarted.Eagerly + ) + + private fun ReadAloudNode.toReadAloudTextLocation(): ReadAloudTextLocation? { + val textRef = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + ?: return null + val href = textRef.url.removeFragment() + val cssSelector = textRef.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + return ReadAloudTextLocation(href = href, cssSelector = cssSelector, textAnchor = null) + } + + private fun ReadAloudNode.toReadAloudAudioLocation(): ReadAloudAudioLocation? { + val audioRef = refs.firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } + ?: return null + + val href = audioRef.url.removeFragment() + val timeOffset = audioRef.url.fragment + ?.let { TemporalFragmentParser.parse(it) } + ?.start + ?.let { TimeOffset(it) } + return ReadAloudAudioLocation(href = href, timeOffset = timeOffset) + } + + private fun Flow.stateInFirst( + scope: CoroutineScope, + started: SharingStarted, + ): StateFlow { + val first = runBlocking { first() } + return stateIn(scope, started, first) } public fun play() { @@ -339,47 +382,31 @@ public class ReadAloudNavigator private constructor } } - public val location: StateFlow = - stateMutable.mapStateIn(coroutineScope) { state -> - val textref = state.node.refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } - checkNotNull(textref) - val href = textref.url.removeFragment() - val cssSelector = textref.url.fragment - ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } - MediaOverlaysLocation( - href = href, - mediaType = resources.first { item -> item.href == href }.mediaType, - cssSelector = cssSelector - ) - } - - public fun goTo(location: ReadAloudGoLocation) { + public fun goTo(location: ReadAloudLocation) { with(navigationHelper) { guidedNavigationTree.firstMatchingLocation(location) ?.let { go(it) } } } - public fun goTo(location: ReadAloudLocation) { - val goLocation = when (location) { - is MediaOverlaysLocation -> - ReadAloudGoLocation( - href = location.href, - cssSelector = location.cssSelector, - textAnchor = location.textAnchor - ) - is TtsLocation -> - throw IllegalStateException() - } - goTo(goLocation) - } - public fun goTo(url: Url) { - val location = ReadAloudGoLocation( + val location = url.fragment?.let { TemporalFragmentParser.parse(it) } + ?.let { timeInterval -> + val href = url.removeFragment() + val timeOffset = TimeOffset(timeInterval.start ?: Duration.ZERO) + ReadAloudAudioLocation(href = href, timeOffset = timeOffset) + } ?: ReadAloudTextLocation( href = url.removeFragment(), cssSelector = url.fragment?.let { CssSelector(it.addPrefix("#")) }, textAnchor = null ) + goTo(location) } + + public fun release() { + with(stateMachine) { + stateMutable.value = stateMutable.value.release() + } + } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt index 7248796f1c..7912a3915a 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -24,23 +24,23 @@ import org.readium.r2.shared.util.data.ReadError import org.readium.r2.shared.util.getOrElse @ExperimentalReadiumApi -public class ReadAloudNavigatorFactory private constructor( +public class ReadAloudNavigatorFactory private constructor( private val publicationMetadata: Metadata, private val guidedNavigationService: GuidedNavigationService, private val resources: List, - private val audioEngineFactory: AudioEngineFactory, - private val ttsEngineProvider: TtsEngineProvider, + private val audioEngineFactory: () -> AudioEngineFactory, + private val ttsEngineProvider: TtsEngineProvider, ) { public companion object { - public operator fun invoke( + public operator fun invoke( application: Application, publication: Publication, - audioEngineProvider: AudioEngineProvider, - ttsEngineProvider: TtsEngineProvider, + audioEngineProvider: AudioEngineProvider, + ttsEngineProvider: TtsEngineProvider, usePrerecordedVoicesWhenAvailable: Boolean = true, - ): ReadAloudNavigatorFactory? { + ): ReadAloudNavigatorFactory? { var guidedNavService: GuidedNavigationService? = null if (usePrerecordedVoicesWhenAvailable) { @@ -52,15 +52,13 @@ public class ReadAloudNavigatorFactory private cons if (guidedNavService == null) { publication.findService(ContentService::class) - ?.let { guidedNavService = TtsGuidedNavigationService(it) } + ?.let { guidedNavService = ContentGuidedNavigationService(it) } } if (guidedNavService == null) { return null } - val audioEngineFactory = audioEngineProvider.createEngineFactory(publication) - val resources = (publication.readingOrder + publication.resources).map { ReadAloudPublication.Item( href = it.url(), @@ -72,7 +70,7 @@ public class ReadAloudNavigatorFactory private cons publicationMetadata = publication.metadata, guidedNavigationService = guidedNavService, resources = resources, - audioEngineFactory = audioEngineFactory, + audioEngineFactory = { audioEngineProvider.createEngineFactory(publication) }, ttsEngineProvider = ttsEngineProvider ) } @@ -94,8 +92,8 @@ public class ReadAloudNavigatorFactory private cons public suspend fun createNavigator( initialSettings: ReadAloudSettings, - initialLocation: ReadAloudGoLocation? = null, - ): Try, Error> { + initialLocation: ReadAloudLocation? = null, + ): Try { val guidedDocs = buildList { val iterator = guidedNavigationService.iterator() while (iterator.hasNext()) { @@ -120,14 +118,12 @@ public class ReadAloudNavigatorFactory private cons resources = resources, ) - val ttsEngineFactory = ttsEngineProvider.createEngineFactory() - val navigator = ReadAloudNavigator( initialSettings = initialSettings, initialLocation = initialLocation, publication = navigatorPublication, - audioEngineFactory = audioEngineFactory, - ttsEngineFactory = ttsEngineFactory + audioEngineFactory = audioEngineFactory(), + ttsEngineFactory = ttsEngineProvider.createEngineFactory() ) return Try.success(navigator) diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt index 6763ff785f..0bba027cc6 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt @@ -12,7 +12,6 @@ import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.guided.GuidedNavigationAudioRef import org.readium.r2.shared.guided.GuidedNavigationText import org.readium.r2.shared.guided.GuidedNavigationTextRef -import org.readium.r2.shared.util.Error import org.readium.r2.shared.util.Language import org.readium.r2.shared.util.TemporalFragmentParser import org.readium.r2.shared.util.TimeInterval @@ -35,14 +34,14 @@ internal data class AudioSegment( override val emptyNodes: Set, ) : ReadAloudSegment -internal data class TtsSegment( +internal data class TtsSegment( override val player: PlaybackEngine, val items: List, override val nodes: List, override val emptyNodes: Set, ) : ReadAloudSegment -internal class ReadAloudSegmentFactory( +internal class ReadAloudSegmentFactory( private val audioEngineFactory: (List) -> PlaybackEngine?, private val ttsEngineFactory: (Language?, List) -> PlaybackEngine?, ) { @@ -92,7 +91,7 @@ internal class ReadAloudSegmentFactory( private fun createTtsSegmentFromNode( firstNode: ReadAloudNode, - ): TtsSegment? { + ): TtsSegment? { var nextNode: ReadAloudNode? = firstNode val segmentLanguage = firstNode.text?.language val textItems = mutableListOf() diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt index 82ffa0284c..bedc0b024c 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -12,8 +12,8 @@ import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.util.Error -internal class ReadAloudStateMachine( - private val dataLoader: ReadAloudDataLoader, +internal class ReadAloudStateMachine( + private val dataLoader: ReadAloudDataLoader, private val navigationHelper: ReadAloudNavigationHelper, ) { @@ -105,6 +105,11 @@ internal class ReadAloudStateMachine( return copy(playWhenReady = true, playerPaused = false) } + fun State.release(): State { + segment.player.release() + return this + } + fun State.onSettingsChanged( newSettings: ReadAloudSettings, ): State { diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt index 9c1ff8f53b..d6b5e2fc1c 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt @@ -50,7 +50,7 @@ public class SystemTtsEngineProvider( ) : TtsEngineProvider { override suspend fun createEngineFactory(): TtsEngineFactory = - tryCreateEngineFactory(maxConnectionRetries) ?: NullPlaybackEngineFactory() + tryCreateEngineFactory(maxConnectionRetries) ?: NullTtsEngineFactory() private suspend fun tryCreateEngineFactory(maxRetries: Int): SystemTtsEngineFactory? { suspend fun onFailure(): SystemTtsEngineFactory? = @@ -91,18 +91,18 @@ public class SystemTtsEngineFactory internal constructor( fullVoices.keys override fun createPlaybackEngine( - voice: SystemTtsEngine.Voice, + voiceId: TtsVoice.Id, utterances: List, - listener: PlaybackEngine.Listener, + listener: PlaybackEngine.Listener, ): PlaybackEngine { - val voice = fullVoices[voice] - checkNotNull(voice) + val voice = fullVoices.keys.firstOrNull { it.id == voiceId } + requireNotNull(voice) { "Voice id $voiceId doesn't match any voice in $voices." } return SystemTtsEngine( context = context, engine = textToSpeech, listener = listener, - systemVoice = voice, + systemVoice = fullVoices[voice]!!, utterances = utterances, maxConnectionRetries = maxConnectionRetries ) @@ -124,7 +124,7 @@ public class SystemTtsEngineFactory internal constructor( public class SystemTtsEngine internal constructor( private val context: Context, engine: TextToSpeech, - private val listener: PlaybackEngine.Listener, + private val listener: PlaybackEngine.Listener, private val systemVoice: AndroidVoice, private val utterances: List, private val maxConnectionRetries: Int, @@ -317,7 +317,6 @@ public class SystemTtsEngine internal constructor( } public override fun stop() { - listener.onStopRequested() doStop() } @@ -421,7 +420,7 @@ public class SystemTtsEngine internal constructor( } private class UtteranceListener( - private val listener: PlaybackEngine.Listener, + private val listener: PlaybackEngine.Listener, ) : UtteranceProgressListener() { override fun onStart(utteranceId: String) { } @@ -447,7 +446,7 @@ public class SystemTtsEngine internal constructor( } override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { - listener.onRangeStarted(start until end) + listener.onPlaybackProgressed(TtsEngineProgress(start until end)) } } } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt index 8fbc40514d..baac6d3c7b 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt @@ -6,6 +6,9 @@ package org.readium.navigator.media.readaloud.preferences +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole import org.readium.r2.shared.util.Language /** @@ -15,10 +18,14 @@ import org.readium.r2.shared.util.Language * * @see ReadAloudPreferences */ +@ExperimentalReadiumApi public data class ReadAloudDefaults( val language: Language? = null, val pitch: Double? = null, val speed: Double? = null, + val voices: Map? = null, + val escapableRoles: Set? = null, + val skippableRoles: Set? = null, val readContinuously: Boolean? = null, ) { init { diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt index 05437314b1..a982ab201f 100644 --- a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt @@ -43,23 +43,35 @@ internal class ReadAloudSettingsResolver( ?: defaults.language ?: Language(Locale.getDefault()) - val defaultSkippableRoles: Set = - setOf( - ASIDE, BIBLIOGRAPHY, ENDNOTES, FOOTNOTE, NOTEREF, PULLQUOTE, - LANDMARKS, LOA, LOI, LOT, LOV, PAGEBREAK, TOC - ) + val skippableRoles: Set = + preferences.skippableRoles + ?: defaults.skippableRoles + ?: setOf( + ASIDE, BIBLIOGRAPHY, ENDNOTES, FOOTNOTE, NOTEREF, PULLQUOTE, + LANDMARKS, LOA, LOI, LOT, LOV, PAGEBREAK, TOC + ) - val defaultEscapableRoles: Set = - setOf(ASIDE, FIGURE, LIST, LIST_ITEM, TABLE, ROW, CELL) + val escapableRoles: Set = + preferences.escapableRoles + ?: defaults.escapableRoles + ?: setOf(ASIDE, FIGURE, LIST, LIST_ITEM, TABLE, ROW, CELL) + + val languagesWithPreferredVoice = + preferences.voices.orEmpty().keys.map { it.removeRegion() } + + val filteredDefaultVoices = defaults.voices.orEmpty() + .filter { it.key.removeRegion() !in languagesWithPreferredVoice } + + val voices = filteredDefaultVoices + preferences.voices.orEmpty() return ReadAloudSettings( language = language, - voices = preferences.voices ?: emptyMap(), + voices = voices, pitch = preferences.pitch ?: defaults.pitch ?: 1.0, speed = preferences.speed ?: defaults.speed ?: 1.0, overrideContentLanguage = preferences.language != null, - escapableRoles = preferences.escapableRoles ?: defaultEscapableRoles, - skippableRoles = preferences.skippableRoles ?: defaultSkippableRoles, + escapableRoles = escapableRoles, + skippableRoles = skippableRoles, readContinuously = preferences.readContinuously ?: defaults.readContinuously ?: true ) }