diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml index 131e44d798..c224ad564b 100644 --- a/.idea/kotlinc.xml +++ b/.idea/kotlinc.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/demos/navigator/build.gradle.kts b/demos/navigator/build.gradle.kts index 8ed8801cb2..00dffca861 100644 --- a/demos/navigator/build.gradle.kts +++ b/demos/navigator/build.gradle.kts @@ -65,7 +65,8 @@ dependencies { implementation(project(":readium:readium-navigator")) implementation(project(":readium:navigators:web:readium-navigator-web-reflowable")) implementation(project(":readium:navigators:web:readium-navigator-web-fixedlayout")) - implementation(project(":readium:adapters:pdfium")) + implementation(project(":readium:navigators:media:readium-navigator-media-readaloud")) + implementation(project(":readium:adapters:exoplayer:readium-adapter-exoplayer-readaloud")) coreLibraryDesugaring(libs.desugar.jdk.libs) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt index 5c7877af17..3b0b8042fd 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoUi.kt @@ -21,9 +21,12 @@ import androidx.compose.runtime.State import androidx.compose.runtime.collectAsState import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.platform.LocalTextToolbar import androidx.core.view.WindowInsetsControllerCompat -import org.readium.demo.navigator.reader.Rendition +import org.readium.demo.navigator.reader.ReadAloudReaderState +import org.readium.demo.navigator.reader.ReadAloudRendition +import org.readium.demo.navigator.reader.SelectNavigatorMenu +import org.readium.demo.navigator.reader.VisualReaderState +import org.readium.demo.navigator.reader.VisualRendition import org.readium.demo.navigator.util.Fullscreenable @Composable @@ -42,7 +45,6 @@ fun Scaffold( ) { content.invoke() - LocalTextToolbar SnackbarHost( modifier = Modifier .align(Alignment.BottomCenter) @@ -67,7 +69,11 @@ fun MainContent( DemoViewModel.State.BookSelection -> true is DemoViewModel.State.Error -> false DemoViewModel.State.Loading -> true - is DemoViewModel.State.Reader -> true + is DemoViewModel.State.Reader -> when (viewmodelState.readerState) { + is VisualReaderState<*, *, *, *> -> true + is ReadAloudReaderState -> false + } + is DemoViewModel.State.NavigatorSelection -> true } } @@ -79,6 +85,10 @@ fun MainContent( } } + is DemoViewModel.State.NavigatorSelection -> { + SelectNavigatorMenu(viewmodelState.viewModel) + } + is DemoViewModel.State.Error -> { Placeholder() LaunchedEffect(viewmodelState.error) { @@ -100,10 +110,19 @@ fun MainContent( viewmodel.onBookClosed() } - Rendition( - readerState = viewmodelState.readerState, - fullScreenState = fullscreenState - ) + when (viewmodelState.readerState) { + is ReadAloudReaderState -> { + ReadAloudRendition( + readerState = viewmodelState.readerState + ) + } + is VisualReaderState<*, *, *, *> -> { + VisualRendition( + readerState = viewmodelState.readerState, + fullScreenState = fullscreenState + ) + } + } } } } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt index 714b246b92..e22f0350c6 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/DemoViewModel.kt @@ -4,19 +4,39 @@ * available in the top-level LICENSE file of the project. */ +@file:OptIn(ExperimentalReadiumApi::class) + package org.readium.demo.navigator import android.app.Application import androidx.lifecycle.AndroidViewModel +import androidx.lifecycle.application import androidx.lifecycle.viewModelScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch +import org.readium.adapter.exoplayer.readaloud.ExoPlayerEngineProvider import org.readium.demo.navigator.reader.ReaderOpener import org.readium.demo.navigator.reader.ReaderState +import org.readium.demo.navigator.reader.SelectNavigatorItem +import org.readium.demo.navigator.reader.SelectNavigatorViewModel +import org.readium.demo.navigator.reader.fixedConfig +import org.readium.demo.navigator.reader.reflowableConfig +import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.media.readaloud.SystemTtsEngineProvider +import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory +import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl +import org.readium.r2.shared.util.DebugError +import org.readium.r2.shared.util.asset.AssetRetriever +import org.readium.r2.shared.util.getOrElse +import org.readium.r2.shared.util.http.DefaultHttpClient import org.readium.r2.shared.util.toDebugDescription +import org.readium.r2.streamer.PublicationOpener +import org.readium.r2.streamer.parser.DefaultPublicationParser import timber.log.Timber class DemoViewModel( @@ -28,6 +48,10 @@ class DemoViewModel( data object BookSelection : State + data class NavigatorSelection( + val viewModel: SelectNavigatorViewModel, + ) : State + data object Loading : State @@ -36,7 +60,7 @@ class DemoViewModel( ) : State data class Reader( - val readerState: ReaderState<*, *, *, *>, + val readerState: ReaderState, ) : State } @@ -44,9 +68,24 @@ class DemoViewModel( Timber.plant(Timber.DebugTree()) } + private val httpClient = + DefaultHttpClient() + + private val assetRetriever = + AssetRetriever(application.contentResolver, httpClient) + + private val publicationParser = + DefaultPublicationParser(application, httpClient, assetRetriever, null) + + private val publicationOpener = + PublicationOpener(publicationParser) + private val readerOpener = ReaderOpener(application) + private val audioEngineProvider = + ExoPlayerEngineProvider(application) + private val stateMutable: MutableStateFlow = MutableStateFlow(State.BookSelection) @@ -56,7 +95,83 @@ class DemoViewModel( stateMutable.value = State.Loading viewModelScope.launch { - readerOpener.open(url) + val asset = assetRetriever.retrieve(url) + .getOrElse { + Timber.d(it.toDebugDescription()) + stateMutable.value = State.Error(it) + return@launch + } + + val publication = publicationOpener.open(asset, allowUserInteraction = false) + .getOrElse { + asset.close() + Timber.d(it.toDebugDescription()) + stateMutable.value = State.Error(it) + return@launch + } + + val reflowableFactory = + ReflowableWebRenditionFactory( + application = application, + publication = publication, + configuration = reflowableConfig + )?.let { SelectNavigatorItem.ReflowableWeb(it) } + + val fixedFactory = + FixedWebRenditionFactory( + application = application, + publication = publication, + configuration = fixedConfig + )?.let { SelectNavigatorItem.FixedWeb(it) } + + val ttsEngineProvider = + SystemTtsEngineProvider(application) + + val readAloudFactory = ReadAloudNavigatorFactory.invoke( + application = application, + publication = publication, + audioEngineProvider = audioEngineProvider, + ttsEngineProvider = ttsEngineProvider + )?.let { SelectNavigatorItem.ReadAloud(it) } + + val factories = listOfNotNull( + reflowableFactory, + fixedFactory, + readAloudFactory + ) + + when (factories.size) { + 0 -> { + val error = DebugError("Publication not supported") + Timber.d(error.toDebugDescription()) + stateMutable.value = State.Error(error) + } + 1 -> { + onNavigatorSelected(url, publication, factories.first()) + } + else -> { + val selectionViewModel = SelectNavigatorViewModel( + items = factories, + onItemSelected = { onNavigatorSelected(url, publication, it) }, + onMenuDismissed = { stateMutable.value = State.BookSelection } + ) + + stateMutable.value = + State.NavigatorSelection(selectionViewModel) + } + } + } + } + + fun onNavigatorSelected( + url: AbsoluteUrl, + publication: Publication, + navigatorItem: SelectNavigatorItem, + ) { + stateMutable.value = State.Loading + + viewModelScope.launch { + readerOpener.open(url, publication, navigatorItem) .onFailure { Timber.d(it.toDebugDescription()) stateMutable.value = State.Error(it) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt index 25070d40b4..21ff0adc18 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/PreferencesManager.kt @@ -9,6 +9,8 @@ package org.readium.demo.navigator.preferences import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow import org.readium.navigator.common.Preferences import org.readium.r2.shared.ExperimentalReadiumApi @@ -21,6 +23,9 @@ class PreferencesManager

>( private val preferencesMutable: MutableStateFlow

= MutableStateFlow(initialPreferences) + val preferences: StateFlow

= + preferencesMutable.asStateFlow() + fun setPreferences(preferences: P) { preferencesMutable.value = preferences } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt new file mode 100644 index 0000000000..c7ab367e07 --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/ReadAloudPreferencesEditor.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.demo.navigator.preferences + +import kotlinx.coroutines.flow.StateFlow +import org.readium.navigator.common.PreferencesEditor +import org.readium.navigator.media.readaloud.SystemTtsEngine +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.navigator.preferences.EnumPreference +import org.readium.r2.navigator.preferences.Preference +import org.readium.r2.navigator.preferences.RangePreference +import org.readium.r2.navigator.preferences.map +import org.readium.r2.navigator.preferences.withSupportedValues +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Language + +@OptIn(ExperimentalReadiumApi::class) +class ReadAloudPreferencesEditor( + private val editor: org.readium.navigator.media.readaloud.preferences.ReadAloudPreferencesEditor, + private val availableVoices: Set, +) : PreferencesEditor { + + override val preferences: ReadAloudPreferences + get() = editor.preferences + + val preferencesState: StateFlow = + editor.preferencesState + + override val settings: ReadAloudSettings + get() = editor.settings + + override fun clear() { + editor.clear() + } + + val language: Preference = + editor.language + + /** + * [ReadAloudPreferencesEditor] supports choosing voices for any language or region. + * For this test app, we've chosen to present to the user only the voice for the + * TTS default language and to ignore regions. + */ + val voice: EnumPreference = run { + val currentLanguage = language.effectiveValue?.removeRegion() + + editor.voices.map( + from = { voiceIds -> + currentLanguage + ?.let { voiceIds[it] } + ?.let { voiceId -> availableVoices.firstOrNull { it.id == voiceId } } + }, + to = { voice -> + currentLanguage + ?.let { editor.voices.value.orEmpty().update(it, voice?.id) } + ?: editor.voices.value.orEmpty() + } + ).withSupportedValues( + availableVoices + .filter { voice -> currentLanguage in voice.languages.map { it.removeRegion() } } + ) + } + val pitch: RangePreference = + editor.pitch + + val speed: RangePreference = + editor.speed + + val readContinuously: Preference = + editor.readContinuously + + private fun Map.update(key: K, value: V?): Map = + buildMap { + putAll(this@update) + if (value == null) { + remove(key) + } else { + put(key, value) + } + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt index 6d458a9072..e332e0dedc 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/preferences/UserPreferences.kt @@ -28,6 +28,7 @@ import org.readium.demo.navigator.reader.LITERATA import org.readium.navigator.common.Preferences import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings +import org.readium.navigator.media.readaloud.SystemTtsEngine import org.readium.navigator.web.fixedlayout.preferences.FixedWebPreferencesEditor import org.readium.navigator.web.reflowable.preferences.ReflowableWebPreferencesEditor import org.readium.r2.navigator.preferences.Axis @@ -122,6 +123,16 @@ fun

, S : Settings, E : PreferencesEditor> UserPreferen visitedColor = editor.visitedColor, wordSpacing = editor.wordSpacing ) + + is ReadAloudPreferencesEditor -> { + MediaUserPreferences( + language = editor.language, + voice = editor.voice, + speed = editor.speed, + pitch = editor.pitch, + readContinuously = editor.readContinuously + ) + } } } } @@ -486,6 +497,51 @@ private fun ReflowableUserPreferences( } } +@Composable +private fun MediaUserPreferences( + language: Preference? = null, + voice: EnumPreference? = null, + speed: RangePreference? = null, + pitch: RangePreference? = null, + readContinuously: Preference? = null, +) { + Column { + if (speed != null) { + StepperItem( + title = "Speed", + preference = speed, + ) + } + + if (pitch != null) { + StepperItem( + title = "Pitch", + preference = pitch, + ) + } + if (language != null) { + LanguageItem( + preference = language + ) + } + + if (voice != null) { + MenuItem( + title = "Voice", + preference = voice, + formatValue = { it?.name ?: "Default" }, + ) + } + + if (readContinuously != null) { + SwitchItem( + title = "Read Continuously", + preference = readContinuously + ) + } + } +} + @Composable private fun Divider() { HorizontalDivider(modifier = Modifier.padding(vertical = 16.dp)) diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt new file mode 100644 index 0000000000..ca007662e5 --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReadAloudRendition.kt @@ -0,0 +1,190 @@ +/* + * Copyright 2024 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class, ExperimentalMaterial3Api::class) + +package org.readium.demo.navigator.reader + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.ArrowOutward +import androidx.compose.material.icons.filled.Pause +import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.SkipNext +import androidx.compose.material.icons.filled.SkipPrevious +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.ModalBottomSheet +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.material3.rememberModalBottomSheetState +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp +import androidx.compose.ui.zIndex +import org.readium.demo.navigator.preferences.UserPreferences +import org.readium.navigator.media.readaloud.ReadAloudTextHighlightLocation +import org.readium.r2.shared.ExperimentalReadiumApi + +@Composable +fun ReadAloudRendition( + readerState: ReadAloudReaderState, +) { + val showPreferences = remember { mutableStateOf(false) } + val preferencesSheetState = rememberModalBottomSheetState() + + if (showPreferences.value) { + ModalBottomSheet( + sheetState = preferencesSheetState, + onDismissRequest = { + showPreferences.value = false + } + ) { + val preferencesEditor = readerState.preferencesEditor.collectAsState() + + UserPreferences( + editor = preferencesEditor.value, + title = "Preferences" + ) + } + } + + val showOutline = rememberSaveable { mutableStateOf(false) } + + if (showOutline.value) { + Outline( + modifier = Modifier + .zIndex(1f) + .fillMaxSize(), + publication = readerState.publication, + onBackActivated = { + showOutline.value = false + }, + onTocItemActivated = { + readerState.navigator.goTo(it) + showOutline.value = false + } + ) + } + + Scaffold( + modifier = Modifier.fillMaxSize(), + topBar = { + RenditionTopBar( + modifier = Modifier.zIndex(10f), + visible = true, + onPreferencesActivated = { showPreferences.value = !showPreferences.value }, + onOutlineActivated = { showOutline.value = !showOutline.value } + ) + } + ) { contentPadding -> + Column( + modifier = Modifier + .padding(contentPadding) + .fillMaxSize(), + ) { + Column( + modifier = Modifier.weight(1f), + verticalArrangement = Arrangement.spacedBy(20.dp, alignment = Alignment.Top) + ) { + val playbackState = readerState.navigator.playback.collectAsState() + + Text("Playback State: ${playbackState.value.state}") + + Text("Play When Ready: ${playbackState.value.playWhenReady}") + + when (val highlightLocation = playbackState.value.nodeHighlightLocation) { + is ReadAloudTextHighlightLocation -> { + Text("Node Highlight Href: ${highlightLocation.href}") + + Text("Node Highlight Css Selector ${highlightLocation.cssSelector?.value}") + + Text("Node Highlight Text ${highlightLocation.textQuote?.text}") + } + null -> {} + } + } + + Toolbar(readerState) + } + } +} + +@Composable +private fun Toolbar( + readerState: ReadAloudReaderState, +) { + val playbackState = readerState.navigator.playback.collectAsState() + + Row( + modifier = Modifier + .fillMaxWidth(), + horizontalArrangement = Arrangement.Center, + verticalAlignment = Alignment.Bottom, + ) { + IconButton( + onClick = { readerState.navigator.skipToPrevious(force = true) } + ) { + Icon( + imageVector = Icons.Default.SkipPrevious, + contentDescription = "Skip to previous" + ) + } + + if (playbackState.value.playWhenReady) { + IconButton( + onClick = { + readerState.navigator.pause() + } + ) { + Icon( + imageVector = Icons.Default.Pause, + contentDescription = "Pause" + ) + } + } else { + IconButton( + onClick = { + readerState.navigator.play() + } + ) { + Icon( + imageVector = Icons.Default.PlayArrow, + contentDescription = "Play" + ) + } + } + + IconButton( + onClick = { readerState.navigator.skipToNext(force = true) } + ) { + Icon( + imageVector = Icons.Default.SkipNext, + contentDescription = "Skip to next" + ) + } + + IconButton( + onClick = { readerState.navigator.escape(force = true) } + ) { + Icon( + imageVector = Icons.Default.ArrowOutward, + contentDescription = "Escape" + ) + } + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt index 3e11e82818..c66cede80b 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderOpener.kt @@ -4,7 +4,11 @@ * available in the top-level LICENSE file of the project. */ -@file:OptIn(ExperimentalReadiumApi::class) +@file:OptIn( + ExperimentalReadiumApi::class, + InternalReadiumApi::class, + ExperimentalCoroutinesApi::class +) package org.readium.demo.navigator.reader @@ -12,7 +16,9 @@ import android.app.Application import androidx.compose.runtime.snapshotFlow import kotlinx.collections.immutable.plus import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import org.readium.demo.navigator.decorations.FixedWebHighlightsManager @@ -21,11 +27,15 @@ import org.readium.demo.navigator.decorations.ReflowableWebHighlightsManager import org.readium.demo.navigator.decorations.pageNumberDecorations import org.readium.demo.navigator.persistence.LocatorRepository import org.readium.demo.navigator.preferences.PreferencesManager +import org.readium.demo.navigator.preferences.ReadAloudPreferencesEditor import org.readium.navigator.common.DecorationController import org.readium.navigator.common.DecorationLocation import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.Settings import org.readium.navigator.common.SettingsController +import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.media.readaloud.SystemTtsEngine +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences import org.readium.navigator.web.fixedlayout.FixedWebGoLocation import org.readium.navigator.web.fixedlayout.FixedWebLocation import org.readium.navigator.web.fixedlayout.FixedWebRenditionController @@ -39,73 +49,61 @@ import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory import org.readium.navigator.web.reflowable.ReflowableWebSelectionLocation import org.readium.navigator.web.reflowable.preferences.ReflowableWebPreferences import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn import org.readium.r2.shared.publication.Locator import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl -import org.readium.r2.shared.util.DebugError import org.readium.r2.shared.util.Error import org.readium.r2.shared.util.Try -import org.readium.r2.shared.util.asset.AssetRetriever import org.readium.r2.shared.util.getOrElse -import org.readium.r2.shared.util.http.DefaultHttpClient -import org.readium.r2.streamer.PublicationOpener -import org.readium.r2.streamer.parser.DefaultPublicationParser class ReaderOpener( private val application: Application, ) { - - private val httpClient = - DefaultHttpClient() - - private val assetRetriever = - AssetRetriever(application.contentResolver, httpClient) - - private val publicationParser = - DefaultPublicationParser(application, httpClient, assetRetriever, null) - - private val publicationOpener = - PublicationOpener(publicationParser) - - suspend fun open(url: AbsoluteUrl): Try, Error> { - val asset = assetRetriever.retrieve(url) - .getOrElse { return Try.failure(it) } - - val publication = publicationOpener.open(asset, allowUserInteraction = false) - .getOrElse { - asset.close() - return Try.failure(it) - } - + suspend fun open( + url: AbsoluteUrl, + publication: Publication, + selectedNavigator: SelectNavigatorItem, + ): Try { val initialLocator = LocatorRepository.getLocator(url) - val readerState = ( - createFixedWebReader(url, publication, initialLocator) - ?: createReflowableWebReader(url, publication, initialLocator) - ) - .or { Try.failure(DebugError("Publication not supported")) } - .getOrElse { error -> - publication.close() - return Try.failure(error) - } + val readerState = when (selectedNavigator) { + is SelectNavigatorItem.ReflowableWeb -> + createReflowableWebReader( + url, + publication, + selectedNavigator.factory, + initialLocator + ) + is SelectNavigatorItem.FixedWeb -> + createFixedWebReader( + url, + publication, + selectedNavigator.factory, + initialLocator + ) + is SelectNavigatorItem.ReadAloud -> + createReadAloudReader( + url, + publication, + selectedNavigator.factory, + initialLocator + ) + }.getOrElse { error -> + publication.close() + return Try.failure(error) + } return Try.success(readerState) } - private fun Try?.or(onNull: () -> Try): Try = - this ?: onNull() - private suspend fun createReflowableWebReader( url: AbsoluteUrl, publication: Publication, + navigatorFactory: ReflowableWebRenditionFactory, initialLocator: Locator?, - ): Try, Error>? { - val navigatorFactory = ReflowableWebRenditionFactory( - application = application, - publication = publication, - configuration = reflowableConfig - ) ?: return null - + ): Try, Error> { val initialLocation = initialLocator?.let { ReflowableWebGoLocation(it) } val coroutineScope = MainScope() @@ -140,7 +138,7 @@ class ReaderOpener( val actionModeFactory = SelectionActionModeFactory(highlightsManager) - val readerState = ReaderState( + val readerState = VisualReaderState( url = url, coroutineScope = coroutineScope, publication = publication, @@ -157,14 +155,9 @@ class ReaderOpener( private suspend fun createFixedWebReader( url: AbsoluteUrl, publication: Publication, + navigatorFactory: FixedWebRenditionFactory, initialLocator: Locator?, - ): Try, Error>? { - val navigatorFactory = FixedWebRenditionFactory( - application = application, - publication = publication, - configuration = fixedConfig - ) ?: return null - + ): Try, Error> { val initialLocation = initialLocator?.let { FixedWebGoLocation(it) } val coroutineScope = MainScope() @@ -195,7 +188,7 @@ class ReaderOpener( val actionModeFactory = SelectionActionModeFactory(highlightsManager) - val readerState = ReaderState( + val readerState = VisualReaderState( url = url, coroutineScope = coroutineScope, publication = publication, @@ -209,6 +202,50 @@ class ReaderOpener( return Try.success(readerState) } + private suspend fun createReadAloudReader( + url: AbsoluteUrl, + publication: Publication, + navigatorFactory: ReadAloudNavigatorFactory, + initialLocator: Locator?, + ): Try { + val coroutineScope = MainScope() + + val initialPreferences = ReadAloudPreferences() + + val preferencesManager = PreferencesManager(initialPreferences) + + val initialSettings = navigatorFactory + .createPreferencesEditor(preferencesManager.preferences.value) + .settings + + val navigator = navigatorFactory.createNavigator(initialSettings) + .getOrElse { return Try.failure(it) } + + val preferencesEditor = preferencesManager.preferences.mapStateIn(coroutineScope) { + @Suppress("UNCHECKED_CAST") + ReadAloudPreferencesEditor( + editor = navigatorFactory.createPreferencesEditor(it), + availableVoices = navigator.voices as Set + ) + } + + preferencesEditor + .flatMapLatest { it.preferencesState } + .onEach { + navigator.settings = preferencesEditor.value.settings + preferencesManager.setPreferences(it) + }.launchIn(coroutineScope) + + val readerState = ReadAloudReaderState( + url = url, + coroutineScope = coroutineScope, + publication = publication, + navigator = navigator, + preferencesEditor = preferencesEditor + ) + return Try.success(readerState) + } + private fun applySettings( coroutineScope: CoroutineScope, settingsController: SettingsController, diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt index ba6765f0ca..3155c02a00 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/ReaderState.kt @@ -10,7 +10,9 @@ package org.readium.demo.navigator.reader import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.StateFlow import org.readium.demo.navigator.decorations.HighlightsManager +import org.readium.demo.navigator.preferences.ReadAloudPreferencesEditor import org.readium.navigator.common.ExportableLocation import org.readium.navigator.common.GoLocation import org.readium.navigator.common.NavigationController @@ -18,11 +20,17 @@ import org.readium.navigator.common.PreferencesEditor import org.readium.navigator.common.RenditionState import org.readium.navigator.common.SelectionController import org.readium.navigator.common.SelectionLocation +import org.readium.navigator.media.readaloud.ReadAloudNavigator import org.readium.r2.shared.ExperimentalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.AbsoluteUrl -data class ReaderState( +sealed interface ReaderState { + + fun close() +} + +data class VisualReaderState( val url: AbsoluteUrl, val coroutineScope: CoroutineScope, val publication: Publication, @@ -31,9 +39,24 @@ data class ReaderState, val onControllerAvailable: (C) -> Unit, val actionModeFactory: SelectionActionModeFactory, -) where C : NavigationController, C : SelectionController { +) : ReaderState where C : NavigationController, C : SelectionController { + + override fun close() { + coroutineScope.cancel() + publication.close() + } +} + +data class ReadAloudReaderState( + val url: AbsoluteUrl, + val coroutineScope: CoroutineScope, + val publication: Publication, + val navigator: ReadAloudNavigator, + val preferencesEditor: StateFlow, +) : ReaderState { - fun close() { + override fun close() { + navigator.release() coroutineScope.cancel() publication.close() } diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt new file mode 100644 index 0000000000..772394888e --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/RenditionTopBar.kt @@ -0,0 +1,52 @@ +@file:OptIn(ExperimentalMaterial3Api::class) + +package org.readium.demo.navigator.reader + +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.TopAppBar +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.res.painterResource +import org.readium.demo.navigator.R + +@Composable +fun RenditionTopBar( + modifier: Modifier, + visible: Boolean, + onPreferencesActivated: () -> Unit, + onOutlineActivated: () -> Unit, +) { + AnimatedVisibility( + modifier = modifier, + visible = visible, + enter = fadeIn(), + exit = fadeOut() + ) { + TopAppBar( + title = { }, + actions = { + IconButton( + onClick = onPreferencesActivated + ) { + Icon( + painterResource(R.drawable.ic_preferences_24), + contentDescription = "Preferences", + ) + } + IconButton( + onClick = onOutlineActivated + ) { + Icon( + painterResource(R.drawable.ic_outline_24), + contentDescription = "Outline" + ) + } + } + ) + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt new file mode 100644 index 0000000000..edd5cf295e --- /dev/null +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/SelectNavigatorMenu.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.demo.navigator.reader + +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.window.PopupProperties +import org.readium.navigator.media.readaloud.ReadAloudNavigatorFactory +import org.readium.navigator.web.fixedlayout.FixedWebRenditionFactory +import org.readium.navigator.web.reflowable.ReflowableWebRenditionFactory +import org.readium.r2.shared.ExperimentalReadiumApi + +class SelectNavigatorViewModel( + val items: List, + val onItemSelected: (SelectNavigatorItem) -> Unit, + val onMenuDismissed: () -> Unit, +) { + + fun select(item: SelectNavigatorItem) { + onItemSelected(item) + } + + fun cancel() { + onMenuDismissed() + } +} + +sealed class SelectNavigatorItem( + val name: String, +) { + + abstract val factory: Any + + data class ReflowableWeb( + override val factory: ReflowableWebRenditionFactory, + ) : SelectNavigatorItem("Reflowable Web Rendition") + + data class FixedWeb( + override val factory: FixedWebRenditionFactory, + ) : SelectNavigatorItem("Fixed Web Rendition") + + data class ReadAloud( + override val factory: ReadAloudNavigatorFactory, + ) : SelectNavigatorItem("Read Aloud Navigator") +} + +@Composable +fun SelectNavigatorMenu( + viewModel: SelectNavigatorViewModel, +) { + SelectNavigatorMenu( + popupProperties = PopupProperties(), + items = viewModel.items, + onItemSelected = { viewModel.select(it) }, + onDismissRequest = viewModel::cancel + ) +} + +@Composable +private fun SelectNavigatorMenu( + popupProperties: PopupProperties, + items: List, + onItemSelected: (SelectNavigatorItem) -> Unit, + onDismissRequest: () -> Unit, +) { + DropdownMenu( + expanded = true, + properties = popupProperties, + onDismissRequest = onDismissRequest + ) { + for (item in items) { + DropdownMenuItem( + text = { Text(item.name) }, + onClick = { onItemSelected(item) } + ) + } + } +} diff --git a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/Rendition.kt b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt similarity index 87% rename from demos/navigator/src/main/java/org/readium/demo/navigator/reader/Rendition.kt rename to demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt index b49a1aae96..dd2622b3f2 100644 --- a/demos/navigator/src/main/java/org/readium/demo/navigator/reader/Rendition.kt +++ b/demos/navigator/src/main/java/org/readium/demo/navigator/reader/VisualRendition.kt @@ -9,16 +9,10 @@ package org.readium.demo.navigator.reader import androidx.activity.compose.BackHandler -import androidx.compose.animation.AnimatedVisibility -import androidx.compose.animation.fadeIn -import androidx.compose.animation.fadeOut import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.ModalBottomSheet -import androidx.compose.material3.TopAppBar import androidx.compose.material3.rememberModalBottomSheetState import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect @@ -30,12 +24,10 @@ import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.res.painterResource import androidx.compose.ui.zIndex import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.launch -import org.readium.demo.navigator.R import org.readium.demo.navigator.decorations.DecorationStyleAnnotationMark import org.readium.demo.navigator.decorations.EditAnnotationDialog import org.readium.demo.navigator.decorations.EditAnnotationViewModel @@ -66,8 +58,8 @@ import org.readium.r2.shared.util.toUri @OptIn(ExperimentalMaterial3Api::class) @Composable -fun Rendition( - readerState: ReaderState, +fun VisualRendition( + readerState: VisualReaderState, fullScreenState: MutableState, ) where C : NavigationController, C : SelectionController { val coroutineScope = rememberCoroutineScope() @@ -116,7 +108,7 @@ fun Rendition } Box { - TopBar( + RenditionTopBar( modifier = Modifier.zIndex(10f), visible = !fullScreenState.value, onPreferencesActivated = { showPreferences.value = !showPreferences.value }, @@ -270,41 +262,3 @@ fun Rendition } } } - -@OptIn(ExperimentalMaterial3Api::class) -@Composable -private fun TopBar( - modifier: Modifier, - visible: Boolean, - onPreferencesActivated: () -> Unit, - onOutlineActivated: () -> Unit, -) { - AnimatedVisibility( - modifier = modifier, - visible = visible, - enter = fadeIn(), - exit = fadeOut() - ) { - TopAppBar( - title = { }, - actions = { - IconButton( - onClick = onPreferencesActivated - ) { - Icon( - painterResource(R.drawable.ic_preferences_24), - contentDescription = "Preferences", - ) - } - IconButton( - onClick = onOutlineActivated - ) { - Icon( - painterResource(R.drawable.ic_outline_24), - contentDescription = "Outline" - ) - } - } - ) - } -} diff --git a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt index de852d41bd..1dd6bbd8cf 100644 --- a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt +++ b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/DefaultExoPlayerCacheProvider.kt @@ -1,3 +1,5 @@ +@file:kotlin.OptIn(InternalReadiumApi::class) + package org.readium.adapter.exoplayer.audio import androidx.annotation.OptIn @@ -7,6 +9,7 @@ import androidx.media3.datasource.cache.Cache import androidx.media3.datasource.cache.CacheDataSink import androidx.media3.datasource.cache.CacheDataSource import org.readium.r2.shared.DelicateReadiumApi +import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.Url import org.readium.r2.shared.util.toUrl diff --git a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt index bc7a5c81ba..09a6a98268 100644 --- a/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt +++ b/readium/adapters/exoplayer/audio/src/main/java/org/readium/adapter/exoplayer/audio/ExoPlayerDataSource.kt @@ -14,6 +14,7 @@ import androidx.media3.datasource.DataSource import androidx.media3.datasource.DataSpec import androidx.media3.datasource.TransferListener import kotlinx.coroutines.runBlocking +import org.readium.r2.shared.InternalReadiumApi import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.util.DebugError import org.readium.r2.shared.util.data.ReadError @@ -28,11 +29,12 @@ import timber.log.Timber * An ExoPlayer's [DataSource] which retrieves resources from a [Publication]. */ @androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) -internal class ExoPlayerDataSource internal constructor( +@InternalReadiumApi +public class ExoPlayerDataSource internal constructor( private val publication: Publication, ) : BaseDataSource(/* isNetwork = */ true) { - class Factory( + public class Factory( private val publication: Publication, private val transferListener: TransferListener? = null, ) : DataSource.Factory { diff --git a/readium/adapters/exoplayer/readaloud/build.gradle.kts b/readium/adapters/exoplayer/readaloud/build.gradle.kts new file mode 100644 index 0000000000..5cdeca3b20 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/build.gradle.kts @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +plugins { + id("readium.library-conventions") + alias(libs.plugins.kotlin.serialization) +} + +android { + namespace = "org.readium.adapter.exoplayer.readaloud" +} + +dependencies { + api(project(":readium:readium-shared")) + api(project(":readium:navigators:media:readium-navigator-media-readaloud")) + + implementation(project(":readium:adapters:exoplayer:readium-adapter-exoplayer-audio")) + + implementation(libs.androidx.media3.common) + implementation(libs.androidx.media3.exoplayer) + implementation(libs.timber) + implementation(libs.kotlinx.coroutines.android) + implementation(libs.kotlinx.serialization.json) + + // Tests + testImplementation(libs.junit) +} diff --git a/readium/adapters/exoplayer/readaloud/gradle.properties b/readium/adapters/exoplayer/readaloud/gradle.properties new file mode 100644 index 0000000000..78bc7e9a97 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/gradle.properties @@ -0,0 +1 @@ +pom.artifactId=readium-adapter-exoplayer-readaloud diff --git a/readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml b/readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..2d10029868 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt new file mode 100644 index 0000000000..fb1c28313d --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngine.kt @@ -0,0 +1,252 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.adapter.exoplayer.readaloud + +import android.app.Application +import androidx.media3.common.AudioAttributes +import androidx.media3.common.C +import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackParameters +import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSource +import androidx.media3.exoplayer.ExoPlaybackException +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import kotlin.properties.Delegates +import kotlinx.coroutines.ExperimentalCoroutinesApi +import org.readium.navigator.media.readaloud.AudioChunk +import org.readium.navigator.media.readaloud.AudioEngineProgress +import org.readium.navigator.media.readaloud.PlaybackEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.findInstance +import org.readium.r2.shared.util.ThrowableError +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.data.ReadException + +/** + * A [PlaybackEngine] based on Media3 ExoPlayer. + */ +@ExperimentalReadiumApi +@OptIn(ExperimentalCoroutinesApi::class) +@androidx.annotation.OptIn(UnstableApi::class) +public class ExoPlayerEngine private constructor( + private val exoPlayer: ExoPlayer, + private val listener: PlaybackEngine.Listener, +) : PlaybackEngine { + + public companion object { + + public operator fun invoke( + application: Application, + dataSourceFactory: DataSource.Factory, + chunks: List, + listener: PlaybackEngine.Listener, + ): ExoPlayerEngine { + val exoPlayer = ExoPlayer.Builder(application) + .setMediaSourceFactory(DefaultMediaSourceFactory(dataSourceFactory)) + .setAudioAttributes( + AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_MUSIC) + .setUsage(C.USAGE_MEDIA) + .build(), + true + ) + .setHandleAudioBecomingNoisy(true) + .build() + + exoPlayer.preloadConfiguration = ExoPlayer.PreloadConfiguration(10_000_000L) + exoPlayer.pauseAtEndOfMediaItems = true + + val mediaItems = chunks.map { item -> + val clippingConfig = MediaItem.ClippingConfiguration.Builder() + .apply { + item.interval?.start?.let { setStartPositionMs(it.inWholeMilliseconds) } + item.interval?.end?.let { setEndPositionMs(it.inWholeMilliseconds) } + }.build() + MediaItem.Builder() + .setUri(item.href.toString()) + .setClippingConfiguration(clippingConfig) + .build() + } + exoPlayer.setMediaItems(mediaItems) + exoPlayer.prepare() + + return ExoPlayerEngine(exoPlayer, listener) + } + } + + private inner class Listener : Player.Listener { + + override fun onPlaybackParametersChanged(playbackParameters: PlaybackParameters) { + } + + override fun onPlayWhenReadyChanged(playWhenReady: Boolean, reason: Int) { + if (reason == Player.PLAY_WHEN_READY_CHANGE_REASON_END_OF_MEDIA_ITEM) { + state = State.Ended + } + } + + override fun onPlaybackStateChanged(playbackState: Int) { + state = when (val stateNow = state) { + State.Idle -> { + stateNow + } + State.Ended -> { + stateNow + } + is State.Running -> { + when (playbackState) { + Player.STATE_READY -> + stateNow.copy(playbackState = PlaybackEngine.PlaybackState.Playing) + Player.STATE_BUFFERING -> + stateNow.copy(playbackState = PlaybackEngine.PlaybackState.Starved) + Player.STATE_ENDED -> + State.Idle + else -> stateNow + } + } + } + } + } + + public sealed class Error( + override val message: String, + override val cause: org.readium.r2.shared.util.Error?, + ) : org.readium.r2.shared.util.Error { + + public data class Engine(override val cause: ThrowableError) : + Error("An error occurred in the ExoPlayer engine.", cause) + + public data class Source(override val cause: ReadError) : + Error("An error occurred while trying to read publication content.", cause) + } + + private sealed interface State { + + object Idle : State + + object Ended : State + + data class Running( + val playbackState: PlaybackEngine.PlaybackState, + val paused: Boolean, + ) : State + } + + override var pitch: Double + get() = exoPlayer.playbackParameters.pitch.toDouble() + set(value) { + exoPlayer.playbackParameters = + exoPlayer.playbackParameters.withPitch(value.toFloat()) + } + + override var speed: Double + get() = exoPlayer.playbackParameters.speed.toDouble() + set(value) { + exoPlayer.playbackParameters = + exoPlayer.playbackParameters.withSpeed(value.toFloat()) + } + + override var itemToPlay: Int by Delegates.observable(0) { property, oldValue, newValue -> + when (state) { + State.Ended -> { + if (newValue != exoPlayer.currentMediaItemIndex + 1) { + exoPlayer.seekTo(newValue, 0) + } + } + State.Idle -> { + if (newValue != exoPlayer.currentMediaItemIndex) { + exoPlayer.seekTo(newValue, 0) + } + } + is State.Running -> { + } + } + } + + private var state: State by Delegates.observable(State.Idle) { property, oldValue, newValue -> + if (newValue != oldValue) { + when { + newValue is State.Ended -> { + listener.onPlaybackCompleted() + } + newValue is State.Running && oldValue is State.Running && + newValue.playbackState != oldValue.playbackState -> { + listener.onPlaybackStateChanged(newValue.playbackState) + } + } + } + } + + init { + exoPlayer.addListener(Listener()) + } + + override fun start() { + val playbackState = when (exoPlayer.playbackState) { + Player.STATE_READY -> PlaybackEngine.PlaybackState.Playing + Player.STATE_BUFFERING -> PlaybackEngine.PlaybackState.Starved + else -> throw IllegalStateException("Unexpected ExoPlayer state ${exoPlayer.playbackState}") + } + + listener.onStartRequested(playbackState) + exoPlayer.playWhenReady = true + state = State.Running(playbackState = playbackState, paused = false) + } + + override fun stop() { + exoPlayer.playWhenReady = false + exoPlayer.seekTo(0) + state = State.Idle + } + + override fun resume() { + state = when (val stateNow = state) { + State.Idle, State.Ended -> { + stateNow + } + is State.Running -> { + exoPlayer.playWhenReady = true + stateNow.copy(paused = false) + } + } + } + + override fun pause() { + state = when (val stateNow = state) { + State.Idle, State.Ended -> { + stateNow + } + is State.Running -> { + exoPlayer.playWhenReady = false + stateNow.copy(paused = true) + } + } + } + + public override fun release() { + exoPlayer.release() + } + + @OptIn(InternalReadiumApi::class) + private fun ExoPlaybackException.toError(): Error { + val readError = + if (type == ExoPlaybackException.TYPE_SOURCE) { + sourceException.findInstance(ReadException::class.java)?.error + } else { + null + } + + return if (readError == null) { + Error.Engine(ThrowableError(this)) + } else { + Error.Source(readError) + } + } +} diff --git a/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt new file mode 100644 index 0000000000..b382993409 --- /dev/null +++ b/readium/adapters/exoplayer/readaloud/src/main/java/org/readium/adapter/exoplayer/readaloud/ExoPlayerEngineProvider.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class) + +package org.readium.adapter.exoplayer.readaloud + +import android.app.Application +import org.readium.adapter.exoplayer.audio.ExoPlayerDataSource +import org.readium.navigator.media.readaloud.AudioChunk +import org.readium.navigator.media.readaloud.AudioEngineFactory +import org.readium.navigator.media.readaloud.AudioEngineProgress +import org.readium.navigator.media.readaloud.AudioEngineProvider +import org.readium.navigator.media.readaloud.PlaybackEngine +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.publication.Publication + +@ExperimentalReadiumApi +public class ExoPlayerEngineProvider( + private val application: Application, +) : AudioEngineProvider { + + override fun createEngineFactory(publication: Publication): ExoPlayerEngineFactory { + val dataSourceFactory = ExoPlayerDataSource.Factory(publication) + return ExoPlayerEngineFactory(application, dataSourceFactory) + } +} + +@ExperimentalReadiumApi +public class ExoPlayerEngineFactory internal constructor( + private val application: Application, + private val dataSourceFactory: ExoPlayerDataSource.Factory, +) : AudioEngineFactory { + + override fun createPlaybackEngine( + chunks: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine { + return ExoPlayerEngine(application, dataSourceFactory, chunks, listener) + } +} diff --git a/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt b/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt index 7164bcea2c..58c7ac9cce 100644 --- a/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt +++ b/readium/navigators/common/src/main/java/org/readium/navigator/common/LocationElements.kt @@ -6,6 +6,7 @@ package org.readium.navigator.common +import kotlin.time.Duration import org.readium.r2.shared.ExperimentalReadiumApi /** @@ -60,3 +61,15 @@ public data class TextQuote( val prefix: String, val suffix: String, ) + +@ExperimentalReadiumApi +public data class TextAnchor( + val prefix: String, + val suffix: String, +) + +@JvmInline +@ExperimentalReadiumApi +public value class TimeOffset( + public val value: Duration, +) diff --git a/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt b/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt index 97c9ba97a4..bc3947ea36 100644 --- a/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt +++ b/readium/navigators/common/src/main/java/org/readium/navigator/common/Locations.kt @@ -37,6 +37,15 @@ public interface TextQuoteLocation : Location { public val textQuote: TextQuote } +/** + * A [Location] including a [TextAnchor]. + */ +@ExperimentalReadiumApi +public interface TextAnchorLocation : Location { + + public val textAnchor: TextAnchor +} + /** * A [Location] including a [CssSelector]. */ @@ -63,3 +72,9 @@ public interface PositionLocation : Location { public val position: Position } + +@ExperimentalReadiumApi +public interface TimeOffsetLocation : Location { + + public val timeOffset: TimeOffset +} diff --git a/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt b/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt index 85353eebc8..8c05ad877b 100644 --- a/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt +++ b/readium/navigators/media/common/src/main/java/org/readium/navigator/media/common/MediaNavigator.kt @@ -59,6 +59,11 @@ public interface MediaNavigator< /** * State of the playback. */ + /* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ public interface Playback { /** diff --git a/readium/navigators/media/readaloud/build.gradle.kts b/readium/navigators/media/readaloud/build.gradle.kts new file mode 100644 index 0000000000..f100c98a4f --- /dev/null +++ b/readium/navigators/media/readaloud/build.gradle.kts @@ -0,0 +1,26 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +plugins { + id("readium.library-conventions") + alias(libs.plugins.kotlin.serialization) +} + +android { + namespace = "org.readium.navigators.media.readaloud" +} + +dependencies { + api(project(":readium:navigators:readium-navigator-common")) + api(project(":readium:navigators:media:readium-navigator-media-common")) + + implementation(libs.androidx.media3.common) + implementation(libs.androidx.media3.session) + + implementation(libs.timber) + implementation(libs.kotlinx.coroutines.android) + implementation(libs.kotlinx.serialization.json) +} diff --git a/readium/navigators/media/readaloud/gradle.properties b/readium/navigators/media/readaloud/gradle.properties new file mode 100644 index 0000000000..d944c2678c --- /dev/null +++ b/readium/navigators/media/readaloud/gradle.properties @@ -0,0 +1 @@ +pom.artifactId=readium-navigator-media-readaloud diff --git a/readium/navigators/media/readaloud/src/main/AndroidManifest.xml b/readium/navigators/media/readaloud/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..2d10029868 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + + diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentGuidedNavigationService.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentGuidedNavigationService.kt new file mode 100644 index 0000000000..07b336fe30 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ContentGuidedNavigationService.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationText +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.publication.Locator +import org.readium.r2.shared.publication.html.cssSelector +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.publication.services.content.Content +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.data.ReadError + +internal class ContentGuidedNavigationService( + private val contentService: ContentService, +) : GuidedNavigationService { + + override fun iterator(): GuidedNavigationIterator { + val contentIterator = contentService.content(start = null).iterator() + return Iterator(contentIterator) + } + + private class Iterator( + private val contentIterator: Content.Iterator, + ) : GuidedNavigationIterator { + + private var guidedNavigationDocument: GuidedNavigationDocument? = null + + private var ended: Boolean = false + + override suspend fun hasNext(): Boolean { + if (ended) { + return false + } + + guidedNavigationDocument = createGuidedNavigationDocument() + ended = true + return true + } + + override suspend fun next(): Try { + val res = checkNotNull(guidedNavigationDocument) + return Try.success(res) + } + + private suspend fun createGuidedNavigationDocument(): GuidedNavigationDocument { + val tree = mutableListOf() + + while (contentIterator.hasNext()) { + val nodes = when (val element = contentIterator.next()) { + is Content.TextElement -> { + element.segments.mapNotNull { segment -> + if (segment.text.isBlank()) { + return@mapNotNull null + } + + GuidedNavigationObject( + refs = setOfNotNull(segment.locator.toTextRef()), + text = GuidedNavigationText( + plain = segment.text, + ssml = null, + language = segment.language + ), + ) + } + } + + is Content.TextualElement -> { + listOfNotNull( + element.text + ?.takeIf { it.isNotBlank() } + ?.let { + GuidedNavigationObject( + refs = setOfNotNull(element.locator.toTextRef()), + text = GuidedNavigationText(it) + ) + } + ) + } + + else -> emptyList() + } + + if (nodes.isNotEmpty()) { + tree.add( + GuidedNavigationObject( + children = nodes + ) + ) + } + } + + return GuidedNavigationDocument(guided = tree) + } + + private fun Locator.toTextRef(): GuidedNavigationTextRef? { + val htmlId = locations.cssSelector + ?.takeIf { it.startsWith("#") } + .orEmpty() + val url = Url("$href$htmlId") + return url?.let { GuidedNavigationTextRef(it) } + } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt new file mode 100644 index 0000000000..4b91720628 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/PlaybackEngine.kt @@ -0,0 +1,203 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import kotlin.time.Duration +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.util.Error +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +@ExperimentalReadiumApi +public interface AudioEngineProvider { + + public fun createEngineFactory( + publication: Publication, + ): AudioEngineFactory +} + +@ExperimentalReadiumApi +public interface AudioEngineFactory { + + public fun createPlaybackEngine( + chunks: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine +} + +@ExperimentalReadiumApi +public data class AudioChunk( + val href: Url, + val interval: TimeInterval?, +) + +@ExperimentalReadiumApi +public interface TtsEngineProvider { + + public suspend fun createEngineFactory(): TtsEngineFactory +} + +@ExperimentalReadiumApi +public interface TtsEngineFactory { + + /** + * Sets of voices available with this [TtsEngineFactory]. + */ + public val voices: Set + + /** + * Creates a [PlaybackEngine] to read [utterances] using the given [voiceId]. + * + * Throws if the given [voiceId] matches no voice in [voices]. + */ + public fun createPlaybackEngine( + voiceId: TtsVoice.Id, + utterances: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine +} + +internal class NullTtsEngineFactory() : TtsEngineFactory { + + override val voices: Set = emptySet() + + override fun createPlaybackEngine( + voiceId: TtsVoice.Id, + utterances: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine { + throw IllegalArgumentException("Unknown voice.") + } +} + +/** + * Engine reading aloud a list of items. + */ +@ExperimentalReadiumApi +public interface PlaybackEngine { + + /** + * State of the playback. + */ + public enum class PlaybackState { + /** + * The playback is ongoing. + */ + Playing, + + /** + * The playback has been momentarily interrupted because of a lack of ready data. + */ + Starved, + } + + /** + * Marker interface for playback progress information. + */ + public interface Progress + + /** + * Listener for a [PlaybackEngine] + */ + public interface Listener { + + /** + * Called after [start] was invoked. [initialState] tells you if the engine has enough data + * to start playing right now or must still wait for data. + */ + public fun onStartRequested(initialState: PlaybackState) + + /** + * Called when the playback state changes. + */ + public fun onPlaybackStateChanged(state: PlaybackState) + + /** + * Called when the last playback request completed. + */ + public fun onPlaybackCompleted() + + /** + * Called when an error occurred during playback. + */ + public fun onPlaybackError(error: E) + + /** + * Called regularly to report progress when this information is available. + */ + public fun onPlaybackProgressed(progress: P) + } + + /** + * Sets the playback pitch. + */ + public var pitch: Double + + /** + * Sets the playback speed. + */ + public var speed: Double + + /** + * Sets the index of the item to play on the next call to [start]. + * + * The behavior is undefined if this property is set during playback. + */ + public var itemToPlay: Int + + /** + * Starts playing the [itemToPlay]-th item. + */ + public fun start() + + /** + * Stops ongoing playback. + */ + public fun stop() + + /** + * Pauses playback. + */ + public fun pause() + + /** + * Resumes playback where it was paused if possible, or starts again otherwise. + */ + public fun resume() + + /** + * Free all used resources. + */ + public fun release() +} + +@ExperimentalReadiumApi +public interface TtsVoice { + + @kotlinx.serialization.Serializable + @JvmInline + public value class Id(public val value: String) + + public val id: Id + + /** + * The languages supported by the voice. + */ + public val languages: Set +} + +@ExperimentalReadiumApi +@JvmInline +public value class TtsEngineProgress(public val value: IntRange) : PlaybackEngine.Progress + +@ExperimentalReadiumApi +@JvmInline +public value class AudioEngineProgress(public val value: Duration) : PlaybackEngine.Progress diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt new file mode 100644 index 0000000000..6382c220a3 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudDataLoader.kt @@ -0,0 +1,66 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import kotlin.properties.Delegates +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.shared.ExperimentalReadiumApi + +internal class ReadAloudDataLoader( + private val segmentFactory: ReadAloudSegmentFactory, + initialSettings: ReadAloudSettings, +) { + data class ItemRef( + val segment: ReadAloudSegment, + val nodeIndex: Int?, + ) + + var settings by Delegates.observable(initialSettings) { property, oldValue, newValue -> + preloadedRefs.clear() + } + + private val preloadedRefs: MutableMap = mutableMapOf() + + fun onPlaybackProgressed(node: ReadAloudNode) { + val nextNode = node.next() ?: return + + if (nextNode !in preloadedRefs) { + loadSegmentForNode(nextNode) + } + } + + fun getItemRef(node: ReadAloudNode): ItemRef? { + loadSegmentForNode(node) + return preloadedRefs[node] + } + + private fun loadSegmentForNode(node: ReadAloudNode): ReadAloudSegment? { + if (node in preloadedRefs) { + return null + } + + val segment = segmentFactory.createSegmentFromNode(node) + ?: return null // Ended + + val refs = computeRefsForSegment(segment) + preloadedRefs.putAll(refs) + + return segment + } + + private fun computeRefsForSegment(segment: ReadAloudSegment): Map { + val plainRefs = segment.nodes + .withIndex() + .associate { (index, node) -> node to ItemRef(segment, index) } + + val emptyRefs = segment.emptyNodes.associateWith { ItemRef(segment, null) } + + return plainRefs + emptyRefs + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt new file mode 100644 index 0000000000..04594a5b48 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudLocations.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.navigator.common.CssSelector +import org.readium.navigator.common.Location +import org.readium.navigator.common.TextAnchor +import org.readium.navigator.common.TextQuote +import org.readium.navigator.common.TimeOffset +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Url + +@ExperimentalReadiumApi +public sealed interface ReadAloudLocation : Location + +@ExperimentalReadiumApi +public data class ReadAloudTextLocation( + override val href: Url, + public val textAnchor: TextAnchor?, + public val cssSelector: CssSelector?, +) : ReadAloudLocation + +@ExperimentalReadiumApi +public data class ReadAloudAudioLocation( + override val href: Url, + public val timeOffset: TimeOffset?, +) : ReadAloudLocation + +@ExperimentalReadiumApi +public sealed interface ReadAloudHighlightLocation : Location + +@ExperimentalReadiumApi +public data class ReadAloudTextHighlightLocation( + override val href: Url, + public val textQuote: TextQuote?, + public val cssSelector: CssSelector?, +) : ReadAloudHighlightLocation diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt new file mode 100644 index 0000000000..c6f89f1f03 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigationHelper.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.util.TemporalFragmentParser + +internal class ReadAloudNavigationHelper( + var settings: ReadAloudSettings, +) { + + fun ReadAloudNode.firstMatchingLocation(location: ReadAloudLocation): ReadAloudNode? = + firstDescendantOrNull { it.matchLocation(location) } + + private fun ReadAloudNode.matchLocation(location: ReadAloudLocation): Boolean = + when (location) { + is ReadAloudAudioLocation -> { + refs.any { ref -> + val refFragment = ref.url.fragment?.let { TemporalFragmentParser.parse(it) } + ref.url.removeFragment() == location.href && refFragment?.start == location.timeOffset?.value + } + } + is ReadAloudTextLocation -> { + refs.any { ref -> + ref.url.removeFragment() == location.href && + ref.url.fragment == location.cssSelector?.value?.removePrefix("#") + } + } + } + + fun ReadAloudNode.isSkippable(): Boolean = + nearestSkippable() != null + + fun ReadAloudNode.isEscapable(): Boolean = + nearestEscapable() != null + + fun ReadAloudNode.firstContentNode(): ReadAloudNode? = + firstDescendantOrNull { it.hasContent() } + + fun ReadAloudNode.hasContent() = + text != null || refs.firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } != null + + fun ReadAloudNode.nextContentNode(): ReadAloudNode? { + val parent = parent + ?: return children.firstOrNull()?.nextContentNode() + + val siblings = parent.children + + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + val next = if (currentIndex < siblings.size - 1) { + siblings[currentIndex + 1] + } else { + parent.next() + } + + return next?.firstContentNode() + } + + fun ReadAloudNode.escape(force: Boolean): ReadAloudNode? = + (nearestEscapable() ?: this.takeIf { force })?.next() + + fun ReadAloudNode.skipToNext(force: Boolean): ReadAloudNode? = + (nearestSkippable() ?: this.takeIf { force })?.next() + + fun ReadAloudNode.skipToPrevious(force: Boolean): ReadAloudNode? = + (nearestSkippable() ?: this.takeIf { force })?.previous() + + private fun ReadAloudNode.nearestEscapable(): ReadAloudNode? = + nearestOrNull { roles.any { it in settings.escapableRoles } } + + private fun ReadAloudNode.nearestSkippable(): ReadAloudNode? = + nearestOrNull { roles.any { it in settings.skippableRoles } } + + private fun ReadAloudNode.nearestOrNull( + predicate: (ReadAloudNode) -> Boolean, + ): ReadAloudNode? = + when { + predicate(this) -> this + parent == null -> null + else -> parent!!.nearestOrNull(predicate) + } + + private fun ReadAloudNode.firstDescendantOrNull( + predicate: (ReadAloudNode) -> Boolean, + ): ReadAloudNode? = + when { + predicate(this) -> this + children.isEmpty() -> null + else -> children.firstNotNullOfOrNull { it.firstDescendantOrNull(predicate) } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt new file mode 100644 index 0000000000..f6873b91c7 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigator.kt @@ -0,0 +1,412 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import android.os.Handler +import android.os.Looper +import kotlin.properties.Delegates +import kotlin.time.Duration +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.runningFold +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import org.readium.navigator.common.CssSelector +import org.readium.navigator.common.TimeOffset +import org.readium.navigator.media.common.MediaNavigator +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.addPrefix +import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.Error as BaseError +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TemporalFragmentParser +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.mediatype.MediaType + +@ExperimentalReadiumApi +public class ReadAloudNavigator private constructor( + private val guidedNavigationTree: ReadAloudNode, + private val resources: List, + private val audioEngineFactory: AudioEngineFactory, + private val ttsEngineFactory: TtsEngineFactory, + initialSettings: ReadAloudSettings, + initialLocation: ReadAloudLocation?, +) { + public companion object { + + internal suspend operator fun invoke( + initialLocation: ReadAloudLocation?, + initialSettings: ReadAloudSettings, + publication: ReadAloudPublication, + audioEngineFactory: AudioEngineFactory, + ttsEngineFactory: TtsEngineFactory, + ): ReadAloudNavigator { + val tree = withContext(Dispatchers.Default) { + ReadAloudNode.fromGuidedNavigationObject(publication.guidedNavigationTree) + } + + return ReadAloudNavigator( + guidedNavigationTree = tree, + resources = publication.resources, + audioEngineFactory = audioEngineFactory, + ttsEngineFactory = ttsEngineFactory, + initialSettings = initialSettings, + initialLocation = initialLocation + ) + } + } + + public data class Playback( + val state: PlaybackState, + val playWhenReady: Boolean, + val node: ReadAloudNode, + private val textItemMediaType: MediaType?, + ) { + val nodeHighlightLocation: ReadAloudHighlightLocation? get() { + val textRef = node.refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + ?: return null + + val href = textRef.url.removeFragment() + val cssSelector = textRef.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + return ReadAloudTextHighlightLocation( + href = href, + textQuote = null, + // mediaType = textItemMediaType, + cssSelector = cssSelector + ) + } + + val utteranceHighlightLocation: ReadAloudHighlightLocation? get() { + return null + } + } + + public sealed interface PlaybackState { + + public data object Ready : PlaybackState, MediaNavigator.State.Ready + + public data object Starved : PlaybackState, MediaNavigator.State.Buffering + + public data object Ended : PlaybackState, MediaNavigator.State.Ended + + public data class Failure(val error: Error) : PlaybackState, MediaNavigator.State.Failure + } + + public sealed class Error( + override val message: String, + override val cause: BaseError?, + ) : BaseError { + + public data class AudioEngineError(override val cause: BaseError) : + Error("An error occurred in the audio engine.", cause) + + public data class TtsEngineError(override val cause: ReadError) : + Error("An error occurred in the TTS engine.", cause) + } + + private inner class PlaybackEngineListener : PlaybackEngine.Listener { + + private val handler = Handler(Looper.getMainLooper()) + + override fun onPlaybackStateChanged(state: PlaybackEngine.PlaybackState) { + handler.post { + with(stateMachine) { + stateMutable.value = stateMutable.value.onPlaybackEngineStateChanged(state) + } + } + } + + override fun onStartRequested(initialState: PlaybackEngine.PlaybackState) { + handler.post { + with(stateMachine) { + stateMutable.value = stateMutable.value.onPlaybackEngineStateChanged(initialState) + } + } + } + + override fun onPlaybackCompleted() { + handler.post { + with(stateMachine) { + stateMutable.value = stateMutable.value.onPlaybackCompleted() + } + } + } + + override fun onPlaybackError(error: BaseError) { + } + + override fun onPlaybackProgressed(progress: PlaybackEngine.Progress) { + } + } + + public var settings: ReadAloudSettings by Delegates.observable(initialSettings) { + property, oldValue, newValue -> + with(stateMachine) { + if (newValue != oldValue) { + stateMutable.value = stateMutable.value.onSettingsChanged(newValue) + } + } + } + + public val voices: Set = + ttsEngineFactory.voices + + private val segmentFactory = ReadAloudSegmentFactory( + audioEngineFactory = { chunks: List -> + audioEngineFactory.createPlaybackEngine( + chunks = chunks, + listener = PlaybackEngineListener() + ) + }, + ttsEngineFactory = { language: Language?, utterances: List -> + val language = + settings.language + .takeIf { settings.overrideContentLanguage } + ?: language + ?: settings.language + + val preferredVoiceWithRegion = + settings.voices[language] + ?.let { voiceId -> ttsEngineFactory.voices.firstOrNull { it.id == voiceId } } + + val preferredVoiceWithoutRegion = + language + .let { settings.voices[it.removeRegion()] } + ?.let { voiceId -> ttsEngineFactory.voices.firstOrNull { it.id == voiceId } } + + val fallbackVoiceWithRegion = ttsEngineFactory.voices + .firstOrNull { language in it.languages } + + val fallbackVoiceWithoutRegion = ttsEngineFactory.voices + .firstOrNull { voice -> language.removeRegion() in voice.languages.map { it.removeRegion() } } + + val voice = preferredVoiceWithRegion + ?: preferredVoiceWithoutRegion + ?: fallbackVoiceWithRegion + ?: fallbackVoiceWithoutRegion + ?: ttsEngineFactory.voices.firstOrNull() + + voice?.let { voice -> + ttsEngineFactory.createPlaybackEngine( + voiceId = voice.id, + utterances = utterances, + listener = PlaybackEngineListener() + ) + } + } + ) + + private val dataLoader = ReadAloudDataLoader(segmentFactory, initialSettings) + + private val navigationHelper = ReadAloudNavigationHelper(initialSettings) + + private val stateMachine = ReadAloudStateMachine(dataLoader, navigationHelper) + + private val initialNode = with(navigationHelper) { + val nodeFromLocation = initialLocation + ?.let { guidedNavigationTree.firstMatchingLocation(it) } + ?.firstContentNode() + nodeFromLocation ?: guidedNavigationTree.firstContentNode() ?: guidedNavigationTree + } + + private val stateMutable: MutableStateFlow = run { + val itemRef = dataLoader.getItemRef(initialNode)!! // there is at least one node to read + val nodeIndex = itemRef.nodeIndex!! // the node has content + + MutableStateFlow( + stateMachine.play( + segment = itemRef.segment, + itemIndex = nodeIndex, + playWhenReady = true, + settings = settings + ) + ) + } + + private val coroutineScope: CoroutineScope = + MainScope() + + public val playback: StateFlow = + stateMutable.mapStateIn(coroutineScope) { state -> + val textRef = state.node.refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + val textHref = textRef?.url?.removeFragment() + val textItemMediaType = textHref?.let { + resources.firstOrNull { item -> item.href == textHref }?.mediaType + } + Playback( + playWhenReady = state.playWhenReady, + state = state.playbackState.toState(), + node = state.node, + textItemMediaType = textItemMediaType + ) + } + private fun ReadAloudStateMachine.PlaybackState.toState(): PlaybackState = + when (this) { + is ReadAloudStateMachine.PlaybackState.Ready -> + PlaybackState.Ready + is ReadAloudStateMachine.PlaybackState.Starved -> + PlaybackState.Starved + is ReadAloudStateMachine.PlaybackState.Ended -> + PlaybackState.Ended + is ReadAloudStateMachine.PlaybackState.Failure -> + PlaybackState.Failure(Error.AudioEngineError(error)) + } + + public val locations: StateFlow> = + stateMutable.runningFold( + emptyList() + ) { prevLocations: List, state -> + val textLocation = state.node.toReadAloudTextLocation() + val audioLocation = state.node.toReadAloudAudioLocation() + buildList { + when (state.segment) { + is AudioSegment -> { + audioLocation?.let { add(it) } + textLocation?.let { add(it) } + } + is TtsSegment -> { + textLocation?.let { add(it) } + audioLocation?.let { add(it) } + } + } + + if (none { it is ReadAloudTextLocation }) { + prevLocations.firstOrNull { it is ReadAloudTextLocation } + ?.let { add(it) } + } + + if (none { it is ReadAloudAudioLocation }) { + prevLocations.firstOrNull { it is ReadAloudAudioLocation } + ?.let { add(it) } + } + } + }.stateInFirst( + scope = coroutineScope, + started = SharingStarted.Eagerly + ) + + private fun ReadAloudNode.toReadAloudTextLocation(): ReadAloudTextLocation? { + val textRef = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + ?: return null + val href = textRef.url.removeFragment() + val cssSelector = textRef.url.fragment + ?.let { fragment -> CssSelector(fragment.addPrefix("#")) } + return ReadAloudTextLocation(href = href, cssSelector = cssSelector, textAnchor = null) + } + + private fun ReadAloudNode.toReadAloudAudioLocation(): ReadAloudAudioLocation? { + val audioRef = refs.firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } + ?: return null + + val href = audioRef.url.removeFragment() + val timeOffset = audioRef.url.fragment + ?.let { TemporalFragmentParser.parse(it) } + ?.start + ?.let { TimeOffset(it) } + return ReadAloudAudioLocation(href = href, timeOffset = timeOffset) + } + + private fun Flow.stateInFirst( + scope: CoroutineScope, + started: SharingStarted, + ): StateFlow { + val first = runBlocking { first() } + return stateIn(scope, started, first) + } + + public fun play() { + with(stateMachine) { + stateMutable.value = stateMutable.value.resume() + } + } + + public fun pause() { + with(stateMachine) { + stateMutable.value = stateMutable.value.pause() + } + } + + public fun go(node: ReadAloudNode) { + with(stateMachine) { + stateMutable.value = stateMutable.value.jump(node) + } + } + + public fun canEscape(): Boolean = + with(navigationHelper) { + stateMutable.value.node.isEscapable() + } + + public fun canSkip(): Boolean = + with(navigationHelper) { + stateMutable.value.node.isSkippable() + } + + public fun escape(force: Boolean = true) { + with(navigationHelper) { + stateMutable.value.node.escape(force) + ?.let { go(it) } + } + } + + public fun skipToPrevious(force: Boolean = true) { + with(navigationHelper) { + stateMutable.value.node.skipToPrevious(force) + ?.let { go(it) } + } + } + + public fun skipToNext(force: Boolean = true) { + with(navigationHelper) { + stateMutable.value.node.skipToNext(force) + ?.let { go(it) } + } + } + + public fun goTo(location: ReadAloudLocation) { + with(navigationHelper) { + guidedNavigationTree.firstMatchingLocation(location) + ?.let { go(it) } + } + } + + public fun goTo(url: Url) { + val location = url.fragment?.let { TemporalFragmentParser.parse(it) } + ?.let { timeInterval -> + val href = url.removeFragment() + val timeOffset = TimeOffset(timeInterval.start ?: Duration.ZERO) + ReadAloudAudioLocation(href = href, timeOffset = timeOffset) + } ?: ReadAloudTextLocation( + href = url.removeFragment(), + cssSelector = url.fragment?.let { CssSelector(it.addPrefix("#")) }, + textAnchor = null + ) + + goTo(location) + } + + public fun release() { + with(stateMachine) { + stateMutable.value = stateMutable.value.release() + } + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt new file mode 100644 index 0000000000..7912a3915a --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNavigatorFactory.kt @@ -0,0 +1,141 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud + +import android.app.Application +import org.readium.navigator.media.readaloud.preferences.ReadAloudDefaults +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferences +import org.readium.navigator.media.readaloud.preferences.ReadAloudPreferencesEditor +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.epub.MediaOverlaysService +import org.readium.r2.shared.publication.services.GuidedNavigationService +import org.readium.r2.shared.publication.services.content.ContentService +import org.readium.r2.shared.util.Error as BaseError +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.getOrElse + +@ExperimentalReadiumApi +public class ReadAloudNavigatorFactory private constructor( + private val publicationMetadata: Metadata, + private val guidedNavigationService: GuidedNavigationService, + private val resources: List, + private val audioEngineFactory: () -> AudioEngineFactory, + private val ttsEngineProvider: TtsEngineProvider, +) { + + public companion object { + + public operator fun invoke( + application: Application, + publication: Publication, + audioEngineProvider: AudioEngineProvider, + ttsEngineProvider: TtsEngineProvider, + usePrerecordedVoicesWhenAvailable: Boolean = true, + ): ReadAloudNavigatorFactory? { + var guidedNavService: GuidedNavigationService? = null + + if (usePrerecordedVoicesWhenAvailable) { + guidedNavService = publication.findService(MediaOverlaysService::class) + } + if (guidedNavService == null) { + guidedNavService = publication.findService(GuidedNavigationService::class) + } + + if (guidedNavService == null) { + publication.findService(ContentService::class) + ?.let { guidedNavService = ContentGuidedNavigationService(it) } + } + + if (guidedNavService == null) { + return null + } + + val resources = (publication.readingOrder + publication.resources).map { + ReadAloudPublication.Item( + href = it.url(), + mediaType = it.mediaType + ) + } + + return ReadAloudNavigatorFactory( + publicationMetadata = publication.metadata, + guidedNavigationService = guidedNavService, + resources = resources, + audioEngineFactory = { audioEngineProvider.createEngineFactory(publication) }, + ttsEngineProvider = ttsEngineProvider + ) + } + } + + public sealed class Error( + override val message: String, + override val cause: BaseError?, + ) : BaseError { + + public class UnsupportedPublication( + cause: BaseError? = null, + ) : Error("Publication is not supported.", cause) + + public class GuidedNavigationService( + override val cause: ReadError, + ) : Error("Failed to acquire guided navigation documents.", cause) + } + + public suspend fun createNavigator( + initialSettings: ReadAloudSettings, + initialLocation: ReadAloudLocation? = null, + ): Try { + val guidedDocs = buildList { + val iterator = guidedNavigationService.iterator() + while (iterator.hasNext()) { + val doc = iterator.next().getOrElse { + return Try.failure(Error.GuidedNavigationService(it)) + } + add(doc) + } + } + + val guidedTree = GuidedNavigationObject( + children = guidedDocs.map { + GuidedNavigationObject(it.guided, roles = emptySet(), refs = emptySet(), text = null) + }, + roles = emptySet(), + refs = emptySet(), + text = null + ) + + val navigatorPublication = ReadAloudPublication( + guidedNavigationTree = guidedTree, + resources = resources, + ) + + val navigator = ReadAloudNavigator( + initialSettings = initialSettings, + initialLocation = initialLocation, + publication = navigatorPublication, + audioEngineFactory = audioEngineFactory(), + ttsEngineFactory = ttsEngineProvider.createEngineFactory() + ) + + return Try.success(navigator) + } + + public fun createPreferencesEditor( + initialPreferences: ReadAloudPreferences, + defaults: ReadAloudDefaults = ReadAloudDefaults(), + ): ReadAloudPreferencesEditor = + ReadAloudPreferencesEditor( + initialPreferences, + publicationMetadata, + defaults, + ) +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt new file mode 100644 index 0000000000..fee6292d64 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudNode.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationRef +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationText + +@ExperimentalReadiumApi +public class ReadAloudNode( + public val text: GuidedNavigationText?, + public val refs: Set, + public val roles: Set, + public val children: List, +) { + + public var parent: ReadAloudNode? = null + internal set + + public companion object { + + internal fun fromGuidedNavigationObject( + guidedNavigationObject: GuidedNavigationObject, + ): ReadAloudNode { + val children = guidedNavigationObject.children + .map { fromGuidedNavigationObject(it) } + + val node = ReadAloudNode( + text = guidedNavigationObject.text, + refs = guidedNavigationObject.refs, + roles = guidedNavigationObject.roles, + children = children, + ) + + children.forEach { it.parent = node } + + return node + } + } +} + +@ExperimentalReadiumApi +public fun ReadAloudNode.next(): ReadAloudNode? { + nextDown()?.let { return it } + + nextRight()?.let { return it } + + return nextUp() +} + +@ExperimentalReadiumApi +public fun ReadAloudNode.previous(): ReadAloudNode? { + previousLeft()?.lastDescendant()?.let { return it } + + previousUp()?.lastDescendant()?.let { return it } + + return null +} + +private fun ReadAloudNode.lastDescendant(): ReadAloudNode? { + val lastChild = children.lastOrNull() + + return if (lastChild == null) { + this + } else { + lastChild.lastDescendant() + } +} + +private fun ReadAloudNode.nextDown(): ReadAloudNode? { + if (children.isEmpty()) { + return null + } + + return children.first() +} + +private fun ReadAloudNode.nextUp(): ReadAloudNode? { + return parent?.nextRight() ?: parent?.nextUp() +} + +private fun ReadAloudNode.previousUp(): ReadAloudNode? { + return parent?.previousLeft() ?: parent?.previousUp() +} + +private fun ReadAloudNode.nextRight(): ReadAloudNode? { + val parent = parent ?: return null + + val siblings = parent.children + + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + return if (currentIndex < siblings.size - 1) { + siblings[currentIndex + 1] + } else { + null + } +} + +private fun ReadAloudNode.previousLeft(): ReadAloudNode? { + val parent = parent ?: return null + + val siblings = parent.children + + val currentIndex = siblings.indexOf(this) + check(currentIndex != -1) + + return if (currentIndex > 0) { + siblings[currentIndex - 1] + } else { + null + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt new file mode 100644 index 0000000000..78797c53c9 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudPublication.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.mediatype.MediaType + +internal class ReadAloudPublication( + val guidedNavigationTree: GuidedNavigationObject, + val resources: List, +) { + data class Item( + val href: Url, + val mediaType: MediaType?, + ) + + val mediaTypes = resources + .mapNotNull { item -> item.mediaType?.let { item.href to it } } + .associate { it } + + fun itemWithHref(href: Url): Item? = + resources.firstOrNull { it.href == href } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt new file mode 100644 index 0000000000..0bba027cc6 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudSegment.kt @@ -0,0 +1,186 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationText +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.TemporalFragmentParser +import org.readium.r2.shared.util.TimeInterval +import org.readium.r2.shared.util.Url + +internal sealed interface ReadAloudSegment { + + val nodes: List + + val player: PlaybackEngine + + val emptyNodes: Set +} + +internal data class AudioSegment( + override val player: PlaybackEngine, + val items: List, + val textRefs: List, + override val nodes: List, + override val emptyNodes: Set, +) : ReadAloudSegment + +internal data class TtsSegment( + override val player: PlaybackEngine, + val items: List, + override val nodes: List, + override val emptyNodes: Set, +) : ReadAloudSegment + +internal class ReadAloudSegmentFactory( + private val audioEngineFactory: (List) -> PlaybackEngine?, + private val ttsEngineFactory: (Language?, List) -> PlaybackEngine?, +) { + + fun createSegmentFromNode(node: ReadAloudNode): ReadAloudSegment? = + createAudioSegmentFromNode(node) + ?: createTtsSegmentFromNode(node) + + private fun createAudioSegmentFromNode( + firstNode: ReadAloudNode, + ): AudioSegment? { + var nextNode: ReadAloudNode? = firstNode + val audioChunks = mutableListOf() + val textRefs = mutableListOf() + val nodes = mutableListOf() + val emptyNodes = mutableSetOf() + + while (nextNode != null && nextNode.content !is TextContent) { + val audioContent = (nextNode.content as? AudioContent) + + if (audioContent != null) { + audioChunks.add(audioContent.toAudioItem()) + nodes.add(nextNode) + textRefs.add(audioContent.textRef) + } else { + emptyNodes.add(nextNode) + } + + nextNode = nextNode.next() + } + + if (audioChunks.isEmpty()) { + return null + } + + val audioEngine = audioEngineFactory(audioChunks) + ?: return null + + return AudioSegment( + player = audioEngine, + items = audioChunks, + textRefs = textRefs, + nodes = nodes, + emptyNodes = emptyNodes + ) + } + + private fun createTtsSegmentFromNode( + firstNode: ReadAloudNode, + ): TtsSegment? { + var nextNode: ReadAloudNode? = firstNode + val segmentLanguage = firstNode.text?.language + val textItems = mutableListOf() + val nodes = mutableListOf() + val emptyNodes = mutableSetOf() + + while ( + nextNode != null && + nextNode.content !is AudioContent && + nextNode.language == segmentLanguage + ) { + val textContent = (nextNode.content as? TextContent) + + if (textContent != null) { + textItems.add(textContent.text) + nodes.add(nextNode) + } else { + emptyNodes.add(nextNode) + } + + nextNode = nextNode.next() + } + + if (textItems.isEmpty()) { + return null + } + + val utterances = textItems.map { it.plain!! } + + val ttsPlayer = ttsEngineFactory(segmentLanguage, utterances) + ?: return null + + return TtsSegment( + player = ttsPlayer, + items = textItems, + nodes = nodes, + emptyNodes = emptyNodes + ) + } + + private val ReadAloudNode.language: Language? get() = when (content) { + is TextContent -> text?.language + else -> null + } + + private val ReadAloudNode.content: NodeContent? get() { + refs + .firstNotNullOfOrNull { it as? GuidedNavigationAudioRef } + ?.let { audioRef -> + + val textRef = refs.firstNotNullOfOrNull { it as? GuidedNavigationTextRef } + + // Ignore audio nodes without textref. Might be changed later. + textRef?.let { + return AudioContent( + href = audioRef.url.removeFragment(), + interval = audioRef.url.timeInterval, + textRef = it.url + ) + } + } + + text + ?.let { + return TextContent(it) + } + + return null + } +} + +private sealed interface NodeContent + +private data class AudioContent( + val href: Url, + val interval: TimeInterval?, + val textRef: Url, +) : NodeContent + +private data class TextContent( + val text: GuidedNavigationText, +) : NodeContent + +private fun AudioContent.toAudioItem(): AudioChunk { + return AudioChunk( + href = href, + interval = interval + ) +} + +private val Url.timeInterval get() = fragment + ?.let { TemporalFragmentParser.parse(it) } diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt new file mode 100644 index 0000000000..bedc0b024c --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/ReadAloudStateMachine.kt @@ -0,0 +1,165 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import org.readium.navigator.media.readaloud.preferences.ReadAloudSettings +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.util.Error + +internal class ReadAloudStateMachine( + private val dataLoader: ReadAloudDataLoader, + private val navigationHelper: ReadAloudNavigationHelper, +) { + + sealed interface PlaybackState { + + data object Ready : PlaybackState + + data object Starved : PlaybackState + + data object Ended : PlaybackState + + data class Failure(val error: Error) : PlaybackState + } + + data class State( + val playbackState: PlaybackState, + val playWhenReady: Boolean, + val node: ReadAloudNode, + val nodeIndex: Int, + val playerPaused: Boolean, + val segment: ReadAloudSegment, + val settings: ReadAloudSettings, + ) + + fun play( + segment: ReadAloudSegment, + itemIndex: Int, + playWhenReady: Boolean, + settings: ReadAloudSettings, + ): State { + segment.player.stop() + segment.player.itemToPlay = itemIndex + segment.applySettings(settings) + + if (playWhenReady) { + segment.player.start() + } + + return State( + playbackState = PlaybackState.Ready, + playWhenReady = playWhenReady, + node = segment.nodes[itemIndex], + nodeIndex = itemIndex, + playerPaused = false, + segment = segment, + settings = settings + ) + } + + fun State.jump(node: ReadAloudNode): State { + val firstContentNode = with(navigationHelper) { node.firstContentNode() } + ?: return copy(playbackState = PlaybackState.Ended) + + val itemRef = dataLoader.getItemRef(firstContentNode) + ?: return copy(playbackState = PlaybackState.Ended) + + val currentSegment = segment + + if (currentSegment != itemRef.segment) { + currentSegment.player.release() + } + + val index = itemRef.nodeIndex!! // This is a content node + return play(itemRef.segment, index, playWhenReady, settings) + } + + fun State.restart(): State { + segment.player.release() + + val itemRef = dataLoader.getItemRef(node) + ?: return copy(playbackState = PlaybackState.Ended) + + val nodeIndex = itemRef.nodeIndex!! + + return play(itemRef.segment, nodeIndex, playWhenReady, settings) + } + + fun State.pause(): State { + segment.player.pause() + return copy(playWhenReady = false, playerPaused = true) + } + + fun State.resume(): State { + if (playerPaused) { + segment.player.resume() + } else { + segment.player.start() + } + return copy(playWhenReady = true, playerPaused = false) + } + + fun State.release(): State { + segment.player.release() + return this + } + + fun State.onSettingsChanged( + newSettings: ReadAloudSettings, + ): State { + navigationHelper.settings = newSettings + dataLoader.settings = newSettings + + return copy(settings = newSettings).restart() + } + + private fun ReadAloudSegment.applySettings(settings: ReadAloudSettings) { + player.speed = settings.speed + player.pitch = settings.pitch + } + + fun State.onPlaybackEngineStateChanged(state: PlaybackEngine.PlaybackState): State { + return when (state) { + PlaybackEngine.PlaybackState.Playing -> + copy(playbackState = PlaybackState.Ready) + PlaybackEngine.PlaybackState.Starved -> + copy(playbackState = PlaybackState.Starved) + } + } + + fun State.onPlaybackCompleted(): State { + return if (nodeIndex + 1 < segment.nodes.size) { + setSegmentItem(index = nodeIndex + 1, playWhenReady = playWhenReady && settings.readContinuously) + } else { + onSegmentEnded() + } + } + + private fun State.setSegmentItem(index: Int, playWhenReady: Boolean): State { + segment.player.itemToPlay = index + if (playWhenReady) { + segment.player.start() + } + return copy(nodeIndex = index, node = segment.nodes[index], playWhenReady = playWhenReady) + } + + private fun State.onSegmentEnded(): State { + val nextNode = node.next() + ?: return copy(playbackState = PlaybackState.Ended) + + val newSegment = dataLoader.getItemRef(nextNode)?.segment + ?: return copy(playbackState = PlaybackState.Ended) + + segment.player.release() + + val playWhenReady = settings.readContinuously + + return play(newSegment, 0, playWhenReady, settings) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt new file mode 100644 index 0000000000..d6b5e2fc1c --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/SystemTtsEngine.kt @@ -0,0 +1,479 @@ +/* + * Copyright 2022 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class, ExperimentalReadiumApi::class) + +package org.readium.navigator.media.readaloud + +import android.annotation.SuppressLint +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import android.speech.tts.TextToSpeech +import android.speech.tts.TextToSpeech.ERROR_INVALID_REQUEST +import android.speech.tts.TextToSpeech.ERROR_NETWORK +import android.speech.tts.TextToSpeech.ERROR_NETWORK_TIMEOUT +import android.speech.tts.TextToSpeech.ERROR_NOT_INSTALLED_YET +import android.speech.tts.TextToSpeech.ERROR_OUTPUT +import android.speech.tts.TextToSpeech.ERROR_SERVICE +import android.speech.tts.TextToSpeech.ERROR_SYNTHESIS +import android.speech.tts.TextToSpeech.Engine +import android.speech.tts.TextToSpeech.OnInitListener +import android.speech.tts.TextToSpeech.QUEUE_ADD +import android.speech.tts.TextToSpeech.SUCCESS +import android.speech.tts.UtteranceProgressListener +import android.speech.tts.Voice as AndroidVoice +import android.speech.tts.Voice.QUALITY_HIGH +import android.speech.tts.Voice.QUALITY_LOW +import android.speech.tts.Voice.QUALITY_NORMAL +import android.speech.tts.Voice.QUALITY_VERY_HIGH +import android.speech.tts.Voice.QUALITY_VERY_LOW +import java.util.UUID +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.tryOrNull +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +public class SystemTtsEngineProvider( + private val context: Context, + private val maxConnectionRetries: Int = 3, +) : TtsEngineProvider { + + override suspend fun createEngineFactory(): TtsEngineFactory = + tryCreateEngineFactory(maxConnectionRetries) ?: NullTtsEngineFactory() + + private suspend fun tryCreateEngineFactory(maxRetries: Int): SystemTtsEngineFactory? { + suspend fun onFailure(): SystemTtsEngineFactory? = + if (maxRetries == 0) { + null + } else { + tryCreateEngineFactory(maxRetries - 1) + } + + val textToSpeech = initializeTextToSpeech(context) + ?: return onFailure() + + // Listing voices is not reliable. + val voices = tryOrNull { textToSpeech.voices } // throws on Nexus 4 + ?.filterNotNull() + ?.takeUnless { it.isEmpty() } + ?.associate { it.toTtsEngineVoice() to it } + ?: return onFailure() + + return SystemTtsEngineFactory( + context = context, + textToSpeech = textToSpeech, + fullVoices = voices, + maxConnectionRetries = maxConnectionRetries + ) + } +} + +@ExperimentalReadiumApi +public class SystemTtsEngineFactory internal constructor( + private val context: Context, + private val textToSpeech: TextToSpeech, + private val fullVoices: Map, + private val maxConnectionRetries: Int, +) : TtsEngineFactory { + + override val voices: Set = + fullVoices.keys + + override fun createPlaybackEngine( + voiceId: TtsVoice.Id, + utterances: List, + listener: PlaybackEngine.Listener, + ): PlaybackEngine { + val voice = fullVoices.keys.firstOrNull { it.id == voiceId } + requireNotNull(voice) { "Voice id $voiceId doesn't match any voice in $voices." } + + return SystemTtsEngine( + context = context, + engine = textToSpeech, + listener = listener, + systemVoice = fullVoices[voice]!!, + utterances = utterances, + maxConnectionRetries = maxConnectionRetries + ) + } +} + +/* + * On some Android implementations (i.e. on Oppo A9 2020 running Android 11), + * the TextToSpeech instance is often disconnected from the underlying service when the playback + * is paused and the app moves to the background. So we try to reset the TextToSpeech before + * actually returning an error. In the meantime, requests to the engine are stored in the adapter + * state. + */ + +/** + * Default [PlaybackEngine] implementation using Android's native text to speech engine. + */ +@ExperimentalReadiumApi +public class SystemTtsEngine internal constructor( + private val context: Context, + engine: TextToSpeech, + private val listener: PlaybackEngine.Listener, + private val systemVoice: AndroidVoice, + private val utterances: List, + private val maxConnectionRetries: Int, +) : PlaybackEngine { + + public companion object { + + /** + * Starts the activity to install additional voice data. + */ + @SuppressLint("QueryPermissionsNeeded") + public fun requestInstallVoice(context: Context) { + val intent = Intent() + .setAction(Engine.ACTION_INSTALL_TTS_DATA) + .setFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + + val availableActivities = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + context.packageManager.queryIntentActivities( + intent, + PackageManager.ResolveInfoFlags.of(0) + ) + } else { + context.packageManager.queryIntentActivities(intent, 0) + } + + if (availableActivities.isNotEmpty()) { + context.startActivity(intent) + } + } + } + + public sealed class Error( + override val message: String, + override val cause: org.readium.r2.shared.util.Error? = null, + ) : org.readium.r2.shared.util.Error { + + /** Denotes a generic operation failure. */ + public data object Unknown : Error("An unknown error occurred.") + + /** Denotes a failure caused by an invalid request. */ + public data object InvalidRequest : Error("Invalid request") + + /** Denotes a failure caused by a network connectivity problems. */ + public data object Network : Error("A network error occurred.") + + /** Denotes a failure caused by network timeout. */ + public data object NetworkTimeout : Error("Network timeout") + + /** Denotes a failure caused by an unfinished download of the voice data. */ + public data object NotInstalledYet : Error("Voice not installed yet.") + + /** Denotes a failure related to the output (audio device or a file). */ + public data object Output : Error("An error related to the output occurred.") + + /** Denotes a failure of a TTS service. */ + public data object Service : Error("An error occurred with the TTS service.") + + /** Denotes a failure of a TTS engine to synthesize the given input. */ + public data object Synthesis : Error("Synthesis failed.") + + /** + * Denotes the language data is missing. + * + * You can open the Android settings to install the missing data with: + * AndroidTtsEngine.requestInstallVoice(context) + */ + public data class LanguageMissingData(val language: Language) : + Error("Language data is missing.") + + /** + * Android's TTS error code. + * See https://developer.android.com/reference/android/speech/tts/TextToSpeech#ERROR + */ + public companion object { + internal fun fromNativeError(code: Int): Error = + when (code) { + ERROR_INVALID_REQUEST -> InvalidRequest + ERROR_NETWORK -> Network + ERROR_NETWORK_TIMEOUT -> NetworkTimeout + ERROR_NOT_INSTALLED_YET -> NotInstalledYet + ERROR_OUTPUT -> Output + ERROR_SERVICE -> Service + ERROR_SYNTHESIS -> Synthesis + else -> Unknown + } + } + } + + /** + * Represents a voice provided by the TTS engine which can speak an utterance. + * + * @param name Voice name + * @param language Language (and region) this voice belongs to. + * @param quality Voice quality. + * @param requiresNetwork Indicates whether using this voice requires an Internet connection. + */ + public data class Voice( + val name: String, + val language: Language, + val quality: Quality = Quality.Normal, + val requiresNetwork: Boolean = false, + ) : TtsVoice { + + override val id: TtsVoice.Id = + TtsVoice.Id("${SystemTtsEngine::class.qualifiedName}-$name}") + + override val languages: Set = + setOf(language) + + public enum class Quality { + Lowest, + Low, + Normal, + High, + Highest, + } + } + + private data class Request( + val id: Id, + val text: String, + ) { + + @JvmInline + value class Id(val value: String) + } + + private sealed class State { + + data class EngineAvailable( + val engine: TextToSpeech, + ) : State() + + data class WaitingForService( + var pendingRequest: Request?, + ) : State() + + data class Failure( + val error: Error, + ) : State() + } + + private val coroutineScope: CoroutineScope = MainScope() + + private var state: State = State.EngineAvailable(engine) + + private var isClosed: Boolean = false + + override var speed: Double = 1.0 + + override var pitch: Double = 1.0 + + override var itemToPlay: Int = 0 + + init { + setupListener(engine) + } + + override fun start() { + listener.onStartRequested(PlaybackEngine.PlaybackState.Playing) + doStart(itemToPlay) + } + + override fun resume() { + doStart(itemToPlay) + } + + private fun doStart(utteranceIndex: Int) { + check(!isClosed) { "Engine is closed." } + + val id = Request.Id(UUID.randomUUID().toString()) + val text = checkNotNull(utterances)[utteranceIndex] + val request = Request(id, text) + + when (val stateNow = state) { + is State.WaitingForService -> { + stateNow.pendingRequest = request + } + is State.Failure -> { + tryReconnect(request) + } + is State.EngineAvailable -> { + if (!speak(stateNow.engine, request)) { + cleanEngine(stateNow.engine) + tryReconnect(request) + } + } + } + } + + public override fun stop() { + doStop() + } + + override fun pause() { + doStop() + } + + private fun doStop() { + when (val stateNow = state) { + is State.EngineAvailable -> { + stateNow.engine.stop() + } + is State.Failure -> { + // Do nothing + } + is State.WaitingForService -> { + stateNow.pendingRequest = null + } + } + } + + public override fun release() { + if (isClosed) { + return + } + + isClosed = true + coroutineScope.cancel() + + when (val stateNow = state) { + is State.EngineAvailable -> { + cleanEngine(stateNow.engine) + } + is State.Failure -> { + // Do nothing + } + is State.WaitingForService -> { + // Do nothing + } + } + } + + private fun speak( + engine: TextToSpeech, + request: Request, + ): Boolean { + return engine.setupPitchAndSpeed() && + engine.setupVoice() && + engine.speak(request.text, QUEUE_ADD, null, request.id.value) == SUCCESS + } + + private fun setupListener(engine: TextToSpeech) { + engine.setOnUtteranceProgressListener(UtteranceListener(listener)) + } + + private fun onReconnectionSucceeded(engine: TextToSpeech) { + val previousState = state as State.WaitingForService + setupListener(engine) + state = State.EngineAvailable(engine) + if (isClosed) { + engine.shutdown() + } else { + previousState.pendingRequest?.let { speak(engine, it) } + } + } + + private fun onReconnectionFailed() { + val error = Error.Service + state = State.Failure(error) + // listener.onError(error) + } + + private fun tryReconnect(request: Request) { + state = State.WaitingForService(request) + coroutineScope.launch { + initializeTextToSpeech(context) + ?.let { onReconnectionSucceeded(it) } + ?: onReconnectionFailed() + } + } + + private fun cleanEngine(engine: TextToSpeech) { + engine.setOnUtteranceProgressListener(null) + engine.shutdown() + } + + private fun TextToSpeech.setupPitchAndSpeed(): Boolean { + if (setSpeechRate(speed.toFloat()) != SUCCESS) { + return false + } + + if (setPitch(pitch.toFloat()) != SUCCESS) { + return false + } + + return true + } + + private fun TextToSpeech.setupVoice(): Boolean { + return setVoice(systemVoice) == SUCCESS + } + + private class UtteranceListener( + private val listener: PlaybackEngine.Listener, + ) : UtteranceProgressListener() { + override fun onStart(utteranceId: String) { + } + + override fun onStop(utteranceId: String, interrupted: Boolean) { + } + + override fun onDone(utteranceId: String) { + listener.onPlaybackCompleted() + } + + @Deprecated( + "Deprecated in the interface", + ReplaceWith("onError(utteranceId, -1)"), + level = DeprecationLevel.ERROR + ) + override fun onError(utteranceId: String) { + onError(utteranceId, -1) + } + + override fun onError(utteranceId: String, errorCode: Int) { + // listener.onError(Error.fromNativeError(errorCode)) + } + + override fun onRangeStart(utteranceId: String, start: Int, end: Int, frame: Int) { + listener.onPlaybackProgressed(TtsEngineProgress(start until end)) + } + } +} + +private fun AndroidVoice.toTtsEngineVoice() = + SystemTtsEngine.Voice( + name = name, + language = Language(locale), + quality = when (quality) { + QUALITY_VERY_HIGH -> SystemTtsEngine.Voice.Quality.Highest + QUALITY_HIGH -> SystemTtsEngine.Voice.Quality.High + QUALITY_NORMAL -> SystemTtsEngine.Voice.Quality.Normal + QUALITY_LOW -> SystemTtsEngine.Voice.Quality.Low + QUALITY_VERY_LOW -> SystemTtsEngine.Voice.Quality.Lowest + else -> throw IllegalStateException("Unexpected voice quality.") + }, + requiresNetwork = isNetworkConnectionRequired + ) + +private suspend fun initializeTextToSpeech( + context: Context, +): TextToSpeech? { + val init = CompletableDeferred() + + val initListener = OnInitListener { status -> + init.complete(status == SUCCESS) + } + val engine = TextToSpeech(context, initListener) + return if (init.await()) engine else null +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt new file mode 100644 index 0000000000..baac6d3c7b --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudDefaults.kt @@ -0,0 +1,35 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.util.Language + +/** + * Default values for the ReadAloudNavigator. + * + * These values will be used as a last resort when no user preference takes precedence. + * + * @see ReadAloudPreferences + */ +@ExperimentalReadiumApi +public data class ReadAloudDefaults( + val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null, + val voices: Map? = null, + val escapableRoles: Set? = null, + val skippableRoles: Set? = null, + val readContinuously: Boolean? = null, +) { + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt new file mode 100644 index 0000000000..9d65c763b2 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferences.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import kotlinx.serialization.Serializable +import org.readium.navigator.common.Preferences +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.util.Language + +/** + * Preferences for the the ReadAloudNavigator. + * + * @param language Language of the publication content. + * @param pitch Playback pitch rate. + * @param speed Playback speed rate. + * @param voices Map of preferred voices for specific languages. + * @param escapableRoles Roles that will be considered as escapable. + * @param skippableRoles Roles that will be considered as skippable. + * @param readContinuously Do not pause after reading each content item. + */ +@ExperimentalReadiumApi +@Serializable +public data class ReadAloudPreferences( + val language: Language? = null, + val pitch: Double? = null, + val speed: Double? = null, + val voices: Map? = null, + val escapableRoles: Set? = null, + val skippableRoles: Set? = null, + val readContinuously: Boolean? = true, +) : Preferences { + + init { + require(pitch == null || pitch > 0) + require(speed == null || speed > 0) + } + + public override fun plus(other: ReadAloudPreferences): ReadAloudPreferences = + ReadAloudPreferences( + language = other.language ?: language, + pitch = other.pitch ?: pitch, + speed = other.speed ?: speed, + voices = other.voices ?: voices, + escapableRoles = other.escapableRoles ?: escapableRoles, + skippableRoles = other.skippableRoles ?: skippableRoles, + readContinuously = other.readContinuously ?: readContinuously + ) +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt new file mode 100644 index 0000000000..2483d3f1bf --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudPreferencesEditor.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class) + +package org.readium.navigator.media.readaloud.preferences + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import org.readium.navigator.common.PreferencesEditor +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.navigator.extensions.format +import org.readium.r2.navigator.preferences.DoubleIncrement +import org.readium.r2.navigator.preferences.Preference +import org.readium.r2.navigator.preferences.PreferenceDelegate +import org.readium.r2.navigator.preferences.RangePreference +import org.readium.r2.navigator.preferences.RangePreferenceDelegate +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.extensions.mapStateIn +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +/** + * Editor for a set of [ReadAloudPreferences]. + * + * Use [ReadAloudPreferencesEditor] to assist you in building a preferences user interface or modifying + * existing preferences. It includes rules for adjusting preferences, such as the supported values + * or ranges. + */ +@ExperimentalReadiumApi +public class ReadAloudPreferencesEditor( + initialPreferences: ReadAloudPreferences, + publicationMetadata: Metadata, + defaults: ReadAloudDefaults, +) : PreferencesEditor { + + private data class State( + val preferences: ReadAloudPreferences, + val settings: ReadAloudSettings, + ) + + private val coroutineScope: CoroutineScope = + MainScope() + private val settingsResolver: ReadAloudSettingsResolver = + ReadAloudSettingsResolver(publicationMetadata, defaults) + + private var state: MutableStateFlow = + MutableStateFlow(initialPreferences.toState()) + + override val preferences: ReadAloudPreferences + get() = state.value.preferences + + override val settings: ReadAloudSettings + get() = state.value.settings + + public val preferencesState: StateFlow = + state.mapStateIn(coroutineScope) { it.preferences } + + override fun clear() { + updateValues { ReadAloudPreferences() } + } + + public val language: Preference = + PreferenceDelegate( + getValue = { preferences.language }, + getEffectiveValue = { state.value.settings.language }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(language = value) } } + ) + + public val pitch: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.pitch }, + getEffectiveValue = { state.value.settings.pitch }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(pitch = value) } }, + supportedRange = 0.1..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(2)}x" } + ) + + public val speed: RangePreference = + RangePreferenceDelegate( + getValue = { preferences.speed }, + getEffectiveValue = { state.value.settings.speed }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(speed = value) } }, + supportedRange = 0.1..Double.MAX_VALUE, + progressionStrategy = DoubleIncrement(0.1), + valueFormatter = { "${it.format(2)}x" } + ) + + public val voices: Preference> = + PreferenceDelegate( + getValue = { preferences.voices }, + getEffectiveValue = { state.value.settings.voices }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(voices = value) } } + ) + + public val readContinuously: Preference = + PreferenceDelegate( + getValue = { preferences.readContinuously }, + getEffectiveValue = { state.value.settings.readContinuously }, + getIsEffective = { true }, + updateValue = { value -> updateValues { it.copy(readContinuously = value) } } + ) + + private fun updateValues(updater: (ReadAloudPreferences) -> ReadAloudPreferences) { + val newPreferences = updater(preferences) + state.value = newPreferences.toState() + } + + private fun ReadAloudPreferences.toState(): State { + return State( + preferences = this, + settings = settingsResolver.settings(this), + ) + } +} diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt new file mode 100644 index 0000000000..e2131299a5 --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettings.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import org.readium.navigator.common.Settings +import org.readium.navigator.media.readaloud.TtsVoice +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.util.Language + +@ExperimentalReadiumApi +public data class ReadAloudSettings( + val language: Language, + val overrideContentLanguage: Boolean, + val pitch: Double, + val speed: Double, + val voices: Map, + val escapableRoles: Set, + val skippableRoles: Set, + val readContinuously: Boolean, +) : Settings diff --git a/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt new file mode 100644 index 0000000000..a982ab201f --- /dev/null +++ b/readium/navigators/media/readaloud/src/main/java/org/readium/navigator/media/readaloud/preferences/ReadAloudSettingsResolver.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.navigator.media.readaloud.preferences + +import java.util.Locale +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.ASIDE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.BIBLIOGRAPHY +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.CELL +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.ENDNOTES +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.FIGURE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.FOOTNOTE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LANDMARKS +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LIST +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LIST_ITEM +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOA +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOI +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOT +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.LOV +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.NOTEREF +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.PAGEBREAK +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.PULLQUOTE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.ROW +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.TABLE +import org.readium.r2.shared.guided.GuidedNavigationRole.Companion.TOC +import org.readium.r2.shared.publication.Metadata +import org.readium.r2.shared.util.Language + +@OptIn(ExperimentalReadiumApi::class) +internal class ReadAloudSettingsResolver( + private val metadata: Metadata, + private val defaults: ReadAloudDefaults, +) { + + fun settings(preferences: ReadAloudPreferences): ReadAloudSettings { + val language = preferences.language + ?: metadata.language + ?: defaults.language + ?: Language(Locale.getDefault()) + + val skippableRoles: Set = + preferences.skippableRoles + ?: defaults.skippableRoles + ?: setOf( + ASIDE, BIBLIOGRAPHY, ENDNOTES, FOOTNOTE, NOTEREF, PULLQUOTE, + LANDMARKS, LOA, LOI, LOT, LOV, PAGEBREAK, TOC + ) + + val escapableRoles: Set = + preferences.escapableRoles + ?: defaults.escapableRoles + ?: setOf(ASIDE, FIGURE, LIST, LIST_ITEM, TABLE, ROW, CELL) + + val languagesWithPreferredVoice = + preferences.voices.orEmpty().keys.map { it.removeRegion() } + + val filteredDefaultVoices = defaults.voices.orEmpty() + .filter { it.key.removeRegion() !in languagesWithPreferredVoice } + + val voices = filteredDefaultVoices + preferences.voices.orEmpty() + + return ReadAloudSettings( + language = language, + voices = voices, + pitch = preferences.pitch ?: defaults.pitch ?: 1.0, + speed = preferences.speed ?: defaults.speed ?: 1.0, + overrideContentLanguage = preferences.language != null, + escapableRoles = escapableRoles, + skippableRoles = skippableRoles, + readContinuously = preferences.readContinuously ?: defaults.readContinuously ?: true + ) + } +} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt new file mode 100644 index 0000000000..e05c38fdb9 --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationDocument.kt @@ -0,0 +1,87 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.guided + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Link +import org.readium.r2.shared.util.Language +import org.readium.r2.shared.util.Url + +/** + * A guided navigation document. + */ +@ExperimentalReadiumApi +public data class GuidedNavigationDocument( + val links: List = emptyList(), + val guided: List, +) + +/** + * A guided navigation object. + */ +@ExperimentalReadiumApi +public data class GuidedNavigationObject( + val children: List = emptyList(), + val roles: Set = emptySet(), + val refs: Set = emptySet(), + val text: GuidedNavigationText? = null, +) + +/** + * A string containing some SSML markup. + */ +@ExperimentalReadiumApi +@JvmInline +public value class SsmlString(public val value: String) + +/** + * Text holder for a guided navigation object. + */ +@ExperimentalReadiumApi +public data class GuidedNavigationText( + val plain: String?, + val ssml: SsmlString? = null, + val language: Language? = null, +) { + init { + require(plain != null || ssml != null) + require(plain == null || plain.isNotEmpty()) + require(ssml == null || ssml.value.isNotEmpty()) + } +} + +/** + * A reference to external content. + */ +@ExperimentalReadiumApi +public sealed interface GuidedNavigationRef { + public val url: Url +} + +/** + * A reference to external text content. + */ +@ExperimentalReadiumApi +public data class GuidedNavigationTextRef( + override val url: Url, +) : GuidedNavigationRef + +/** + * A reference to external image content. + */ +@ExperimentalReadiumApi +public data class GuidedNavigationImageRef( + override val url: Url, +) : GuidedNavigationRef + +/** + * A reference to external audio content. + */ +@ExperimentalReadiumApi +public data class GuidedNavigationAudioRef( + override val url: Url, +) : GuidedNavigationRef diff --git a/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt new file mode 100644 index 0000000000..eac467dedb --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/guided/GuidedNavigationRole.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.guided + +import kotlinx.serialization.Serializable + +/** + * A role usable in a guided navigation object. + */ +@Serializable +@JvmInline +public value class GuidedNavigationRole(public val value: String) { + + public companion object { + + /* + * Inherited from HTML and/or ARIA + */ + + public val ASIDE: GuidedNavigationRole = GuidedNavigationRole("aside") + public val CELL: GuidedNavigationRole = GuidedNavigationRole("cell") + public val DEFINITION: GuidedNavigationRole = GuidedNavigationRole("definition") + public val FIGURE: GuidedNavigationRole = GuidedNavigationRole("figure") + public val LIST: GuidedNavigationRole = GuidedNavigationRole("list") + public val LIST_ITEM: GuidedNavigationRole = GuidedNavigationRole("listItem") + public val ROW: GuidedNavigationRole = GuidedNavigationRole("row") + public val TABLE: GuidedNavigationRole = GuidedNavigationRole("table") + public val TERM: GuidedNavigationRole = GuidedNavigationRole("term") + + /* + * Inherited from DPUB ARIA 1.0 + */ + + public val ABSTRACT: GuidedNavigationRole = GuidedNavigationRole("abstract") + public val ACKNOWLEDGMENTS: GuidedNavigationRole = GuidedNavigationRole("acknowledgments") + public val AFTERWORD: GuidedNavigationRole = GuidedNavigationRole("afterword") + public val APPENDIX: GuidedNavigationRole = GuidedNavigationRole("appendix") + public val BACKLINK: GuidedNavigationRole = GuidedNavigationRole("backlink") + public val BIBLIOGRAPHY: GuidedNavigationRole = GuidedNavigationRole("bibliography") + public val BIBLIOREF: GuidedNavigationRole = GuidedNavigationRole("biblioref") + public val CHAPTER: GuidedNavigationRole = GuidedNavigationRole("chapter") + public val COLOPHON: GuidedNavigationRole = GuidedNavigationRole("colophon") + public val CONCLUSION: GuidedNavigationRole = GuidedNavigationRole("conclusion") + public val COVER: GuidedNavigationRole = GuidedNavigationRole("cover") + public val CREDIT: GuidedNavigationRole = GuidedNavigationRole("credit") + public val CREDITS: GuidedNavigationRole = GuidedNavigationRole("credits") + public val DEDICATION: GuidedNavigationRole = GuidedNavigationRole("dedication") + public val ENDNOTES: GuidedNavigationRole = GuidedNavigationRole("endnotes") + public val EPIGRAPH: GuidedNavigationRole = GuidedNavigationRole("epigraph") + public val EPILOGUE: GuidedNavigationRole = GuidedNavigationRole("epilogue") + public val ERRATA: GuidedNavigationRole = GuidedNavigationRole("errata") + public val EXAMPLE: GuidedNavigationRole = GuidedNavigationRole("example") + public val FOOTNOTE: GuidedNavigationRole = GuidedNavigationRole("footnote") + public val GLOSSARY: GuidedNavigationRole = GuidedNavigationRole("glossary") + public val GLOSSREF: GuidedNavigationRole = GuidedNavigationRole("glossref") + public val INDEX: GuidedNavigationRole = GuidedNavigationRole("index") + public val INTRODUCTION: GuidedNavigationRole = GuidedNavigationRole("introduction") + public val NOTEREF: GuidedNavigationRole = GuidedNavigationRole("noteref") + public val NOTICE: GuidedNavigationRole = GuidedNavigationRole("notice") + public val PAGEBREAK: GuidedNavigationRole = GuidedNavigationRole("pagebreak") + public val PAGELIST: GuidedNavigationRole = GuidedNavigationRole("page-list") + public val PART: GuidedNavigationRole = GuidedNavigationRole("part") + public val PREFACE: GuidedNavigationRole = GuidedNavigationRole("preface") + public val PROLOGUE: GuidedNavigationRole = GuidedNavigationRole("prologue") + public val PULLQUOTE: GuidedNavigationRole = GuidedNavigationRole("pullquote") + public val QNA: GuidedNavigationRole = GuidedNavigationRole("qna") + public val SUBTITLE: GuidedNavigationRole = GuidedNavigationRole("subtitle") + public val TIP: GuidedNavigationRole = GuidedNavigationRole("tip") + public val TOC: GuidedNavigationRole = GuidedNavigationRole("toc") + + /* + * Inherited from EPUB 3 Structural Semantics Vocabulary 1.1 + */ + + public val LANDMARKS: GuidedNavigationRole = GuidedNavigationRole("landmarks") + public val LOA: GuidedNavigationRole = GuidedNavigationRole("loa") + public val LOI: GuidedNavigationRole = GuidedNavigationRole("loi") + public val LOT: GuidedNavigationRole = GuidedNavigationRole("lot") + public val LOV: GuidedNavigationRole = GuidedNavigationRole("lov") + } +} diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt new file mode 100644 index 0000000000..4808bbd43f --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/epub/Services.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.publication.epub + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.PublicationServicesHolder +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.publication.services.GuidedNavigationService + +/** + * Provides a list of guided navigation documents mimicking media overlays available in Publication. + */ +@ExperimentalReadiumApi +public interface MediaOverlaysService : GuidedNavigationService + +/** + * Returns an iterator providing access to all the guided navigation documents mimicking media + * overlays of the publication. + */ +@ExperimentalReadiumApi +public fun Publication.mediaOverlaysIterator(): GuidedNavigationIterator? = + mediaOverlaysService?.iterator() + +@ExperimentalReadiumApi +private val PublicationServicesHolder.mediaOverlaysService: GuidedNavigationService? + get() { + findService(MediaOverlaysService::class)?.let { return it } + return null + } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt new file mode 100644 index 0000000000..1d31fabc2c --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/publication/services/GuidedNavigationService.kt @@ -0,0 +1,63 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.publication.services + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.PublicationServicesHolder +import org.readium.r2.shared.util.Closeable +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.data.ReadError + +/** + * Provides a list of Guided Navigation documents for a Publication. + */ +@ExperimentalReadiumApi +public interface GuidedNavigationService : Publication.Service { + + public fun iterator(): GuidedNavigationIterator +} + +/** + * Iterator providing access to all guided navigation documents of a Publication. + */ +@ExperimentalReadiumApi +public interface GuidedNavigationIterator : Closeable { + + /** + * Prepares the next guided navigation document for retrieval by the invocation of next. + * + * Does nothing if the the end has been reached. + */ + public suspend operator fun hasNext(): Boolean + + /** + * Retrieves the next guided navigation document, prepared by the preceding call to [hasNext], + * or throws an IllegalStateException if hasNext was not invoked. + */ + public suspend operator fun next(): Try + + /** + * Closes any resources allocated by the iterator. + */ + override fun close() {} +} + +/** + * Returns an iterator providing access to all the guided navigation documents of the publication. + */ +@ExperimentalReadiumApi +public fun Publication.guidedNavigationIterator(): GuidedNavigationIterator? = + guidedNavigationService?.iterator() + +@OptIn(ExperimentalReadiumApi::class) +private val PublicationServicesHolder.guidedNavigationService: GuidedNavigationService? + get() { + findService(GuidedNavigationService::class)?.let { return it } + return null + } diff --git a/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt b/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt new file mode 100644 index 0000000000..5c998c7122 --- /dev/null +++ b/readium/shared/src/main/java/org/readium/r2/shared/util/TemporalFragmentParser.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.util + +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +/** + * Temporal dimension parser for + * [Media Fragment specification](https://www.w3.org/TR/media-frags/#naming-time). + * + * Supports only Normal Play Time specified as seconds at the moment. + */ +public object TemporalFragmentParser { + + public fun parse(value: String): TimeInterval? { + if (!value.startsWith("t=")) { + return null + } + + val nptValue = value.removePrefix("t=").removePrefix("npt:") + return parseNormalPlayTime(nptValue) + } + + private fun parseNormalPlayTime(value: String): TimeInterval? { + val components = value.split(",", limit = 2) + + val startOffset = components.getOrNull(0) + ?.toDoubleOrNull() + ?.seconds + + val endOffset = components.getOrNull(1) + ?.toDoubleOrNull() + ?.seconds + + return TimeInterval(startOffset, endOffset) + } +} + +public data class TimeInterval( + val start: Duration?, + val end: Duration?, +) diff --git a/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt b/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt new file mode 100644 index 0000000000..eecfff52b5 --- /dev/null +++ b/readium/shared/src/test/java/org/readium/r2/shared/util/TemporalFragmentParserTest.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2023 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +package org.readium.r2.shared.util + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.time.Duration.Companion.seconds + +class TemporalFragmentParserTest { + + @Test + fun `fragment which is not temporal is rejected`() { + assertNull(TemporalFragmentParser.parse("htmlId")) + } + + @Test + fun `start only in seconds is accepted`() { + assertEquals( + TimeInterval(start = 4.seconds, null), + TemporalFragmentParser.parse("t=4") + ) + } + + @Test + fun `end only in seconds is accepted`() { + assertEquals( + TimeInterval(start = null, end = 40.seconds), + TemporalFragmentParser.parse("t=,40") + ) + } + + @Test + fun `start and end in seconds are accepted`() { + assertEquals( + TimeInterval(4.seconds, 60.seconds), + TemporalFragmentParser.parse("t=4,60") + ) + } + + @Test + fun `npt prefix is accepted`() { + assertEquals( + TimeInterval(40.seconds, null), + TemporalFragmentParser.parse("t=npt:40") + ) + } + + @Test + fun `floating point values are accepted`() { + assertEquals( + TimeInterval(40.5.seconds, 83.235.seconds), + TemporalFragmentParser.parse("t=40.500,83.235") + ) + } +} diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt index 49ba1cd52b..f82a4c2d72 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/Constants.kt @@ -27,7 +27,7 @@ internal object Vocabularies { const val MEDIA = "http://www.idpf.org/epub/vocab/overlays/#" const val RENDITION = "http://www.idpf.org/vocab/rendition/#" - const val TYPE = "http://idpf.org/epub/vocab/structure/#" // this is a guessed value + const val TYPE = "http://idpf.org/epub/vocab/structure/#" const val DCTERMS = "http://purl.org/dc/terms/" const val A11Y = "http://www.idpf.org/epub/vocab/package/a11y/#" diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt index 86c8c296e5..fb52bc27e4 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/EpubParser.kt @@ -14,6 +14,7 @@ import org.readium.r2.shared.publication.Link import org.readium.r2.shared.publication.Publication import org.readium.r2.shared.publication.encryption.Encryption import org.readium.r2.shared.publication.epub.EpubEncryptionParser +import org.readium.r2.shared.publication.epub.MediaOverlaysService import org.readium.r2.shared.publication.services.content.DefaultContentService import org.readium.r2.shared.publication.services.content.iterators.HtmlResourceContentIterator import org.readium.r2.shared.publication.services.search.StringSearchService @@ -86,13 +87,15 @@ public class EpubParser( val encryptionData = parseEncryptionData(asset.container) - val manifest = ManifestAdapter( + val (manifest, mediaOverlays) = ManifestAdapter( packageDocument = packageDocument, navigationData = parseNavigationData(packageDocument, asset.container), encryptionData = encryptionData, displayOptions = parseDisplayOptions(asset.container) ).adapt() + val smils = mediaOverlays.map { it.url() } + var container = asset.container manifest.metadata.identifier?.let { id -> val deobfuscator = EpubDeobfuscator(id, encryptionData) @@ -110,7 +113,11 @@ public class EpubParser( HtmlResourceContentIterator.Factory() ) ) - ) + ).also { + if (smils.isNotEmpty()) { + it[MediaOverlaysService::class] = SmilBasedMediaOverlaysService.createFactory(smils) + } + } ) return Try.success(builder) diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt index 61a979498c..c50e38344a 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ManifestAdapter.kt @@ -24,10 +24,15 @@ internal class ManifestAdapter( private val encryptionData: Map = emptyMap(), private val displayOptions: Map = emptyMap(), ) { + + data class Result( + val manifest: Manifest, + val mediaOverlays: List, + ) private val epubVersion = packageDocument.epubVersion private val spine = packageDocument.spine - fun adapt(): Manifest { + fun adapt(): Result { // Compute metadata val metadata = MetadataAdapter( epubVersion, @@ -37,7 +42,7 @@ internal class ManifestAdapter( ).adapt(packageDocument.metadata) // Compute links - val (readingOrder, resources) = ResourceAdapter( + val (readingOrder, resources, mediaOverlays) = ResourceAdapter( packageDocument.spine, packageDocument.manifest, encryptionData, @@ -70,7 +75,7 @@ internal class ManifestAdapter( } // Build Publication object - return Manifest( + val manifest = Manifest( metadata = metadata.metadata, links = emptyList(), readingOrder = readingOrder, @@ -78,5 +83,7 @@ internal class ManifestAdapter( tableOfContents = toc, subcollections = subcollections.toMap() ) + + return Result(manifest = manifest, mediaOverlays = mediaOverlays) } } diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt new file mode 100644 index 0000000000..02f71ac4bd --- /dev/null +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/MediaOverlaysService.kt @@ -0,0 +1,242 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(InternalReadiumApi::class, ExperimentalReadiumApi::class) + +package org.readium.r2.streamer.parser.epub + +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationRole +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.publication.Publication +import org.readium.r2.shared.publication.epub.MediaOverlaysService +import org.readium.r2.shared.publication.services.GuidedNavigationIterator +import org.readium.r2.shared.util.Try +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.data.Container +import org.readium.r2.shared.util.data.ReadError +import org.readium.r2.shared.util.data.decodeXml +import org.readium.r2.shared.util.data.readDecodeOrElse +import org.readium.r2.shared.util.fromEpubHref +import org.readium.r2.shared.util.resource.Resource +import org.readium.r2.shared.util.use +import org.readium.r2.shared.util.xml.ElementNode + +/** + * A GuidedNavigationService producing guided navigation documents from media overlays SMIL files. + */ +@ExperimentalReadiumApi +public class SmilBasedMediaOverlaysService( + private val smilFiles: List, + private val container: Container, +) : MediaOverlaysService { + + override fun iterator(): GuidedNavigationIterator { + return MediaOverlaysIterator(smilFiles, container) + } + + public companion object { + + public fun createFactory( + smilFiles: List, + ): ( + Publication.Service.Context, + ) -> SmilBasedMediaOverlaysService = + { context -> + SmilBasedMediaOverlaysService( + smilFiles = smilFiles, + container = context.container + ) + } + } +} + +private class MediaOverlaysIterator( + private val smils: List, + private val container: Container, +) : GuidedNavigationIterator { + + private var lastRead: Int = -1 + private var nextDoc: Try? = null + + override suspend fun hasNext(): Boolean { + if (lastRead >= smils.size - 1) { + return false + } + + lastRead++ + nextDoc = parse(smils[lastRead]) + return true + } + + private suspend fun parse(href: Url): Try { + val resource = container[href] + ?: throw IllegalStateException("Cannot find resource in the container: $href") + + val xmlDoc = resource.use { res -> + res.readDecodeOrElse( + decode = { it.decodeXml() }, + recover = { return Try.failure(it) } + ) + } + + return SmilParser.parse(xmlDoc, href) + ?.let { Try.success(it) } + ?: Try.failure(ReadError.Decoding("Cannot parse SMIL file $href")) + } + + override suspend fun next(): Try { + return nextDoc ?: throw IllegalStateException("next was called before hasNext.") + } +} + +internal object SmilParser { + + fun parse(document: ElementNode, fileHref: Url): GuidedNavigationDocument? { + val docPrefixes = document.getAttrNs("prefix", Namespaces.OPS) + ?.let { parsePrefixes(it) }.orEmpty() + val prefixMap = CONTENT_RESERVED_PREFIXES + docPrefixes // prefix element overrides reserved prefixes + val body = document.getFirst("body", Namespaces.SMIL) ?: return null + return parseSeq(body, fileHref, prefixMap) + ?.let { GuidedNavigationDocument(links = emptyList(), guided = it.children) } + } + + private fun parseSeq( + node: ElementNode, + filePath: Url, + prefixMap: Map, + ): GuidedNavigationObject? { + val roles = parseRoles(node, prefixMap) + val children: MutableList = mutableListOf() + for (child in node.getAll()) { + if (child.name == "par" && child.namespace == Namespaces.SMIL) { + parsePar(child, filePath, prefixMap)?.let { children.add(it) } + } else if (child.name == "seq" && child.namespace == Namespaces.SMIL) { + parseSeq(child, filePath, prefixMap)?.let { children.add(it) } + } + } + + return GuidedNavigationObject(children = children, roles = roles, refs = emptySet(), text = null) + } + + private fun parsePar( + node: ElementNode, + filePath: Url, + prefixMap: Map, + ): GuidedNavigationObject? { + val roles = parseRoles(node, prefixMap) + val text = node.getFirst("text", Namespaces.SMIL) + ?.getAttr("src") + ?.let { Url.fromEpubHref(it) } + ?: return null + val audio = node.getFirst("audio", Namespaces.SMIL) + ?.let { audioNode -> + val src = audioNode.getAttr("src") + ?.let { Url.fromEpubHref(it) } + ?.toString() + ?: return null + val begin = audioNode.getAttr("clipBegin") + ?.let { ClockValueParser.parse(it) } + ?: "" + val end = audioNode.getAttr("clipEnd") + ?.let { ClockValueParser.parse(it) } + ?: "" + Url("$src#t=$begin,$end") + } + + val refs = setOfNotNull( + GuidedNavigationTextRef(filePath.resolve(text)), + audio?.let { GuidedNavigationAudioRef(filePath.resolve(it)) } + ) + + return GuidedNavigationObject( + children = emptyList(), + text = null, + refs = refs, + roles = roles + ) + } + + private fun parseRoles( + node: ElementNode, + prefixMap: Map, + ): Set { + val typeAttr = node.getAttrNs("type", Namespaces.OPS) ?: "" + val candidates = if (typeAttr.isNotEmpty()) { + parseProperties(typeAttr).map { + resolveProperty( + it, + prefixMap, + DEFAULT_VOCAB.TYPE + ) + }.toSet() + } else { + emptySet() + } + + return candidates.mapNotNull { + when (it.removePrefix(Vocabularies.TYPE)) { + "aside" -> GuidedNavigationRole.ASIDE + "table-cell" -> GuidedNavigationRole.CELL + "glossdef" -> GuidedNavigationRole.DEFINITION + "figure" -> GuidedNavigationRole.FIGURE + "list" -> GuidedNavigationRole.LIST + "list-item" -> GuidedNavigationRole.LIST_ITEM + "table-row" -> GuidedNavigationRole.ROW + "table" -> GuidedNavigationRole.TABLE + "glossterm" -> GuidedNavigationRole.TERM + + "abstract" -> GuidedNavigationRole.ABSTRACT + "acknowledgments" -> GuidedNavigationRole.ACKNOWLEDGMENTS + "afterword" -> GuidedNavigationRole.AFTERWORD + "appendix" -> GuidedNavigationRole.APPENDIX + "backlink" -> GuidedNavigationRole.BACKLINK + "bibliography" -> GuidedNavigationRole.BIBLIOGRAPHY + "biblioref" -> GuidedNavigationRole.BIBLIOREF + "chapter" -> GuidedNavigationRole.CHAPTER + "colophon" -> GuidedNavigationRole.COLOPHON + "conclusion" -> GuidedNavigationRole.CONCLUSION + "cover" -> GuidedNavigationRole.COVER + "credit" -> GuidedNavigationRole.CREDIT + "credits" -> GuidedNavigationRole.CREDITS + "dedication" -> GuidedNavigationRole.DEDICATION + "endnotes" -> GuidedNavigationRole.ENDNOTES + "epigraph" -> GuidedNavigationRole.EPIGRAPH + "epilogue" -> GuidedNavigationRole.EPILOGUE + "errata" -> GuidedNavigationRole.ERRATA + "example" -> GuidedNavigationRole.EXAMPLE + "footnote" -> GuidedNavigationRole.FOOTNOTE + "glossary" -> GuidedNavigationRole.GLOSSARY + "glossref" -> GuidedNavigationRole.GLOSSREF + "index" -> GuidedNavigationRole.INDEX + "introduction" -> GuidedNavigationRole.INTRODUCTION + "noteref" -> GuidedNavigationRole.NOTEREF + "notice" -> GuidedNavigationRole.NOTICE + "pagebreak" -> GuidedNavigationRole.PAGEBREAK + "page-list" -> GuidedNavigationRole.PAGELIST + "part" -> GuidedNavigationRole.PART + "preface" -> GuidedNavigationRole.PREFACE + "prologue" -> GuidedNavigationRole.PROLOGUE + "pullquote" -> GuidedNavigationRole.PULLQUOTE + "qna" -> GuidedNavigationRole.QNA + "subtitle" -> GuidedNavigationRole.SUBTITLE + "tip" -> GuidedNavigationRole.TIP + "toc" -> GuidedNavigationRole.TOC + + "landmarks" -> GuidedNavigationRole.LANDMARKS + "loa" -> GuidedNavigationRole.LOA + "loi" -> GuidedNavigationRole.LOI + "lot" -> GuidedNavigationRole.LOT + "lov" -> GuidedNavigationRole.LOV + else -> null + } + }.toSet() + } +} diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt index 16af59e154..2f352a49b0 100644 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt +++ b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/ResourceAdapter.kt @@ -27,6 +27,7 @@ internal class ResourceAdapter( data class Links( val readingOrder: List, val resources: List, + val mediaOverlays: List, ) @Suppress("Unchecked_cast") @@ -47,9 +48,11 @@ internal class ResourceAdapter( } } val readingOrderAllIds = computeIdsWithFallbacks(readingOrderIds) - val resourceItems = manifest.filterNot { it.id in readingOrderAllIds } + val smilIds = manifest.mapNotNull { it.mediaOverlay } + val smils = smilIds.mapNotNull { id -> itemById[id]?.let { item -> computeLink(item) } } + val resourceItems = manifest.filterNot { it.id in readingOrderAllIds || it.id in smilIds } val resources = resourceItems.map { computeLink(it) } - return Links(readingOrder, resources) + return Links(readingOrder, resources, smils) } /** Recursively find the ids contained in fallback chains of items with [ids]. */ diff --git a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt b/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt deleted file mode 100644 index 0f43398f44..0000000000 --- a/readium/streamer/src/main/java/org/readium/r2/streamer/parser/epub/SmilParser.kt +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2022 Readium Foundation. All rights reserved. - * Use of this source code is governed by the BSD-style license - * available in the top-level LICENSE file of the project. - */ - -@file:OptIn(InternalReadiumApi::class) - -package org.readium.r2.streamer.parser.epub - -import org.readium.r2.shared.DelicateReadiumApi -import org.readium.r2.shared.InternalReadiumApi -import org.readium.r2.shared.MediaOverlayNode -import org.readium.r2.shared.MediaOverlays -import org.readium.r2.shared.util.Url -import org.readium.r2.shared.util.fromEpubHref -import org.readium.r2.shared.util.xml.ElementNode - -internal object SmilParser { - /* According to https://www.w3.org/publishing/epub3/epub-mediaoverlays.html#sec-overlays-content-conf - a Media Overlay Document MAY refer to more than one EPUB Content Document - This might be possible only using Canonical Fragment Identifiers - since the unique body and each seq element MUST reference - one EPUB Content Document by means of its attribute epub:textref - */ - - fun parse(document: ElementNode, filePath: Url): MediaOverlays? { - val body = document.getFirst("body", Namespaces.SMIL) ?: return null - return parseSeq(body, filePath)?.let { MediaOverlays(it) } - } - - @OptIn(DelicateReadiumApi::class) - private fun parseSeq(node: ElementNode, filePath: Url): List? { - val children: MutableList = mutableListOf() - for (child in node.getAll()) { - if (child.name == "par" && child.namespace == Namespaces.SMIL) { - parsePar(child, filePath)?.let { children.add(it) } - } else if (child.name == "seq" && child.namespace == Namespaces.SMIL) { - parseSeq(child, filePath)?.let { children.addAll(it) } - } - } - - /* No wrapping media overlay can be created unless: - - all child media overlays reference the same audio file - - the seq element has an textref attribute (this is mandatory according to the EPUB spec) - */ - val textref = node.getAttrNs("textref", Namespaces.OPS) - ?.let { Url.fromEpubHref(it) } - val audioFiles = children.mapNotNull(MediaOverlayNode::audioFile) - return if (textref != null && audioFiles.distinct().size == 1) { // hierarchy - val normalizedTextref = filePath.resolve(textref) - listOf(mediaOverlayFromChildren(normalizedTextref, children)) - } else { - children - } - } - - private fun parsePar(node: ElementNode, filePath: Url): MediaOverlayNode? { - val text = node.getFirst("text", Namespaces.SMIL) - ?.getAttr("src") - ?.let { Url.fromEpubHref(it) } - ?: return null - val audio = node.getFirst("audio", Namespaces.SMIL) - ?.let { audioNode -> - val src = audioNode.getAttr("src") - val begin = audioNode.getAttr("clipBegin")?.let { ClockValueParser.parse(it) } ?: "" - val end = audioNode.getAttr("clipEnd")?.let { ClockValueParser.parse(it) } ?: "" - "$src#t=$begin,$end" - } - ?.let { Url.fromEpubHref(it) } - - return MediaOverlayNode( - filePath.resolve(text), - audio?.let { filePath.resolve(audio) } - ) - } - - private fun mediaOverlayFromChildren(text: Url, children: List): MediaOverlayNode { - require(children.isNotEmpty() && children.mapNotNull { it.audioFile }.distinct().size <= 1) - val audioChildren = children.mapNotNull { if (it.audioFile != null) it else null } - val file = audioChildren.first().audioFile - val start = audioChildren.first().clip.start ?: "" - val end = audioChildren.last().clip.end ?: "" - val audio = Url.fromEpubHref("$file#t=$start,$end") - return MediaOverlayNode(text, audio, children, listOf("section")) - } -} diff --git a/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt new file mode 100644 index 0000000000..a20411e241 --- /dev/null +++ b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/MediaOverlaysServiceTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2025 Readium Foundation. All rights reserved. + * Use of this source code is governed by the BSD-style license + * available in the top-level LICENSE file of the project. + */ + +@file:OptIn(ExperimentalReadiumApi::class, InternalReadiumApi::class) + +package org.readium.r2.streamer.parser.epub + +import java.io.File +import kotlin.reflect.KClass +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlinx.coroutines.runBlocking +import org.junit.runner.RunWith +import org.readium.r2.shared.ExperimentalReadiumApi +import org.readium.r2.shared.InternalReadiumApi +import org.readium.r2.shared.guided.GuidedNavigationAudioRef +import org.readium.r2.shared.guided.GuidedNavigationDocument +import org.readium.r2.shared.guided.GuidedNavigationObject +import org.readium.r2.shared.guided.GuidedNavigationRef +import org.readium.r2.shared.guided.GuidedNavigationTextRef +import org.readium.r2.shared.util.RelativeUrl +import org.readium.r2.shared.util.Url +import org.readium.r2.shared.util.data.Container +import org.readium.r2.shared.util.file.DirectoryContainer +import org.readium.r2.shared.util.resource.Resource +import org.readium.r2.shared.util.xml.XmlParser +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class SmilBasedMediaOverlaysServiceTest { + + private val smilDir = requireNotNull( + SmilBasedMediaOverlaysServiceTest::class.java + .getResource("smil") + ?.path + ?.let { File(it) } + ) + + private val smilUrls = listOf( + RelativeUrl("chapter_001_overlay.smil")!!, + RelativeUrl("chapter_002_overlay.smil")!! + ) + + private val container: Container = DirectoryContainer( + root = smilDir, + entries = smilUrls.toSet() + ) + + @Test + fun `smil files are chained`() = runBlocking { + val service = SmilBasedMediaOverlaysService(smilUrls, container) + val iterator = service.iterator() + val docs = mutableListOf() + while (iterator.hasNext()) { + val doc = assertNotNull(iterator.next().getOrNull()) + docs.add(doc) + } + assert(docs.size == 2) + } +} + +@RunWith(RobolectricTestRunner::class) +class SmilParserTest { + + private val chapter1File: File = requireNotNull( + SmilParserTest::class.java + .getResource("smil/chapter_001_overlay.smil") + ?.path + ?.let { File(it) } + ) + + @Suppress("UNCHECKED_CAST") + private fun firstRefOfClass( + nodes: List, + klass: KClass, + ): T? { + for (node in nodes) { + node.refs + .firstOrNull { klass.isInstance(it) } + ?.let { return it as T } + + return firstRefOfClass(node.children, klass) + } + + return null + } + + private fun parseSmilDoc(): GuidedNavigationDocument { + val root = chapter1File + .inputStream() + .use { XmlParser().parse(it) } + + val doc = + SmilParser.parse(root, Url("OPS/chapter_001_overlay.smil")!!) + + return assertNotNull(doc) + } + + @Test + fun `all leaves are parsed`() { + fun assertSize(nodes: List) { + assert(nodes.size == 1 || nodes.size == 27) + + if (nodes.size == 27) { + return + } + for (node in nodes) { + assertSize(node.children) + } + } + + val guidedNavDoc = parseSmilDoc() + assertSize(guidedNavDoc.guided) + } + + @Test + fun `generated href are relative to SMIL`() { + val guidedNavDoc = parseSmilDoc() + val firstTextRef = assertNotNull(firstRefOfClass(guidedNavDoc.guided, GuidedNavigationTextRef::class)) + assertEquals(Url("OPS/chapter_001.xhtml#c01h01")!!, firstTextRef.url) + + val firstAudioRef = assertNotNull(firstRefOfClass(guidedNavDoc.guided, GuidedNavigationAudioRef::class)) + assertEquals(Url("OPS/audio/mobydick_001_002_melville.mp4")!!, firstAudioRef.url.removeFragment()) + } + + @Test + fun `audio clips are correct`() { + val guidedNavDoc = parseSmilDoc() + val firstAudioRef = assertNotNull(firstRefOfClass(guidedNavDoc.guided, GuidedNavigationAudioRef::class)) + assertEquals("t=24.5,29.268", firstAudioRef.url.fragment) + } +} diff --git a/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt index 9a70986678..d759c6ed70 100644 --- a/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt +++ b/readium/streamer/src/test/java/org/readium/r2/streamer/parser/epub/PackageDocumentTest.kt @@ -37,7 +37,7 @@ fun parsePackageDocument(path: String): Manifest { ?.let { ManifestAdapter(it) } ?.adapt() checkNotNull(pub) - return pub + return pub.manifest } const val PARSE_PUB_TIMEOUT = 1000L // milliseconds diff --git a/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil new file mode 100644 index 0000000000..cd366fdf10 --- /dev/null +++ b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_001_overlay.smil @@ -0,0 +1,139 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil new file mode 100644 index 0000000000..16e0d08106 --- /dev/null +++ b/readium/streamer/src/test/resources/org/readium/r2/streamer/parser/epub/smil/chapter_002_overlay.smil @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index cecf327a4b..156f55f1dc 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -74,10 +74,18 @@ include(":readium:navigators:media:tts") project(":readium:navigators:media:tts") .name = "readium-navigator-media-tts" +include(":readium:navigators:media:readaloud") +project(":readium:navigators:media:readaloud") + .name = "readium-navigator-media-readaloud" + include(":readium:adapters:exoplayer:audio") project(":readium:adapters:exoplayer:audio") .name = "readium-adapter-exoplayer-audio" +include(":readium:adapters:exoplayer:readaloud") +project(":readium:adapters:exoplayer:readaloud") + .name = "readium-adapter-exoplayer-readaloud" + include(":readium:opds") project(":readium:opds") .name = "readium-opds"