From 36403595f06bc1a1ac51af34acced3da033f1e29 Mon Sep 17 00:00:00 2001 From: PhilKes Date: Tue, 29 Oct 2024 17:37:29 +0100 Subject: [PATCH 1/5] Add Google Keep Importer --- app/build.gradle | 17 ++ .../com/philkes/notallyx/data/DataUtil.kt | 222 +++++++++++++++ .../philkes/notallyx/data/dao/BaseNoteDao.kt | 4 +- .../notallyx/data/imports/ExternalImporter.kt | 10 + .../notallyx/data/imports/ImportException.kt | 3 + .../notallyx/data/imports/NotesImporter.kt | 89 ++++++ .../data/imports/google/GoogleKeepImporter.kt | 169 ++++++++++++ .../notallyx/data/imports/google/KeepNote.kt | 25 ++ .../com/philkes/notallyx/data/model/Audio.kt | 2 +- .../notallyx/data/model/FileAttachment.kt | 2 +- .../notallyx/presentation/UiExtensions.kt | 26 ++ .../main/fragment/SettingsFragment.kt | 41 +++ .../view/main/sorting/BaseNoteSort.kt | 6 +- .../view/note/audio/AudioControlView.kt | 6 +- .../presentation/viewmodel/BaseNoteModel.kt | 29 +- .../presentation/viewmodel/NotallyModel.kt | 159 +---------- .../philkes/notallyx/utils/backup/Export.kt | 12 +- app/src/main/res/layout/fragment_settings.xml | 7 + app/src/main/res/values/strings.xml | 6 + .../data/imports/NotesImporterTest.kt | 237 ++++++++++++++++ .../imports/google/GoogleKeepImporterTest.kt | 255 ++++++++++++++++++ .../resources/imports/googlekeep/Takeout.zip | Bin 0 -> 1217167 bytes 22 files changed, 1172 insertions(+), 155 deletions(-) create mode 100644 app/src/main/java/com/philkes/notallyx/data/DataUtil.kt create mode 100644 app/src/main/java/com/philkes/notallyx/data/imports/ExternalImporter.kt create mode 100644 app/src/main/java/com/philkes/notallyx/data/imports/ImportException.kt create mode 100644 app/src/main/java/com/philkes/notallyx/data/imports/NotesImporter.kt create mode 100644 app/src/main/java/com/philkes/notallyx/data/imports/google/GoogleKeepImporter.kt create mode 100644 app/src/main/java/com/philkes/notallyx/data/imports/google/KeepNote.kt create mode 100644 app/src/test/kotlin/com/philkes/notallyx/data/imports/NotesImporterTest.kt create mode 100644 app/src/test/kotlin/com/philkes/notallyx/data/imports/google/GoogleKeepImporterTest.kt create mode 100644 app/src/test/resources/imports/googlekeep/Takeout.zip diff --git a/app/build.gradle b/app/build.gradle index ea37ec95..8ea212d5 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -7,6 +7,8 @@ plugins { id 'kotlin-parcelize' id 'com.google.devtools.ksp' id 'com.ncorti.ktfmt.gradle' version '0.20.1' + id 'org.jetbrains.kotlin.plugin.serialization' version '1.9.0' + } android { @@ -54,6 +56,12 @@ android { packagingOptions.resources { excludes += ["DebugProbesKt.bin", "META-INF/**.version", "kotlin/**.kotlin_builtins", "kotlin-tooling-metadata.json"] } + + testOptions{ + unitTests{ + includeAndroidResources true + } + } } tasks.withType(KotlinCompile).configureEach { @@ -111,10 +119,19 @@ dependencies { implementation "com.github.bumptech.glide:glide:4.15.1" implementation "com.davemorrissey.labs:subsampling-scale-image-view-androidx:3.10.0" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.3" + testImplementation "junit:junit:4.13.2" testImplementation "androidx.test:core:1.6.1" + testImplementation "androidx.test:core-ktx:1.6.1" testImplementation "org.mockito:mockito-core:5.13.0" testImplementation "org.mockito.kotlin:mockito-kotlin:5.4.0" testImplementation "io.mockk:mockk:1.13.12" testImplementation "org.json:json:20180813" + testImplementation "org.assertj:assertj-core:3.24.2" + testImplementation "org.robolectric:robolectric:4.13" + + androidTestImplementation "androidx.room:room-testing:$roomVersion" + androidTestImplementation "androidx.work:work-testing:2.9.1" + } \ No newline at end of file diff --git a/app/src/main/java/com/philkes/notallyx/data/DataUtil.kt b/app/src/main/java/com/philkes/notallyx/data/DataUtil.kt new file mode 100644 index 00000000..fc15c844 --- /dev/null +++ b/app/src/main/java/com/philkes/notallyx/data/DataUtil.kt @@ -0,0 +1,222 @@ +package com.philkes.notallyx.data + +import android.app.Application +import android.content.ContentResolver +import android.graphics.BitmapFactory +import android.media.MediaMetadataRetriever +import android.net.Uri +import android.webkit.MimeTypeMap +import androidx.documentfile.provider.DocumentFile +import com.philkes.notallyx.R +import com.philkes.notallyx.data.model.Audio +import com.philkes.notallyx.data.model.FileAttachment +import com.philkes.notallyx.presentation.viewmodel.NotallyModel.FileType +import com.philkes.notallyx.utils.FileError +import com.philkes.notallyx.utils.IO.copyToFile +import com.philkes.notallyx.utils.IO.getExternalAudioDirectory +import com.philkes.notallyx.utils.IO.getExternalFilesDirectory +import com.philkes.notallyx.utils.IO.getExternalImagesDirectory +import com.philkes.notallyx.utils.IO.rename +import com.philkes.notallyx.utils.Operations +import com.philkes.notallyx.utils.getFileName +import java.io.File +import java.io.FileInputStream +import java.util.UUID +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +class DataUtil { + companion object { + + suspend fun addFile( + app: Application, + uri: Uri, + directory: File, + fileType: FileType, + errorWhileRenaming: Int = R.string.error_while_renaming_file, + proposedMimeType: String? = null, + ): Pair { + return withContext(Dispatchers.IO) { + val document = requireNotNull(DocumentFile.fromSingleUri(app, uri)) + val displayName = document.name ?: app.getString(R.string.unknown_name) + try { + + /* + If we have reached this point, an SD card (emulated or real) exists and externalRoot + is not null. externalRoot.exists() can be false if the folder `Images` has been deleted after + the previous line, but externalRoot itself can't be null + */ + val temp = File(directory, "Temp") + + val inputStream = requireNotNull(app.contentResolver.openInputStream(uri)) + inputStream.copyToFile(temp) + + val originalName = app.getFileName(uri) + when (fileType) { + FileType.IMAGE -> { + val options = BitmapFactory.Options() + options.inJustDecodeBounds = true + BitmapFactory.decodeFile(temp.path, options) + val mimeType = options.outMimeType ?: proposedMimeType + + if (mimeType != null) { + val extension = getExtensionForMimeType(mimeType) + if (extension != null) { + val name = "${UUID.randomUUID()}.$extension" + if (temp.rename(name)) { + return@withContext Pair( + FileAttachment(name, originalName ?: name, mimeType), + null, + ) + } else { + // I don't expect this error to ever happen but just in + // case + return@withContext Pair( + null, + FileError( + displayName, + app.getString(errorWhileRenaming), + fileType, + ), + ) + } + } else + return@withContext Pair( + null, + FileError( + displayName, + app.getString(R.string.image_format_not_supported), + fileType, + ), + ) + } else + return@withContext Pair( + null, + FileError( + displayName, + app.getString(R.string.invalid_image), + fileType, + ), + ) + } + + FileType.ANY -> { + val (mimeType, fileExtension) = + determineMimeTypeAndExtension( + proposedMimeType, + uri, + app.contentResolver, + ) + val name = "${UUID.randomUUID()}${fileExtension}" + if (temp.rename(name)) { + return@withContext Pair( + FileAttachment(name, originalName ?: name, mimeType), + null, + ) + } else { + // I don't expect this error to ever happen but just in case + return@withContext Pair( + null, + FileError( + displayName, + app.getString(errorWhileRenaming), + fileType, + ), + ) + } + } + } + } catch (exception: Exception) { + Operations.log(app, exception) + return@withContext Pair( + null, + FileError(displayName, app.getString(R.string.unknown_error), fileType), + ) + } + } + } + + private fun determineMimeTypeAndExtension( + proposedMimeType: String?, + uri: Uri, + contentResolver: ContentResolver, + ) = + if (proposedMimeType != null && proposedMimeType.contains("/")) { + Pair(proposedMimeType, ".${uri.lastPathSegment?.substringAfterLast(".")}") + } else { + val actualMimeType = contentResolver.getType(uri) ?: "application/octet-stream" + Pair( + actualMimeType, + MimeTypeMap.getSingleton().getExtensionFromMimeType(actualMimeType)?.let { + ".${it}" + } ?: "", + ) + } + + suspend fun addFile( + app: Application, + uri: Uri, + proposedMimeType: String? = null, + ): Pair { + val filesRoot = app.getExternalFilesDirectory() + requireNotNull(filesRoot) { "filesRoot is null" } + return addFile(app, uri, filesRoot, FileType.ANY, proposedMimeType = proposedMimeType) + } + + suspend fun addImage( + app: Application, + uri: Uri, + proposedMimeType: String? = null, + ): Pair { + val imagesRoot = app.getExternalImagesDirectory() + requireNotNull(imagesRoot) { "imagesRoot is null" } + return addFile( + app, + uri, + imagesRoot, + FileType.IMAGE, + proposedMimeType = proposedMimeType, + ) + } + + suspend fun addAudio(app: Application, original: File, deleteOriginalFile: Boolean): Audio { + return withContext(Dispatchers.IO) { + /* + Regenerate because the directory may have been deleted between the time of activity creation + and audio recording + */ + val audioRoot = app.getExternalAudioDirectory() + requireNotNull(audioRoot) { "audioRoot is null" } + + /* + If we have reached this point, an SD card (emulated or real) exists and audioRoot + is not null. audioRoot.exists() can be false if the folder `Audio` has been deleted after + the previous line, but audioRoot itself can't be null + */ + val name = "${UUID.randomUUID()}.m4a" + val final = File(audioRoot, name) + val input = FileInputStream(original) + input.copyToFile(final) + + if (deleteOriginalFile) { + original.delete() + } + + val retriever = MediaMetadataRetriever() + retriever.setDataSource(final.path) + val duration = + retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) ?: "-1" + Audio(name, duration.toLong(), System.currentTimeMillis()) + } + } + + private fun getExtensionForMimeType(type: String): String? { + return when (type) { + "image/png" -> "png" + "image/jpeg" -> "jpg" + "image/webp" -> "webp" + else -> null + } + } + } +} diff --git a/app/src/main/java/com/philkes/notallyx/data/dao/BaseNoteDao.kt b/app/src/main/java/com/philkes/notallyx/data/dao/BaseNoteDao.kt index 3969e4cd..e47ee096 100644 --- a/app/src/main/java/com/philkes/notallyx/data/dao/BaseNoteDao.kt +++ b/app/src/main/java/com/philkes/notallyx/data/dao/BaseNoteDao.kt @@ -40,7 +40,9 @@ interface BaseNoteDao { @Query("SELECT * FROM BaseNote WHERE folder = 'NOTES' ORDER BY pinned DESC, timestamp DESC") suspend fun getAllNotes(): List - @Query("SELECT * FROM BaseNote") fun getAll(): LiveData> + @Query("SELECT * FROM BaseNote") fun getAllAsync(): LiveData> + + @Query("SELECT * FROM BaseNote") fun getAll(): List @Query("SELECT * FROM BaseNote WHERE id IN (:ids)") fun getByIds(ids: LongArray): List diff --git a/app/src/main/java/com/philkes/notallyx/data/imports/ExternalImporter.kt b/app/src/main/java/com/philkes/notallyx/data/imports/ExternalImporter.kt new file mode 100644 index 00000000..b470a79e --- /dev/null +++ b/app/src/main/java/com/philkes/notallyx/data/imports/ExternalImporter.kt @@ -0,0 +1,10 @@ +package com.philkes.notallyx.data.imports + +import android.app.Application +import java.io.File +import java.io.InputStream + +interface ExternalImporter { + + fun importFrom(inputStream: InputStream, app: Application): Pair +} diff --git a/app/src/main/java/com/philkes/notallyx/data/imports/ImportException.kt b/app/src/main/java/com/philkes/notallyx/data/imports/ImportException.kt new file mode 100644 index 00000000..ae239ba4 --- /dev/null +++ b/app/src/main/java/com/philkes/notallyx/data/imports/ImportException.kt @@ -0,0 +1,3 @@ +package com.philkes.notallyx.data.imports + +class ImportException(val textResId: Int, cause: Throwable? = null) : RuntimeException(cause) diff --git a/app/src/main/java/com/philkes/notallyx/data/imports/NotesImporter.kt b/app/src/main/java/com/philkes/notallyx/data/imports/NotesImporter.kt new file mode 100644 index 00000000..b3006ec8 --- /dev/null +++ b/app/src/main/java/com/philkes/notallyx/data/imports/NotesImporter.kt @@ -0,0 +1,89 @@ +package com.philkes.notallyx.data.imports + +import android.app.Application +import android.util.Log +import androidx.core.net.toUri +import com.philkes.notallyx.R +import com.philkes.notallyx.data.DataUtil +import com.philkes.notallyx.data.NotallyDatabase +import com.philkes.notallyx.data.imports.google.GoogleKeepImporter +import com.philkes.notallyx.data.model.Audio +import com.philkes.notallyx.data.model.BaseNote +import com.philkes.notallyx.data.model.FileAttachment +import com.philkes.notallyx.data.model.Label +import com.philkes.notallyx.presentation.viewmodel.NotallyModel +import java.io.File +import java.io.InputStream + +class NotesImporter(private val app: Application, private val database: NotallyDatabase) { + + suspend fun import(inputStream: InputStream, importSource: ImportSource) { + val (import, importDataFolder) = + when (importSource) { + ImportSource.GOOGLE_KEEP -> GoogleKeepImporter().importFrom(inputStream, app) + } + database.getLabelDao().insert(import.labels) + importFiles(import.files, importDataFolder, NotallyModel.FileType.ANY) + importFiles(import.images, importDataFolder, NotallyModel.FileType.IMAGE) + importAudios(import.audios, importDataFolder) + database.getBaseNoteDao().insert(import.baseNotes) + } + + private suspend fun importFiles( + files: List, + sourceFolder: File, + fileType: NotallyModel.FileType, + ) { + files.forEach { file -> + val uri = File(sourceFolder, file.localName).toUri() + val (fileAttachment, error) = + if (fileType == NotallyModel.FileType.IMAGE) + DataUtil.addImage(app, uri, file.mimeType) + else DataUtil.addFile(app, uri, file.mimeType) + fileAttachment?.let { + file.localName = fileAttachment.localName + file.originalName = fileAttachment.originalName + file.mimeType = fileAttachment.mimeType + } + error?.let { Log.d(TAG, "Failed to import: $error") } + } + } + + private suspend fun importAudios(audios: List