diff --git a/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileDownloader.kt b/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileDownloader.kt index cd99074..16aec73 100644 --- a/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileDownloader.kt +++ b/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileDownloader.kt @@ -1,6 +1,5 @@ package com.cognifide.gradle.common.file.transfer -import com.cognifide.gradle.common.build.ProgressLogger import com.cognifide.gradle.common.utils.Formats import com.cognifide.gradle.common.CommonExtension import java.io.File @@ -11,14 +10,15 @@ class FileDownloader(private val common: CommonExtension) { private var processedBytes: Long = 0 - private var loggedKb: Long = 0 - private var startTime: Long = -1 var size: Long = 0 + var chunkSize: Int = common.prop.int("fileTransfer.downloader.chunkSize") ?: CHUNK_SIZE + fun download(input: InputStream, target: File) { - common.progressLogger { + common.progress { + updater { update(currentProgress(target)) } input.use { inputStream -> target.parentFile.mkdirs() startTime = System.currentTimeMillis() @@ -27,12 +27,12 @@ class FileDownloader(private val common: CommonExtension) { var finished = false try { - val buf = ByteArray(TRANSFER_CHUNK_100_KB) + val buf = ByteArray(chunkSize) var read = inputStream.read(buf) while (read >= 0) { output.write(buf, 0, read) - logProgress("Downloading", read.toLong(), target) + processedBytes += read.toLong() read = inputStream.read(buf) } @@ -48,27 +48,19 @@ class FileDownloader(private val common: CommonExtension) { } } - private fun ProgressLogger.logProgress(operation: String, readBytes: Long, file: File) { - processedBytes += readBytes - - val processedKb = processedBytes / KILOBYTE - if (processedKb > loggedKb) { - val fileName = file.name.removeSuffix(FileTransferManager.TMP_SUFFIX) - val msg = if (size > 0) { - "$operation: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" + - " (${Formats.percent(processedBytes, size)}," + - " time left: ${Formats.duration(remainingTime())})" - } else { - "$operation: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}" - } - - progress(msg) - - loggedKb = processedKb + private fun currentProgress(file: File): String { + val fileName = file.name.removeSuffix(FileTransferManager.TMP_SUFFIX) + return if (size > 0) { + "Downloading: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" + + " (${Formats.percent(processedBytes, size)}," + + " time left: ${Formats.duration(remainingTime())})" + } else { + "Downloading: $fileName | ${Formats.fileSizeBytesToHuman(processedBytes)}" } } private fun remainingTime(): Long { + if (processedBytes == 0L) return 0 val elapsedTime = System.currentTimeMillis() - startTime val allTime = (elapsedTime * size / processedBytes) @@ -76,8 +68,8 @@ class FileDownloader(private val common: CommonExtension) { } companion object { - const val TRANSFER_CHUNK_100_KB = 100 * 1024 - const val KILOBYTE = 1024 + + const val CHUNK_SIZE = 512 * KILOBYTE } } diff --git a/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileTransferManager.kt b/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileTransferManager.kt index 781e2f9..65384f3 100644 --- a/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileTransferManager.kt +++ b/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileTransferManager.kt @@ -9,6 +9,7 @@ import com.cognifide.gradle.common.file.transfer.http.HttpFileTransfer import com.cognifide.gradle.common.file.transfer.resolve.ResolveFileTransfer import com.cognifide.gradle.common.file.transfer.sftp.SftpFileTransfer import com.cognifide.gradle.common.file.transfer.smb.SmbFileTransfer +import com.cognifide.gradle.common.utils.Formats import com.cognifide.gradle.common.utils.using import java.io.File @@ -128,7 +129,7 @@ class FileTransferManager(private val common: CommonExtension) : FileTransfer { */ fun downloadUsing(transfer: FileTransfer, dirUrl: String, fileName: String, target: File) { if (target.exists()) { - common.logger.info("Skipping downloading file from URL '$dirUrl/$fileName' to '$target' as of it already exists.") + logger.info("Skipping downloading file from URL '$dirUrl/$fileName' to '$target' as of it already exists.") return } @@ -139,8 +140,11 @@ class FileTransferManager(private val common: CommonExtension) : FileTransfer { tmp.delete() } + val started = System.currentTimeMillis() transfer.downloadFrom(dirUrl, fileName, tmp) tmp.renameTo(target) + + logger.info("Downloaded file from URL '$dirUrl/$fileName' to '$target' in ${Formats.durationWordsSince(started)}") } /** @@ -171,7 +175,10 @@ class FileTransferManager(private val common: CommonExtension) : FileTransfer { logger.debug("Cannot check status of uploaded file at URL '$fileUrl'", e) } + val started = System.currentTimeMillis() transfer.uploadTo(dirUrl, fileName, source) + + logger.info("Uploaded file from '$source' to URL '$dirUrl/$fileName' in ${Formats.durationWordsSince(started)}") } /** diff --git a/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileUploader.kt b/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileUploader.kt index 678076b..41523a3 100644 --- a/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileUploader.kt +++ b/src/main/kotlin/com/cognifide/gradle/common/file/transfer/FileUploader.kt @@ -1,7 +1,6 @@ package com.cognifide.gradle.common.file.transfer import com.cognifide.gradle.common.CommonExtension -import com.cognifide.gradle.common.build.ProgressLogger import com.cognifide.gradle.common.utils.Formats import java.io.File import java.io.OutputStream @@ -10,14 +9,16 @@ class FileUploader(private val common: CommonExtension) { private var processedBytes: Long = 0 - private var loggedKb: Long = 0 - private var size: Long = 0 private var startTime: Long = -1 + var chunkSize: Int = common.prop.int("fileTransfer.uploader.chunkSize") ?: CHUNK_SIZE + fun upload(file: File, output: OutputStream, cleanup: (File) -> Unit = {}) { - common.progressLogger { + common.progress { + updater { update(currentProgress(file)) } + file.inputStream().use { input -> size = file.length() startTime = System.currentTimeMillis() @@ -25,12 +26,12 @@ class FileUploader(private val common: CommonExtension) { var finished = false try { - val buf = ByteArray(TRANSFER_CHUNK_100_KB) + val buf = ByteArray(chunkSize) var read = input.read(buf) while (read >= 0) { output.write(buf, 0, read) - logProgress("Uploading", read.toLong(), file) + processedBytes += read.toLong() read = input.read(buf) } @@ -46,26 +47,18 @@ class FileUploader(private val common: CommonExtension) { } } - private fun ProgressLogger.logProgress(operation: String, readBytes: Long, file: File) { - processedBytes += readBytes - - val processedKb = processedBytes / KILOBYTE - if (processedKb > loggedKb) { - val msg = if (size > 0) { - "$operation: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" + - " (${Formats.percent(processedBytes, size)}," + - " time left: ${Formats.duration(remainingTime())})" - } else { - "$operation: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}" - } - - progress(msg) - - loggedKb = processedKb + private fun currentProgress(file: File): String { + return if (size > 0) { + "Uploading: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}/${Formats.fileSizeBytesToHuman(size)}" + + " (${Formats.percent(processedBytes, size)}," + + " time left: ${Formats.duration(remainingTime())})" + } else { + "Uploading: ${file.name} | ${Formats.fileSizeBytesToHuman(processedBytes)}" } } private fun remainingTime(): Long { + if (processedBytes == 0L) return 0 val elapsedTime = System.currentTimeMillis() - startTime val allTime = (elapsedTime * size / processedBytes) @@ -73,8 +66,8 @@ class FileUploader(private val common: CommonExtension) { } companion object { - const val TRANSFER_CHUNK_100_KB = 100 * 1024 - const val KILOBYTE = 1024 + + const val CHUNK_SIZE = 512 * KILOBYTE } } diff --git a/src/main/kotlin/com/cognifide/gradle/common/utils/Formats.kt b/src/main/kotlin/com/cognifide/gradle/common/utils/Formats.kt index 41a381f..f759815 100644 --- a/src/main/kotlin/com/cognifide/gradle/common/utils/Formats.kt +++ b/src/main/kotlin/com/cognifide/gradle/common/utils/Formats.kt @@ -243,8 +243,12 @@ object Formats { fun duration(millis: Long): String = DurationFormatUtils.formatDuration(millis.coerceAtLeast(0L), "mm:ss") + fun durationWords(millis: Long): String = DurationFormatUtils.formatDurationWords(millis.coerceAtLeast(0L), true, true) + fun durationSince(millis: Long) = duration(System.currentTimeMillis() - millis) + fun durationWordsSince(millis: Long) = durationWords(System.currentTimeMillis() - millis) + fun durationFit(thenMillis: Long, thenZoneId: ZoneId, durationMillis: Long): Boolean { val nowTimestamp = LocalDateTime.now().atZone(ZoneId.systemDefault()) val thenTimestamp = localDateTimeAt(thenMillis, thenZoneId)