From fdbd97d84e8252c1ba00379cfe233aa961dbf410 Mon Sep 17 00:00:00 2001 From: soywiz Date: Fri, 5 Mar 2021 00:03:56 +0100 Subject: [PATCH] Some general + Win32 optimizations --- gradle.properties | 1 + korau/build.gradle | 1 + .../com/soywiz/korau/sound/AudioSamples.kt | 86 +++++++--- .../AudioSamplesTest.kt | 12 ++ .../com/soywiz/korau/sound/WaveOutProcess.kt | 150 ++++++++++++------ .../korau/sound/Win32NativeSoundProvider.kt | 61 +++---- 6 files changed, 217 insertions(+), 94 deletions(-) create mode 100644 korau/src/commonTest/kotlin/com.soywiz.korau.sound/AudioSamplesTest.kt diff --git a/gradle.properties b/gradle.properties index fa8ed11..9a36bf1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,6 +9,7 @@ group=com.soywiz.korlibs.korau version=2.0.0-SNAPSHOT korioVersion=2.0.9 +kmemVersion=2.0.9 # bintray location project.bintray.org=korlibs diff --git a/korau/build.gradle b/korau/build.gradle index a5ca861..e37fecd 100644 --- a/korau/build.gradle +++ b/korau/build.gradle @@ -50,6 +50,7 @@ kotlin { dependencies { commonMainApi("com.soywiz.korlibs.korio:korio:$korioVersion") + commonMainApi("com.soywiz.korlibs.kmem:kmem:$kmemVersion") def jnaVersion = "5.7.0" add("jvmMainApi", "net.java.dev.jna:jna:$jnaVersion") diff --git a/korau/src/commonMain/kotlin/com/soywiz/korau/sound/AudioSamples.kt b/korau/src/commonMain/kotlin/com/soywiz/korau/sound/AudioSamples.kt index c6e1052..ced0653 100644 --- a/korau/src/commonMain/kotlin/com/soywiz/korau/sound/AudioSamples.kt +++ b/korau/src/commonMain/kotlin/com/soywiz/korau/sound/AudioSamples.kt @@ -3,6 +3,7 @@ package com.soywiz.korau.sound import com.soywiz.kds.iterators.* import com.soywiz.kmem.* import com.soywiz.korau.internal.* +import com.soywiz.korio.lang.* import kotlin.math.* interface IAudioSamples { @@ -19,16 +20,23 @@ interface IAudioSamples { internal fun AudioSamples.resample(scale: Double, totalSamples: Int = (this.totalSamples * scale).toInt(), out: AudioSamples = AudioSamples(channels, totalSamples)): AudioSamples { val iscale = 1.0 / scale for (c in 0 until channels) { - val outc = out[c] val inpc = this[c] - for (n in 0 until totalSamples) { - // @TODO: Increase quality - outc[n] = inpc[(n * iscale).toInt()] + fastShortTransfer.use(out[c]) { outc -> + for (n in 0 until totalSamples) { + // @TODO: Increase quality + outc[n] = inpc[(n * iscale).toInt()] + } } } return out } +fun AudioSamples.resample(srcFreq: Int, dstFreq: Int): AudioSamples = + resample(dstFreq.toDouble() / srcFreq.toDouble()) + +fun AudioSamples.resampleIfRequired(srcFreq: Int, dstFreq: Int): AudioSamples = + if (srcFreq == dstFreq) this else resample(srcFreq, dstFreq) + class AudioSamplesProcessor(val channels: Int, val totalSamples: Int, val data: Array = Array(channels) { FloatArray(totalSamples) }) { fun reset(): AudioSamplesProcessor { for (ch in 0 until channels) data[ch].fill(0f) @@ -73,6 +81,7 @@ class AudioSamplesProcessor(val channels: Int, val totalSamples: Int, val data: class AudioSamples(override val channels: Int, override val totalSamples: Int, val data: Array = Array(channels) { ShortArray(totalSamples) }) : IAudioSamples { //val interleaved by lazy { interleaved() } + internal val fastShortTransfer = FastShortTransfer() operator fun get(channel: Int): ShortArray = data[channel] @@ -109,7 +118,7 @@ class AudioSamples(override val channels: Int, override val totalSamples: Int, v class AudioSamplesInterleaved(override val channels: Int, override val totalSamples: Int, val data: ShortArray = ShortArray(totalSamples * channels)) : IAudioSamples { //val separared by lazy { separated() } - + internal val fastShortTransfer = FastShortTransfer() private fun index(channel: Int, sample: Int) = (sample * channels) + channel override operator fun get(channel: Int, sample: Int): Short = data[index(channel, sample)] @@ -126,13 +135,49 @@ fun AudioSamples.copyOfRange(start: Int, end: Int): AudioSamples { return out } +fun AudioSamples.interleaved(out: AudioSamplesInterleaved = AudioSamplesInterleaved(channels, totalSamples)): AudioSamplesInterleaved { + assert(out.data.size >= totalSamples * channels) + when (channels) { + 1 -> arraycopy(this.data[0], 0, out.data, 0, totalSamples) + 2 -> arrayinterleave( + out.data, 0, + this.data[0], 0, + this.data[1], 0, + totalSamples, + out.fastShortTransfer + ) + else -> { + out.fastShortTransfer.use(out.data) { outData -> + val channels = channels + for (c in 0 until channels) { + var m = c + for (n in 0 until totalSamples) { + outData[m] = this[c, n] + m += channels + } + } + } + } + } + return out +} + fun IAudioSamples.interleaved(out: AudioSamplesInterleaved = AudioSamplesInterleaved(channels, totalSamples)): AudioSamplesInterleaved { - val channels = channels - for (c in 0 until channels) { - var m = c - for (n in 0 until totalSamples) { - out.data[m] = this[c, n] - m += channels + assert(out.data.size >= totalSamples * channels) + when (this) { + is AudioSamples -> this.interleaved(out) + is AudioSamplesInterleaved -> arraycopy(this.data, 0, out.data, 0, totalSamples * channels) + else -> { + out.fastShortTransfer.use(out.data) { outData -> + val channels = channels + for (c in 0 until channels) { + var m = c + for (n in 0 until totalSamples) { + outData[m] = this[c, n] + m += channels + } + } + } } } return out @@ -147,14 +192,17 @@ fun AudioSamplesInterleaved.applyProps(speed: Double, panning: Double, volume: D val rratio = ((((panning + 1.0) / 2.0).clamp01()) * volume).toFloat() val lratio = ((1.0 - rratio) * volume).toFloat() - if (channels == 2) { - for (n in 0 until out.totalSamples) { - out[0, n] = (this[0, (n * speedf).toInt()] * lratio).toInt().toShort() - out[1, n] = (this[1, (n * speedf).toInt()] * rratio).toInt().toShort() - } - } else { - for (n in out.data.indices) { - out.data[n] = (this.data[(n * speedf).toInt()] * lratio).toInt().toShort() + out.fastShortTransfer.use(out.data) { outData -> + var m = 0 + if (channels == 2) { + for (n in 0 until out.totalSamples) { + outData[m++] = (outData[(n * speedf).toInt() * 2 + 0] * lratio).toInt().toShort() + outData[m++] = (outData[(n * speedf).toInt() * 2 + 1] * rratio).toInt().toShort() + } + } else { + for (n in out.data.indices) { + outData[m++] = (outData[(n * speedf).toInt()] * lratio).toInt().toShort() + } } } diff --git a/korau/src/commonTest/kotlin/com.soywiz.korau.sound/AudioSamplesTest.kt b/korau/src/commonTest/kotlin/com.soywiz.korau.sound/AudioSamplesTest.kt new file mode 100644 index 0000000..2c14961 --- /dev/null +++ b/korau/src/commonTest/kotlin/com.soywiz.korau.sound/AudioSamplesTest.kt @@ -0,0 +1,12 @@ +package com.soywiz.korau.sound + +import kotlin.test.* + +class AudioSamplesTest { + @Test + fun test() { + val samples = AudioSamples(2, 22050) + val samples2 = samples.resample(22050, 44100) + assertEquals(44100, samples2.totalSamples) + } +} diff --git a/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/WaveOutProcess.kt b/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/WaveOutProcess.kt index 87ab6bb..c3c212e 100644 --- a/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/WaveOutProcess.kt +++ b/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/WaveOutProcess.kt @@ -38,13 +38,40 @@ private class ConcurrentDeque { private interface WaveOutPart private object WaveOutEnd : WaveOutPart +private object WaveOutFlush : WaveOutPart -private class WaveOutData(val data: ShortArray) : WaveOutPart { +private class WaveOutReopen(val freq: Int) : WaveOutPart { init { this.freeze() } } -private class WaveOutSetVolume(val volume: Double) : WaveOutPart { + +private interface WaveOutDataBase : WaveOutPart { + fun computeData(): ShortArray +} + +//private class WaveOutData(val data: ShortArray) : WaveOutDataBase { +// override fun computeData(): ShortArray = data +// +// init { +// this.freeze() +// } +//} + +private class WaveOutDataEx( + val adata: Array, + val pitch: Double, + val volume: Double, + val panning: Double, + val freq: Int +) : WaveOutDataBase { + override fun computeData(): ShortArray = + AudioSamples(2, adata[0].size, Array(2) { adata[it % adata.size] }) + //.resampleIfRequired(freq, 44100) + .interleaved() + .applyProps(pitch, panning, volume) + .data + init { this.freeze() } @@ -53,12 +80,14 @@ class WaveOutProcess(val freq: Int, val nchannels: Int) { private val sPosition = AtomicLong(0L) private val sLength = AtomicLong(0L) private val completed = AtomicLong(0L) + private val numPendingChunks = AtomicLong(0L) private val deque = ConcurrentDeque() private val info = AtomicReference?>(null) val position get() = sPosition.value val length get() = sLength.value - val isCompleted get() = completed.value != 0L + //val isCompleted get() = completed.value != 0L + val pendingAudio get() = numPendingChunks.value != 0L || deque.size > 0 init { freeze() @@ -66,19 +95,30 @@ class WaveOutProcess(val freq: Int, val nchannels: Int) { val pendingCommands get() = deque.size - fun addData(data: ShortArray) { - sLength.addAndGet(data.size / nchannels) - deque.add(WaveOutData(data)) - } - - fun setVolume(volume: Double) { - deque.add(WaveOutSetVolume(volume)) + //fun addData(data: ShortArray) { + // sLength.addAndGet(data.size / nchannels) + // deque.add(WaveOutData(data)) + //} + + fun addData(samples: AudioSamples, offset: Int, size: Int, pitch: Double, volume: Double, panning: Double, freq: Int) { + sLength.addAndGet(size) + deque.add(WaveOutDataEx( + Array(samples.channels) { samples.data[it].copyOfRange(offset, offset + size) }, + pitch, volume, panning, freq + )) } fun stop() { deque.add(WaveOutEnd) } + fun reopen(freq: Int) { + sPosition.value = 0L + sLength.value = 0L + completed.value = 0L + deque.add(WaveOutReopen(freq)) + } + fun stopAndWait() { stop() info?.value?.consume { } @@ -88,73 +128,87 @@ class WaveOutProcess(val freq: Int, val nchannels: Int) { val _info = this _info.info.value = _worker.execute(TransferMode.SAFE, { _info }) { info -> memScoped { - val format = alloc().apply { - this.wFormatTag = WAVE_FORMAT_PCM.convert() - this.nChannels = info.nchannels.convert() // 2? - this.nSamplesPerSec = info.freq.convert() // 44100? - this.wBitsPerSample = Short.SIZE_BITS.convert() // 16 - this.nBlockAlign = (info.nchannels * Short.SIZE_BYTES).convert() - this.nAvgBytesPerSec = this.nSamplesPerSec * this.nBlockAlign - this.cbSize = sizeOf().convert() - //this.cbSize = 0.convert() - } + val nchannels = info.nchannels // 2 val hWaveOut = alloc() - - waveOutOpen(hWaveOut.ptr, WAVE_MAPPER, format.ptr, 0.convert(), 0.convert(), CALLBACK_NULL) val pendingChunks = ArrayDeque() - fun updatePosition() { - // Update position - memScoped { - val time = alloc() - time.wType = TIME_BYTES.convert() - waveOutGetPosition(hWaveOut!!.value, time.ptr, sizeOf().convert()) - //info.position.value = time.u.cb.toLong() / Short.SIZE_BYTES / info.nchannels - info.sPosition.value = time.u.cb.toLong() / info.nchannels - } - } - fun clearCompletedChunks() { - updatePosition() while (pendingChunks.isNotEmpty() && pendingChunks.first().completed) { val chunk = pendingChunks.removeFirst() waveOutUnprepareHeader(hWaveOut.value, chunk.hdr.ptr, sizeOf().convert()) + info.sPosition.addAndGet(chunk.data.size / nchannels) chunk.dispose() } } + fun waveReset() { + clearCompletedChunks() + while (pendingChunks.isNotEmpty()) { + Sleep(5.convert()) + clearCompletedChunks() + } + waveOutReset(hWaveOut.value) + info.sPosition.value = 0L + } + + fun waveClose() { + waveReset() + waveOutClose(hWaveOut.value) + } + + var openedFreq = 0 + + fun waveOpen(freq: Int) { + openedFreq = freq + memScoped { + val format = alloc().apply { + this.wFormatTag = WAVE_FORMAT_PCM.convert() + this.nChannels = nchannels.convert() // 2? + this.nSamplesPerSec = freq.convert() + this.wBitsPerSample = Short.SIZE_BITS.convert() // 16 + this.nBlockAlign = (info.nchannels * Short.SIZE_BYTES).convert() + this.nAvgBytesPerSec = this.nSamplesPerSec * this.nBlockAlign + this.cbSize = sizeOf().convert() + //this.cbSize = 0.convert() + } + + waveOutOpen(hWaveOut.ptr, WAVE_MAPPER, format.ptr, 0.convert(), 0.convert(), CALLBACK_NULL) + } + } + + waveOpen(info.freq) + try { process@while (true) { - Sleep(5.convert()) - updatePosition() + clearCompletedChunks() while (true) { val it = info.deque.consume() ?: break //println("CONSUME: $item") when (it) { + is WaveOutReopen -> { + if (it.freq != openedFreq) { + waveClose() + waveOpen(it.freq) + } + } is WaveOutEnd -> break@process - is WaveOutData -> { - val chunk = WaveOutChunk(it.data) + is WaveOutDataBase -> { + val chunk = WaveOutChunk(it.computeData()) //info.sLength.addAndGet(chunk.data.size / info.nchannels) pendingChunks.add(chunk) waveOutPrepareHeader(hWaveOut.value, chunk.hdr.ptr, sizeOf().convert()) waveOutWrite(hWaveOut.value, chunk.hdr.ptr, sizeOf().convert()) - clearCompletedChunks() } - is WaveOutSetVolume -> { - waveOutSetVolume(hWaveOut.value, (it.volume.clamp01() * 0xFFFF).toInt().convert()) + is WaveOutFlush -> { + waveReset() } } - } + Sleep(5.convert()) } } finally { //println("finalizing...") - while (pendingChunks.isNotEmpty()) { - Sleep(5.convert()) - clearCompletedChunks() - } - waveOutReset(hWaveOut.value) - waveOutClose(hWaveOut.value) + waveClose() info.completed.value = 1L } } diff --git a/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/Win32NativeSoundProvider.kt b/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/Win32NativeSoundProvider.kt index 9f3a769..0739080 100644 --- a/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/Win32NativeSoundProvider.kt +++ b/korau/src/mingwX64Main/kotlin/com/soywiz/korau/sound/Win32NativeSoundProvider.kt @@ -18,19 +18,26 @@ private val Win32NativeSoundProvider_workerPool = Pool { Worker.start(name = "Win32NativeSoundProvider$it") } +@ThreadLocal +private val Win32NativeSoundProvider_WaveOutProcess = Pool { + WaveOutProcess(44100, 2).start(Win32NativeSoundProvider_workerPool.alloc()) +} + + object Win32NativeSoundProvider : NativeSoundProvider(), Disposable { //override val audioFormats: AudioFormats = AudioFormats(WAV, NativeMp3DecoderFormat, NativeOggVorbisDecoderFormat) //override val audioFormats: AudioFormats = AudioFormats(WAV, NativeMp3DecoderAudioFormat, PureJavaMp3DecoderAudioFormat, NativeOggVorbisDecoderFormat) override val audioFormats: AudioFormats = AudioFormats(WAV, MP3Decoder, NativeOggVorbisDecoderFormat) - val workerPool get() = Win32NativeSoundProvider_workerPool + //val workerPool get() = Win32NativeSoundProvider_workerPool + val workerPool get() = Win32NativeSoundProvider_WaveOutProcess override fun createAudioStream(coroutineContext: CoroutineContext, freq: Int): PlatformAudioOutput = Win32PlatformAudioOutput(Win32NativeSoundProvider, coroutineContext, freq) override fun dispose() { - while (workerPool.itemsInPool > 0) { - workerPool.alloc().requestTermination() + while (Win32NativeSoundProvider_workerPool.itemsInPool > 0) { + Win32NativeSoundProvider_workerPool.alloc().requestTermination() } } } @@ -40,9 +47,9 @@ class Win32PlatformAudioOutput( coroutineContext: CoroutineContext, val freq: Int ) : PlatformAudioOutput(coroutineContext, freq) { - private var process: WaveOutProcess = WaveOutProcess(freq, nchannels = 2) + private var process: WaveOutProcess? = null - override val availableSamples: Int get() = (process.length - process.position).toInt() + override val availableSamples: Int get() = if (process != null) (process!!.length - process!!.position).toInt() else 0 //.also { println("Win32PlatformAudioOutput.availableSamples. length=${process.length}, position=${process.position}, value=$it") } override var pitch: Double = 1.0 @@ -51,26 +58,23 @@ class Win32PlatformAudioOutput( override suspend fun add(samples: AudioSamples, offset: Int, size: Int) { // More than 1 second queued, let's wait a bit - if (availableSamples > freq) { + if (process == null || availableSamples > freq) { delay(200.milliseconds) } - // @TODO: All this things could be done at worker level - process.addData(samples.copyOfRange(offset, offset + size).interleaved().applyProps(pitch, panning, volume).ensureTwoChannels().data) - //process.addData(samples.data[0].copyOfRange(offset, offset + size)) + process!!.addData(samples, offset, size, pitch, volume, panning, freq) } - var worker: Worker? = null - override fun start() { - if (worker != null) return - worker = provider.workerPool.alloc() + process = provider.workerPool.alloc() + .also { it.reopen(freq) } //println("Win32PlatformAudioOutput.START WORKER: $worker") - process.start(worker!!) } override suspend fun wait() { - while (!process.isCompleted) { + //while (!process.isCompleted) { + while (availableSamples > 0) { + //while (process?.pendingAudio == true) { delay(10.milliseconds) //println("WAITING...: process.isCompleted=${process.isCompleted}") } @@ -78,18 +82,21 @@ class Win32PlatformAudioOutput( override fun stop() { //println("Win32PlatformAudioOutput.STOP WORKER: $worker") - val worker = this.worker ?: return - process.stop() - launchImmediately(coroutineContext) { - try { - wait() - } catch (e: CancellationException) { - // Do nothing - } catch (e: Throwable) { - Console.error("Error in Win32PlatformAudioOutput.stop:") - e.printStackTrace() - } finally { - provider.workerPool.free(worker) + //process.stop() + val process = this.process + this.process = null + if (process != null) { + launchImmediately(coroutineContext) { + try { + wait() + } catch (e: CancellationException) { + // Do nothing + } catch (e: Throwable) { + Console.error("Error in Win32PlatformAudioOutput.stop:") + e.printStackTrace() + } finally { + provider.workerPool.free(process) + } } } }