From e6b7a3e130a3990b1707dcb9177d097296cdc0e9 Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Sat, 26 Aug 2023 17:52:09 -0500 Subject: [PATCH 01/14] moving this to the global scope so the DOM doesn't need to be read every time --- web/h264.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/h264.js b/web/h264.js index 798145f..9bb4a8b 100644 --- a/web/h264.js +++ b/web/h264.js @@ -1,5 +1,6 @@ var img = document.getElementById("image"); var canvas = document.getElementById("canvas"); +var statsElement = document.getElementById('stats'); img.style.display = "none"; canvas.style.display = "block"; @@ -9,7 +10,7 @@ canvas.height = window.innerHeight; const worker = new Worker('h264_worker.js'); const offscreenCanvas = canvas.transferControlToOffscreen(); -worker.postMessage({canvas: offscreenCanvas, displayWidth: displayWidth, displayHeight: displayHeight, windowWidth: window.innerWidth, windowHeight: window.innerHeight}, [offscreenCanvas]); +worker.postMessage({ canvas: offscreenCanvas, displayWidth: displayWidth, displayHeight: displayHeight, windowWidth: window.innerWidth, windowHeight: window.innerHeight }, [offscreenCanvas]); worker.onmessage = function (event) { const stats = event.data; @@ -17,12 +18,12 @@ worker.onmessage = function (event) { }; function updateStatsDisplay(stats) { - const statsElement = document.getElementById('stats'); + if (statsElement) { statsElement.textContent = 'Skipped Frames: ' + stats.skippedFrames + ' Buffer size: ' + stats.bufferLength; } } function drawDisplayFrame(arrayBuffer) { - worker.postMessage({h264Data: arrayBuffer}); + worker.postMessage({ h264Data: arrayBuffer }); } \ No newline at end of file From 3c54f9686a744710a608817686087caba43515e1 Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Sun, 17 Sep 2023 09:41:24 -0500 Subject: [PATCH 02/14] Reuse the same object instead of repeating the operator multiple times --- web/imgTag.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/imgTag.js b/web/imgTag.js index 3dccec9..361bcc0 100644 --- a/web/imgTag.js +++ b/web/imgTag.js @@ -1,5 +1,6 @@ var img = document.getElementById("image"); var canvas = document.getElementById("canvas"); +var urlCreator = window.URL || window.webkitURL; img.style.display = "block"; canvas.style.display = "none"; @@ -7,11 +8,11 @@ var currentImageUrl; function drawDisplayFrame(blob) { if (currentImageUrl) { - (window.URL || window.webkitURL).revokeObjectURL(currentImageUrl); + urlCreator.revokeObjectURL(currentImageUrl); } - var urlCreator = window.URL || window.webkitURL; - var imageUrl = urlCreator.createObjectURL(blob); + + let imageUrl = urlCreator.createObjectURL(blob); currentImageUrl = imageUrl; img.src = imageUrl; From 7f39bcaefe67d64dc4ceaff60805aee466e018bc Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Sun, 17 Sep 2023 11:32:58 -0500 Subject: [PATCH 03/14] correct audio toggle handling false seems to be sent as a string occasionally so lets handle it both ways in this case if("false") is evaluating to true --- .fvm/fvm_config.json | 4 ++++ .gitignore | 1 + .vscode/launch.json | 24 ++++++++++++++++++++++++ .vscode/settings.json | 11 +++++++++++ web/android.html | 16 ++++++++++++++-- 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 .fvm/fvm_config.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json diff --git a/.fvm/fvm_config.json b/.fvm/fvm_config.json new file mode 100644 index 0000000..2bb3348 --- /dev/null +++ b/.fvm/fvm_config.json @@ -0,0 +1,4 @@ +{ + "flutterSdkVersion": "3.13.3", + "flavors": {} +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 429cdc2..590c68f 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ app.*.map.json /android/app/debug /android/app/profile /android/app/release +.fvm/flutter_sdk \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..e583cc6 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [{ + "name": "flutter-app", + "request": "launch", + "type": "dart" + }, + { + "name": "flutter-app (profile mode)", + "request": "launch", + "type": "dart", + "flutterMode": "profile" + }, + { + "name": "flutter-app (release mode)", + "request": "launch", + "type": "dart", + "flutterMode": "release" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e779d8 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "dart.flutterSdkPath": ".fvm/flutter_sdk", + // Remove .fvm files from search + "search.exclude": { + "**/.fvm": true + }, + // Remove from file watching + "files.watcherExclude": { + "**/.fvm": true + } +} \ No newline at end of file diff --git a/web/android.html b/web/android.html index 9297d5e..5273cd1 100644 --- a/web/android.html +++ b/web/android.html @@ -71,10 +71,21 @@ return; } const config = JSON.parse(event.data); + audioWebsocketUrl = config.audioWebsocketUrl; displayWebsocketUrl = config.displayWebsocketUrl; gpsWebsocketUrl = config.gpsWebsocketUrl; touchScreenWebsocketUrl = config.touchScreenWebsocketUrl; + + //check if isAudioEnabled is a boolean + if (typeof config.isAudioEnabled !== "boolean") { + if (config.isAudioEnabled === "false") { + config.isAudioEnabled = false; + } else { + config.isAudioEnabled = true; + } + } + isAudioEnabled = config.isAudioEnabled; audioVolume = config.audioVolume; displayRenderer = config.displayRenderer; @@ -101,6 +112,7 @@ function createAudioSocket(url) { if (!isAudioEnabled) { + console.log("Audio: Disabled"); return; } audioSocket = new ReconnectingWebSocket(url, null, { binaryType: 'arraybuffer' }); @@ -270,7 +282,7 @@ vertical_accuracy: String(5/*pos.accuracy*/), timestamp: String(pos.timestamp) }); - + return locationData; } @@ -286,4 +298,4 @@ - + \ No newline at end of file From c3729773924ba92c6da720ef674022f58c1a0ddd Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Sun, 24 Sep 2023 11:22:48 -0500 Subject: [PATCH 04/14] optimize pcmplayer to remove unnecessary data conversions Since we always use Float32, we have no need to do conversions. This yields improvements because the conversion function was rebuilding the buffer even though it was formatted correctly. --- web/pcmplayer.js | 81 ++++++++++++++++-------------------------------- 1 file changed, 27 insertions(+), 54 deletions(-) diff --git a/web/pcmplayer.js b/web/pcmplayer.js index 38fc589..218d182 100644 --- a/web/pcmplayer.js +++ b/web/pcmplayer.js @@ -1,8 +1,9 @@ (function (global, factory) { typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : - typeof define === 'function' && define.amd ? define(factory) : - (global = typeof globalThis !== 'undefined' ? globalThis : global || self, global.PCMPlayer = factory()); -})(this, (function () { 'use strict'; + typeof define === 'function' && define.amd ? define(factory) : + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, global.PCMPlayer = factory()); +})(this, (function () { + 'use strict'; class PCMPlayer { constructor(option) { @@ -20,30 +21,15 @@ this.option = Object.assign({}, defaultOption, option); // 实例最终配置参数 this.samples = new Float32Array(); // 样本存放区域 this.interval = setInterval(this.flush.bind(this), this.option.flushTime); - this.convertValue = this.getConvertValue(); + this.typedArray = this.getTypedArray(); this.initAudioContext(); } - getConvertValue() { - // 根据传入的目标编码位数 - // 选定转换数据所需要的基本值 - const inputCodecs = { - 'Int8': 128, - 'Int16': 32768, - 'Int32': 2147483648, - 'Float32': 1 - }; - if (!inputCodecs[this.option.inputCodec]) { - throw new Error('wrong codec.please input one of these codecs:Int8,Int16,Int32,Float32') - } - return inputCodecs[this.option.inputCodec] - } + getTypedArray() { - // 根据传入的目标编码位数 - // 选定前端的所需要的保存的二进制数据格式 - // 完整TypedArray请看文档 + // Select the binary data format that needs to be saved by the front end according to the target encoding bit number. See the complete TypedArray in the documentation // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray const typedArrays = { 'Int8': Int8Array, @@ -58,9 +44,9 @@ } initAudioContext() { - // 初始化音频上下文的东西 + // Initialize the audio context this.audioCtx = new (window.parent.AudioContext || window.parent.webkitAudioContext)(); - // 控制音量的 GainNode + // GainNode that controls the volume // https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/createGain this.gainNode = this.audioCtx.createGain(); this.gainNode.gain.value = 1.0; @@ -69,15 +55,14 @@ } static isTypedArray(data) { - // 检测输入的数据是否为 TypedArray 类型或 ArrayBuffer 类型 + //Check whether the input data is of the TypedArray type or ArrayBuffer type return (data.byteLength && data.buffer && data.buffer.constructor == ArrayBuffer) || data.constructor == ArrayBuffer; } isSupported(data) { - // 数据类型是否支持 - // 目前支持 ArrayBuffer 或者 TypedArray + // Currently, the code supports ArrayBuffer or TypedArray if (!PCMPlayer.isTypedArray(data)) { - throw new Error('请传入ArrayBuffer或者任意TypedArray') + throw new Error('Not a ArrayBuffer or TypedArray') } return true } @@ -85,40 +70,28 @@ feed(data) { this.isSupported(data); - // 获取格式化后的buffer - data = this.getFormatedValue(data); - // 开始拷贝buffer数据 - // 新建一个Float32Array的空间 + // getting the formatted buffer. + if (data.constructor == ArrayBuffer) { + data = new Float32Array(data); + } else { + data = new Float32Array(data.buffer); + } + // the code is starting to copy data from a buffer. It then creates a new Float32Array to store the copied data. const tmp = new Float32Array(this.samples.length + data.length); - // console.log(data, this.samples, this.samples.length) - // 复制当前的实例的buffer值（历史buff) - // 从头（0）开始复制 + tmp.set(this.samples, 0); - // 复制传入的新数据 - // 从历史buff位置开始 + // Copy the new data passed in, starting from the historical buffer position tmp.set(data, this.samples.length); - // 将新的完整buff数据赋值给samples - // interval定时器也会从samples里面播放数据 + // code is assigning a new buffer of data to the samples variable. The interval timer will then play the data from the samples variable. this.samples = tmp; } - getFormatedValue(data) { + getFormattedValue(data) { if (data.constructor == ArrayBuffer) { - data = new this.typedArray(data); + return new Float32Array(data); } else { - data = new this.typedArray(data.buffer); + return new Float32Array(data.buffer); } - - let float32 = new Float32Array(data.length); - - for (let i = 0; i < data.length; i++) { - // buffer 缓冲区的数据，需要是IEEE754 里32位的线性PCM，范围从-1到+1 - // 所以对数据进行除法 - // 除以对应的位数范围，得到-1到+1的数据 - // float32[i] = data[i] / 0x8000; - float32[i] = data[i] / this.convertValue; - } - return float32 } volume(volume) { @@ -136,14 +109,14 @@ flush() { if (!this.samples.length) return - var bufferSource = this.audioCtx.createBufferSource(); + let bufferSource = this.audioCtx.createBufferSource(); const length = this.samples.length / this.option.channels; const audioBuffer = this.audioCtx.createBuffer(this.option.channels, length, this.option.sampleRate); for (let channel = 0; channel < this.option.channels; channel++) { const audioData = audioBuffer.getChannelData(channel); let offset = channel; - let decrement = 50; + for (let i = 0; i < length; i++) { audioData[i] = this.samples[offset]; offset += this.option.channels; From ebbbc98e6542e1db4e55ff9280d27f59920ad568 Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Sun, 24 Sep 2023 11:37:18 -0500 Subject: [PATCH 05/14] Handle audio drift? --- web/pcmplayer.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web/pcmplayer.js b/web/pcmplayer.js index 218d182..da6bd22 100644 --- a/web/pcmplayer.js +++ b/web/pcmplayer.js @@ -76,6 +76,17 @@ } else { data = new Float32Array(data.buffer); } + + //handle too many samples building up in the buffer + //This will allow half a second of audio to be buffered at any time. + const maxSamples = this.audioCtx.sampleRate / 2; + const totalSamples = this.samples.length + data.length; + + if (totalSamples > maxSamples) { + const samplesToDiscard = totalSamples - maxSamples; + this.samples = this.samples.subarray(samplesToDiscard); + } + // the code is starting to copy data from a buffer. It then creates a new Float32Array to store the copied data. const tmp = new Float32Array(this.samples.length + data.length); @@ -83,6 +94,9 @@ // Copy the new data passed in, starting from the historical buffer position tmp.set(data, this.samples.length); // code is assigning a new buffer of data to the samples variable. The interval timer will then play the data from the samples variable. + + + this.samples = tmp; } From bf37595480d387d692d02916e2a5c9a25428ea90 Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Sun, 24 Sep 2023 13:37:12 -0500 Subject: [PATCH 06/14] Update pcmplayer.js --- web/pcmplayer.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/pcmplayer.js b/web/pcmplayer.js index da6bd22..9e15e0a 100644 --- a/web/pcmplayer.js +++ b/web/pcmplayer.js @@ -12,14 +12,14 @@ init(option) { const defaultOption = { - inputCodec: 'Int16', // 传入的数据是采用多少位编码，默认16位 - channels: 1, // 声道数 - sampleRate: 8000, // 采样率 单位Hz - flushTime: 1000 // 缓存时间 单位 ms + inputCodec: 'Int16', // The number of bits used for encoding the input data, default is 16 bits + channels: 1, // Number of channels + sampleRate: 8000, // Sampling rate in Hz + flushTime: 1000 // Cache time in milliseconds }; - this.option = Object.assign({}, defaultOption, option); // 实例最终配置参数 - this.samples = new Float32Array(); // 样本存放区域 + this.option = Object.assign({}, defaultOption, option); // Final configuration parameters for the instance + this.samples = new Float32Array(); // Sample storage area this.interval = setInterval(this.flush.bind(this), this.option.flushTime); this.typedArray = this.getTypedArray(); From e3cb5954ea80af6120a76d4badcd3f427866b40c Mon Sep 17 00:00:00 2001 From: "david@vanronk.net" Date: Tue, 26 Sep 2023 20:28:29 -0500 Subject: [PATCH 07/14] save --- web/android.html | 54 +++++++------------------------------- web/audioplayback.js | 50 +++++++++++++++++++++++++++++++++++ web/pcmplayer-processor.js | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 44 deletions(-) create mode 100644 web/audioplayback.js create mode 100644 web/pcmplayer-processor.js diff --git a/web/android.html b/web/android.html index 5273cd1..3697e39 100644 --- a/web/android.html +++ b/web/android.html @@ -25,8 +25,9 @@ display: none; } - + + @@ -34,14 +35,14 @@