Improvements to text to speech (forked lib)

iaincollins · May 22, 2022 · 5bc50ce · 5bc50ce
1 parent cb7f246
commit 5bc50ce
Show file tree

Hide file tree

Showing 20 changed files with 736 additions and 92 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "icarus",
-  "version": "0.13.6",
+  "version": "0.13.7",
   "description": "ICARUS Terminal for Elite Dangerous",
   "scripts": {
     "build": "npm run build:client && npm run build:app && npm run build:service && npm run build:package",
@@ -55,6 +55,7 @@
     "glob": "^7.1.7",
     "http-proxy": "^1.18.1",
     "nedb-promises": "^5.0.2",
+    "one-time": "^0.0.4",
     "pjxml": "^1.1.0",
     "say": "^0.16.0",
     "serve-static": "^1.14.1",

diff --git a/src/client/components/settings.js b/src/client/components/settings.js
@@ -8,7 +8,7 @@ function Settings ({ visible, toggleVisible = () => {}, defaultActiveSettingsPan
 
   return (
     <>
-      <div className='modal-dialog__background' style={{ opacity: visible ? 1 : 0, visibility: visible ? 'visible' : 'hidden' }}/>
+      <div className='modal-dialog__background' style={{ opacity: visible ? 1 : 0, visibility: visible ? 'visible' : 'hidden' }} onClick={toggleVisible}/>
       <div className='modal-dialog' style={{ opacity: visible ? 1 : 0, visibility: visible ? 'visible' : 'hidden' }}>
         <h2 className='modal-dialog__title'>Settings</h2>
         <hr />
@@ -58,29 +58,46 @@ function SoundSettings ({visible}) {
     <div className='modal-dialog__panel modal-dialog__panel--with-navigation scrollable'>
       <h3 className='text-primary'>Sounds</h3>
       <p>
-        Voice alerts can give confirmation of commands and relay important information.
-        They complement in text notifications, but are not the same.
+        ICARUS Terminal includes a voice assistant that can give confirmation of
+        commands and relay information about your ship and your surroundings.
       </p>
-      <p>
-        Audio will be played through the computer ICARUS Terminal is running on. 
+      <p className='text-danger'>
+        This feature is highly experimental and not compatible with all voices.
       </p>
-      <h4 className='text-primary'>Voice alerts</h4>
-      <select value={preferences?.voice ?? 'None'} disabled={!voices || !preferences} name='voices' onChange={async (e) => {
+      <h4 className='text-primary'>Voice assistant</h4>
+      <select
+        value={preferences?.voice ?? 'None'}
+        disabled={!voices || !preferences}
+        name='voices'
+        style={{width: '20rem'}}
+        onChange={async (e) => {
         const voice = e.target.value
         const newPreferences = JSON.parse(JSON.stringify(preferences))
         newPreferences.voice = voice === 'None' ? null : voice
         setPreferences(await sendEvent('setPreferences', newPreferences))
         if (voice !== 'None') {
-          sendEvent('speakText', { text: `Voice alerts will use the voice ${voice}`, voice })
+          sendEvent('testVoice', { voice })
         }
       }}>
-        <option value='None'>None</option>
-        <option disabled>-</option>
-        {voices && voices.map(voice => <option key={`voice_${voice}`}>{voice}</option>)}
+        {voices && preferences && <>
+          <option value='None'>None</option>
+          <option disabled>─</option>
+          {voices && voices.map(voice => <option key={`voice_${voice}`}>{voice}</option>)}
+        </>}
       </select>
+      <br/><br/>
+      <h4 className='text-primary'>About voice assistant</h4>
+      <p>
+        The current implementation is only intended for debugging / testing. 
+      </p>
+      <p>
+        Audio will be played through the computer ICARUS Terminal is running on. 
+      </p>
+      <p>
+        This setting uses your computers native Text To Speech capabilities.
+      </p>
       <p>
-        Note: This setting uses your computers native Text To Speech capability. Third party / commercial
-        voices can provide improved voice audio quality.
+        Third party / commercial voices can provide improved voice audio quality.
       </p>
     </div>
   )
@@ -118,7 +135,7 @@ function ThemeSettings () {
 
   return (
     <div className='modal-dialog__panel modal-dialog__panel--with-navigation scrollable'>
-      <h3 className='text-primary'>Theme settings</h3>
+      <h3 className='text-primary'>Theme</h3>
       <p>
         You can select a primary and secondary theme color and adjust the contrast for each color using the sliders.
       </p>

diff --git a/src/client/css/form/select.css b/src/client/css/form/select.css
@@ -0,0 +1,12 @@
+select {
+  appearance: none;
+  margin: .5rem 0;
+  background: var(--color-primary-dark);
+  border: .2rem solid var(--color-primary);
+  color: var(--color-primary);
+  padding: .25rem .5rem;
+  font-family: "Jura", sans-serif;
+  font-size: var(--base-font-size);
+  font-weight: bold;
+  border-color: var(--color-primary);
+}
diff --git a/src/client/css/main.css b/src/client/css/main.css
@@ -13,6 +13,7 @@
 @import "form/button-group.css";
 @import "form/input.css";
 @import "form/checkbox.css";
+@import "form/select.css";
 @import "text.css";
 @import "table.css";
 
@@ -203,8 +204,9 @@ body::after {
   bottom: 0;
   right: 0;
   z-index: 10000;
-  background: rgba(0,0,255,.06);
+  background: rgba(0,0,255,.05);
   pointer-events: none;
+  z-index: 30000;
 }
 
 header {

diff --git a/src/client/css/ui/layout.css b/src/client/css/ui/layout.css
@@ -1,5 +1,4 @@
 .layout__overlay {
-  z-index: 100;
   pointer-events: none;
 }
 

diff --git a/src/client/css/ui/modal-dialog.css b/src/client/css/ui/modal-dialog.css
@@ -5,7 +5,7 @@
   left: 0;
   right: 0;
   background: rgba(0,0,0,.5);
-  backdrop-filter: blur(.25rem);
+  backdrop-filter: blur(.1rem);
   z-index: 19999;
   transition: .5s ease-in-out;
 }

diff --git a/src/client/pages/nav/list.js b/src/client/pages/nav/list.js
@@ -89,7 +89,7 @@ export default function NavListPage () {
 
   return (
     <>
-      <div className='modal-dialog__background' style={{ opacity: helpVisible ? 1 : 0, visibility: helpVisible ? 'visible' : 'hidden' }}/>
+      <div className='modal-dialog__background' style={{ opacity: helpVisible ? 1 : 0, visibility: helpVisible ? 'visible' : 'hidden' }} onClick={() => setHelpVisible(!helpVisible)}/>
       <div className='modal-dialog' style={{ opacity: helpVisible ? 1 : 0, visibility: helpVisible ? 'visible' : 'hidden' }}>
         <h2 className='modal-dialog__title'>Help</h2>
         <hr />

diff --git a/src/service/lib/event-handlers.js b/src/service/lib/event-handlers.js
@@ -25,6 +25,7 @@ const KEYBINDS_MAP = {
   hardpoints: 'DeployHardpointToggle'
 }
 
+// FIXME Refactor Preferences handling into a singleton
 const PREFERENCES_DIR = path.join(os.homedir(), 'AppData', 'Local', 'ICARUS Terminal')
 const PREFERENCES_FILE = path.join(PREFERENCES_DIR, 'Preferences.json')
 
@@ -37,16 +38,11 @@ const Inventory = require('./event-handlers/inventory')
 const CmdrStatus = require('./event-handlers/cmdr-status')
 const NavRoute = require('./event-handlers/nav-route')
 const TextToSpeech = require('./event-handlers/text-to-speech')
-const { clear } = require('console')
 
 class EventHandlers {
-  constructor ({ eliteLog, eliteJson, preferences }) {
+  constructor ({ eliteLog, eliteJson }) {
     this.eliteLog = eliteLog
     this.eliteJson = eliteJson
-    this.preferences = preferences
-
-    // TTS needs access to preferences (e.g. is tts on/off, which voice to)
-    this.textToSpeech = new TextToSpeech({ eliteLog, eliteJson, preferences })
 
     this.system = new System({ eliteLog })
     this.shipStatus = new ShipStatus({ eliteLog, eliteJson })
@@ -58,45 +54,18 @@ class EventHandlers {
     // These handlers depend on calls to other handlers
     this.blueprints = new Blueprints({ engineers: this.engineers, materials: this.materials, shipStatus: this.shipStatus })
     this.navRoute = new NavRoute({ eliteLog, eliteJson, system: this.system })
+    this.textToSpeech = new TextToSpeech({ eliteLog, eliteJson, cmdrStatus: this.cmdrStatus, shipStatus: this.shipStatus })
 
-    this.currentCmdrStatus = null
-    this.lastVoiceAlertDebounce = false
     return this
   }
 
   // logEventHandler is fired on every in-game log event
   logEventHandler(logEvent) {
-    this.textToSpeech.speechEventHandler(logEvent)
+    this.textToSpeech.logEventHandler(logEvent)
   }
 
-  async gameStateChangeHandler() {
-    const previousCmdStatus = JSON.parse(JSON.stringify(this.currentCmdrStatus))
-    this.currentCmdrStatus = await this.cmdrStatus.getCmdrStatus()
-
-    if (!this.lastVoiceAlertDebounce && previousCmdStatus) {
-      // TODO improve with better debounce function
-      this.lastVoiceAlertDebounce = true
-      setTimeout(() => { this.lastVoiceAlertDebounce = false }, 1000)
-
-      if (this.currentCmdrStatus?.flags?.lightsOn !== previousCmdStatus?.flags?.lightsOn) {
-        this.textToSpeech.speak(this.currentCmdrStatus?.flags?.lightsOn ? 'Lights On' : 'Lights Off')
-      }
-      if (this.currentCmdrStatus?.flags?.nightVision !== previousCmdStatus?.flags?.nightVision) {
-        this.textToSpeech.speak(this.currentCmdrStatus?.flags?.nightVision ? 'Night Vision On' : 'Night Vision Off')
-      }
-      if (this.currentCmdrStatus?.flags?.cargoScoopDeployed !== previousCmdStatus?.flags?.cargoScoopDeployed) {
-        this.textToSpeech.speak(this.currentCmdrStatus?.flags?.cargoScoopDeployed ? 'Cargo Hatch Open' : 'Cargo Hatch Closed')
-      }
-      if (this.currentCmdrStatus?.flags?.landingGearDown !== previousCmdStatus?.flags?.landingGearDown) {
-        this.textToSpeech.speak(this.currentCmdrStatus?.flags?.landingGearDown ? 'Landing Gear Down' : 'Landing Gear Up')
-      }
-      if (this.currentCmdrStatus?.flags?.hardpointsDeployed !== previousCmdStatus?.flags?.hardpointsDeployed) {
-        this.textToSpeech.speak(this.currentCmdrStatus?.flags?.hardpointsDeployed ? 'Hardpoints Deployed' : 'Hardpoints Retracted')
-      }
-      if (this.currentCmdrStatus?.flags?.hudInAnalysisMode !== previousCmdStatus?.flags?.hudInAnalysisMode) {
-        this.textToSpeech.speak(this.currentCmdrStatus?.flags?.hudInAnalysisMode ? 'Analysis mode activated' : 'Combat mode activated')
-      }
-    }
+  gameStateChangeHandler(event) {
+    this.textToSpeech.gameStateChangeHandler(event)
   }
 
   // Return handlers for events that are fired from the client
@@ -135,7 +104,12 @@ class EventHandlers {
           return preferences
         },
         getVoices: () => this.textToSpeech.getVoices(),
-        speakText: ({text, voice}) => this.textToSpeech.speak(text, voice, true),
+        testVoice: ({voice}) => {
+          // Escape voice name when passing as text as precaution to clean 
+          // input (NB: voice name argument is checked internally)
+          const text = `Voice assistant will use ${voice.replace(/[^a-z0-9 -]/gi,'')}`
+          this.textToSpeech.speak(text, voice, true)
+        },
         toggleSwitch: async ({ switchName }) => {
           return false
           /*

diff --git a/src/service/lib/event-handlers/text-to-speech.js b/src/service/lib/event-handlers/text-to-speech.js
@@ -1,57 +1,95 @@
 const os = require('os')
 const path = require('path')
 const fs = require('fs')
-const say = require('say')
+const say = require('../say')
 
+// FIXME Refactor Preferences handling into a singleton
 const PREFERENCES_DIR = path.join(os.homedir(), 'AppData', 'Local', 'ICARUS Terminal')
 const PREFERENCES_FILE = path.join(PREFERENCES_DIR, 'Preferences.json')
 
 class TextToSpeech {
-  constructor ({ eliteLog, eliteJson, preferences }) {
+  constructor ({ eliteLog, eliteJson, cmdrStatus, shipStatus }) {
     this.eliteLog = eliteLog
     this.eliteJson = eliteJson
-    this.preferences = preferences || {}
+    this.cmdrStatus = cmdrStatus
+    this.shipStatus = shipStatus
+
+    this.currentCmdrStatus = null
+    this.voiceAlertDebounce = null
+
     return this
   }
 
   async speak(text, voice, force) {
     // Only fire if Text To Speech voice has been selected in preferences
-    this.preferences = fs.existsSync(PREFERENCES_FILE) ? JSON.parse(fs.readFileSync(PREFERENCES_FILE)) : {}
-    if (!force && !this?.preferences?.voice) return
-    const _voice = voice || this?.preferences?.voice
+    const preferences = fs.existsSync(PREFERENCES_FILE) ? JSON.parse(fs.readFileSync(PREFERENCES_FILE)) : {}
+    if (!force && !preferences?.voice) return
+    const _voice = voice || preferences?.voice || (await this.getVoices())[0]
+
+    // Only allow valid voice names (also combats potential shell escaping)
+    if (!_voice || !(await this.getVoices()).includes(_voice)) return 
+
     say.speak(text, _voice)
   }
 
-  speechEventHandler(message) {
-    if (message.event === 'StartJump' && message.StarSystem) this.speak(`Jumping to ${message.StarSystem}`)
-    if (message.event === 'FSDJump') this.speak(`Jump complete. Arrived in ${message.StarSystem}`)
-    if (message.event === 'ApproachBody') this.speak(`Approaching ${message.Body}`)
-    if (message.event === 'LeaveBody') this.speak(`Leaving ${message.Body}`)
-    if (message.event === 'NavRoute') this.speak('New route plotted')
-    if (message.event === 'DockingGranted') this.speak(`Docking at ${message.StationName}`)
-    if (message.event === 'Docked') this.speak(`Docked at ${message.StationName}`)
-    if (message.event === 'Undocked') this.speak(`Now leaving ${message.StationName}`)
-    if (message.event === 'ApproachSettlement') this.speak(`Approaching ${message.Name}`)
-    if (message.event === 'MarketBuy') this.speak(`Purchased ${message.Count} ${message.Count === 1 ? 'tonne' : `tonnes`} of ${message.Type_Localised || message.Type}`)
-    if (message.event === 'MarketSell') this.speak(`Sold  ${message.Count} ${message.Count === 1 ? 'tonne' : `tonnes`} of  ${message.Type_Localised || message.Type}`)
-    if (message.event === 'BuyDrones') this.speak(`Purchased ${message.Count} Limpet ${message.Count === 1 ? 'Drone' : `Drones`}`)
-    if (message.event === 'SellDrones') this.speak(`Sold ${message.Count} Limpet ${message.Count === 1 ? 'Drone' : `Drones`}`)
-    if (message.event === 'CargoDepot' && message.UpdateType === 'Collect') this.speak(`Collected ${message.Count} ${message.Count === 1 ? 'tonne' : `tonnes`} of ${message.CargoType.replace(/([a-z])([A-Z])/g, '$1 $2')}`)
-    if (message.event === 'CargoDepot' && message.UpdateType === 'Deliver') this.speak(`Delivered  ${message.Count} ${message.Count === 1 ? 'tonne' : `tonnes`} of ${message.CargoType.replace(/([a-z])([A-Z])/g, '$1 $2')}`)
-    if (message.event === 'Scanned') this.speak('Scan detected')
-    if (message.event === 'FSSDiscoveryScan') {
-      if (message.NonBodyCount > 0) {
-        this.speak(`Discovery Scan Complete. ${message.BodyCount} ${message.BodyCount === 1 ? 'Body' : 'Bodies'} found and ${message.NonBodyCount} other ${message.NonBodyCount === 1 ? 'object' : 'objects'} detected in system.`)
-      } else {
-        this.speak(`Discovery Scan Complete. ${message.BodyCount} ${message.BodyCount === 1 ? 'Body' : 'Bodies'} found in system.`)
+  logEventHandler(logEvent) {
+    if (logEvent.event === 'StartJump' && logEvent.StarSystem) this.speak(`Jumping to ${logEvent.StarSystem}`)
+    if (logEvent.event === 'FSDJump') this.speak(`Jump complete. Arrived in ${logEvent.StarSystem}`)
+    if (logEvent.event === 'ApproachBody') this.speak(`Approaching ${logEvent.Body}`)
+    if (logEvent.event === 'LeaveBody') this.speak(`Leaving ${logEvent.Body}`)
+    if (logEvent.event === 'NavRoute') this.speak('New route plotted')
+    if (logEvent.event === 'DockingGranted') this.speak(`Docking at ${logEvent.StationName}`)
+    if (logEvent.event === 'Docked') this.speak(`Docked at ${logEvent.StationName}`)
+    if (logEvent.event === 'Undocked') this.speak(`Now leaving ${logEvent.StationName}`)
+    if (logEvent.event === 'ApproachSettlement') this.speak(`Approaching ${logEvent.Name}`)
+    if (logEvent.event === 'MarketBuy') this.speak(`Purchased ${logEvent.Count} ${logEvent.Count === 1 ? 'tonne' : `tonnes`} of ${logEvent.Type_Localised || logEvent.Type}`)
+    if (logEvent.event === 'MarketSell') this.speak(`Sold  ${logEvent.Count} ${logEvent.Count === 1 ? 'tonne' : `tonnes`} of  ${logEvent.Type_Localised || logEvent.Type}`)
+    if (logEvent.event === 'BuyDrones') this.speak(`Purchased ${logEvent.Count} Limpet ${logEvent.Count === 1 ? 'Drone' : `Drones`}`)
+    if (logEvent.event === 'SellDrones') this.speak(`Sold ${logEvent.Count} Limpet ${logEvent.Count === 1 ? 'Drone' : `Drones`}`)
+    if (logEvent.event === 'CargoDepot' && logEvent.UpdateType === 'Collect') this.speak(`Collected ${logEvent.Count} ${logEvent.Count === 1 ? 'tonne' : `tonnes`} of ${logEvent.CargoType.replace(/([a-z])([A-Z])/g, '$1 $2')}`)
+    if (logEvent.event === 'CargoDepot' && logEvent.UpdateType === 'Deliver') this.speak(`Delivered  ${logEvent.Count} ${logEvent.Count === 1 ? 'tonne' : `tonnes`} of ${logEvent.CargoType.replace(/([a-z])([A-Z])/g, '$1 $2')}`)
+    if (logEvent.event === 'Scanned') this.speak('Scan detected')
+    if (logEvent.event === 'FSSDiscoveryScan') this.speak(`Discovery Scan Complete. ${logEvent.BodyCount} ${logEvent.BodyCount === 1 ? 'Body' : 'Bodies'} found in system.`)
+  }
+
+  async gameStateChangeHandler() {
+    // TODO Refine so this logic is only evaluated on changes to Status.json
+    const previousCmdStatus = JSON.parse(JSON.stringify(this.currentCmdrStatus))
+    this.currentCmdrStatus = await this.cmdrStatus.getCmdrStatus()
+    const shipStatus = await this.shipStatus.getShipStatus()
+
+    // Only evaluate these if we are on board the ship, there is a previous 
+    // status (i.e. not at startup) and we have not recently alerted
+    if (shipStatus?.onBoard && previousCmdStatus && !this.voiceAlertDebounce) {
+      // TODO improve with better debounce function
+      this.voiceAlertDebounce = true
+      setTimeout(() => { this.voiceAlertDebounce = false }, 1000)
+
+      // These functions handle changes to ship state
+      if (this.currentCmdrStatus?.flags?.lightsOn !== previousCmdStatus?.flags?.lightsOn) {
+        this.speak(this.currentCmdrStatus?.flags?.lightsOn ? 'Lights On' : 'Lights Off')
+      }
+      if (this.currentCmdrStatus?.flags?.nightVision !== previousCmdStatus?.flags?.nightVision) {
+        this.speak(this.currentCmdrStatus?.flags?.nightVision ? 'Night Vision On' : 'Night Vision Off')
+      }
+      if (this.currentCmdrStatus?.flags?.cargoScoopDeployed !== previousCmdStatus?.flags?.cargoScoopDeployed) {
+        this.speak(this.currentCmdrStatus?.flags?.cargoScoopDeployed ? 'Cargo Hatch Open' : 'Cargo Hatch Closed')
+      }
+      if (this.currentCmdrStatus?.flags?.landingGearDown !== previousCmdStatus?.flags?.landingGearDown) {
+        this.speak(this.currentCmdrStatus?.flags?.landingGearDown ? 'Landing Gear Down' : 'Landing Gear Up')
+      }
+      if (this.currentCmdrStatus?.flags?.supercruise === false && this.currentCmdrStatus?.flags?.hardpointsDeployed !== previousCmdStatus?.flags?.hardpointsDeployed) {
+        this.speak(this.currentCmdrStatus?.flags?.hardpointsDeployed ? 'Hardpoints Deployed' : 'Hardpoints Retracted')
+      }
+      if (this.currentCmdrStatus?.flags?.hudInAnalysisMode !== previousCmdStatus?.flags?.hudInAnalysisMode) {
+        this.speak(this.currentCmdrStatus?.flags?.hudInAnalysisMode ? 'Analysis mode activated' : 'Combat mode activated')
       }
     }
   }
 
   async getVoice() {
-    this.preferences = fs.existsSync(PREFERENCES_FILE) ? JSON.parse(fs.readFileSync(PREFERENCES_FILE)) : {}
-    if (this?.preferences?.voice)
-      return this.preferences.voice
+    const preferences = fs.existsSync(PREFERENCES_FILE) ? JSON.parse(fs.readFileSync(PREFERENCES_FILE)) : {}
+    if (preferences?.voice) return preferences.voice
 
     return await this.getVoices()[0]
   }