Respect "expectingInput" in "inputHint" (#2149)

* Use React to start/stop microphone instead of saga

* Respecting input hints

* Fix

* Fix 2

* Fix 3

* Fix 4

* Stop speak on typing

* Stop dictate on ignoringInput

* Only handle event when supposed to

* Fix 5

* Should only mark to speak if not empty

* Add comment

* Clean up

* Rename

* Add entry

* Add comment

* Clean up

* Send typing indicator on speech

CHANGELOG.md

@@ -67,6 +67,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 -  Fix [#2134](https://github.com/Microsoft/BotFramework-WebChat/issues/2134). Added `azure-pipelines.yml` for embed package, by [@compulim](https://github.com/compulim) in PR [#2135](https://github.com/Microsoft/BotFramework-WebChat/pull/2135)
 -  Fix [#2106](https://github.com/Microsoft/BotFramework-WebChat/issues/2016). Fix `AdaptiveCardHostConfig` warning associated with the `CommonCard` component, by [@tdurnford](https://github.com/tdurnford) in PR [#2108](https://github.com/Microsoft/BotFramework-WebChat/pull/2108)
 -  Fix [#1872](https://github.com/Microsoft/BotFramework-WebChat/issues/1872). Fixed `observeOnce` to unsubscribe properly, by [@compulim](https://github.com/compulim) in PR [#2140](https://github.com/Microsoft/BotFramework-WebChat/pull/2140)
+-  Fix [#2022](https://github.com/Microsoft/BotFramework-WebChat/issues/2022). Fixed `"expectingInput"` in `inputHint` is not respected, by [@compulim](https://github.com/compulim) and [@corinagum](https://github.com/corinagum) in PR [#2149](https://github.com/Microsoft/BotFramework-WebChat/pull/2149)
 
 ### Samples
 

packages/component/src/BasicSendBox.js

@@ -51,13 +51,12 @@ const BasicSendBox = ({ className, dictationStarted, styleSet, webSpeechPonyfill
 
 BasicSendBox.defaultProps = {
   className: '',
-  dictationStarted: false,
   webSpeechPonyfill: undefined
 };
 
 BasicSendBox.propTypes = {
   className: PropTypes.string,
-  dictationStarted: PropTypes.bool,
+  dictationStarted: PropTypes.bool.isRequired,
   styleSet: PropTypes.shape({
     sendBox: PropTypes.any.isRequired
   }).isRequired,
@@ -66,8 +65,15 @@ BasicSendBox.propTypes = {
   })
 };
 
-export default connectToWebChat(({ dictateState, styleSet, webSpeechPonyfill }) => ({
-  dictationStarted: dictateState === STARTING || dictateState === DICTATING,
+// TODO: [P3] We should consider exposing core/src/definitions and use it instead
+function activityIsSpeakingOrQueuedToSpeak({ channelData: { speak } = {} }) {
+  return !!speak;
+}
+
+export default connectToWebChat(({ activities, dictateState, styleSet, webSpeechPonyfill }) => ({
+  dictationStarted:
+    (dictateState === STARTING || dictateState === DICTATING) &&
+    !activities.filter(activityIsSpeakingOrQueuedToSpeak).length,
   styleSet,
   webSpeechPonyfill
 }))(BasicSendBox);

packages/component/src/Dictation.js

@@ -20,6 +20,7 @@ class Dictation extends React.Component {
 
   handleDictate({ result: { transcript } = {} }) {
     const {
+      dictateState,
       setDictateInterims,
       setDictateState,
       setSendBox,
@@ -28,43 +29,51 @@ class Dictation extends React.Component {
       submitSendBox
     } = this.props;
 
-    setDictateInterims([]);
-    setDictateState(IDLE);
-    stopDictate();
+    if (dictateState === DICTATING || dictateState === STARTING) {
+      setDictateInterims([]);
+      setDictateState(IDLE);
+      stopDictate();
 
-    if (transcript) {
-      setSendBox(transcript);
-      submitSendBox('speech');
-      startSpeakingActivity();
+      if (transcript) {
+        setSendBox(transcript);
+        submitSendBox('speech');
+        startSpeakingActivity();
+      }
     }
   }
 
   handleDictating({ results = [] }) {
-    const { setDictateInterims, setDictateState, setSendBox } = this.props;
+    const { dictateState, postActivity, sendTypingIndicator, setDictateInterims, setDictateState } = this.props;
 
-    const interims = results.map(({ transcript }) => transcript);
+    if (dictateState === DICTATING || dictateState === STARTING) {
+      const interims = results.map(({ transcript }) => transcript);
 
-    setDictateInterims(interims);
-    setDictateState(DICTATING);
-
-    // This is for two purposes:
-    // 1. Set send box will also trigger send typing
-    // 2. If the user cancelled out, the interim result will be in the send box so the user can update it before send
-    setSendBox(interims.join(' '));
+      setDictateInterims(interims);
+      setDictateState(DICTATING);
+      sendTypingIndicator && postActivity({ type: 'typing' });
+    }
   }
 
   handleError(event) {
-    const { onError, setDictateState, stopDictate } = this.props;
+    const { dictateState, onError, setDictateState, stopDictate } = this.props;
 
-    setDictateState(IDLE);
-    stopDictate();
+    if (dictateState === DICTATING || dictateState === STARTING) {
+      setDictateState(IDLE);
+      stopDictate();
 
-    onError && onError(event);
+      onError && onError(event);
+    }
   }
 
   render() {
     const {
-      props: { dictateState, disabled, language, webSpeechPonyfill: { SpeechGrammarList, SpeechRecognition } = {} },
+      props: {
+        dictateState,
+        disabled,
+        language,
+        numSpeakingActivities,
+        webSpeechPonyfill: { SpeechGrammarList, SpeechRecognition } = {}
+      },
       handleDictate,
       handleDictating,
       handleError
@@ -78,7 +87,7 @@ class Dictation extends React.Component {
         onProgress={handleDictating}
         speechGrammarList={SpeechGrammarList}
         speechRecognition={SpeechRecognition}
-        started={!disabled && (dictateState === STARTING || dictateState === DICTATING)}
+        started={!disabled && (dictateState === STARTING || dictateState === DICTATING) && !numSpeakingActivities}
       />
     );
   }
@@ -94,7 +103,9 @@ Dictation.propTypes = {
   dictateState: PropTypes.number.isRequired,
   disabled: PropTypes.bool,
   language: PropTypes.string.isRequired,
+  numSpeakingActivities: PropTypes.number.isRequired,
   onError: PropTypes.func,
+  postActivity: PropTypes.func.isRequired,
   setDictateInterims: PropTypes.func.isRequired,
   setDictateState: PropTypes.func.isRequired,
   setSendBox: PropTypes.func.isRequired,
@@ -109,9 +120,12 @@ Dictation.propTypes = {
 
 export default connectToWebChat(
   ({
+    activities,
     dictateState,
     disabled,
     language,
+    postActivity,
+    sendTypingIndicator,
     setDictateInterims,
     setDictateState,
     setSendBox,
@@ -123,6 +137,9 @@ export default connectToWebChat(
     dictateState,
     disabled,
     language,
+    numSpeakingActivities: activities.filter(({ channelData: { speak } = {} }) => speak).length,
+    postActivity,
+    sendTypingIndicator,
     setDictateInterims,
     setDictateState,
     setSendBox,

packages/component/src/SendBox/MicrophoneButton.js

@@ -31,10 +31,11 @@ const ROOT_CSS = css({
 
 const connectMicrophoneButton = (...selectors) =>
   connectToWebChat(
-    ({ disabled, dictateState, language, startDictate, stopDictate }) => ({
+    ({ disabled, dictateInterims, dictateState, language, setSendBox, startDictate, stopDictate }) => ({
       click: () => {
         if (dictateState === DictateState.STARTING || dictateState === DictateState.DICTATING) {
           stopDictate();
+          setSendBox(dictateInterims.join(' '));
         } else {
           startDictate();
         }

packages/component/src/SendBox/TextBox.js

@@ -17,11 +17,12 @@ const ROOT_CSS = css({
 
 const connectSendTextBox = (...selectors) =>
   connectToWebChat(
-    ({ disabled, focusSendBox, language, scrollToEnd, sendBoxValue, setSendBox, submitSendBox }) => ({
+    ({ disabled, focusSendBox, language, scrollToEnd, sendBoxValue, setSendBox, stopDictate, submitSendBox }) => ({
       disabled,
       language,
       onChange: ({ target: { value } }) => {
         setSendBox(value);
+        stopDictate();
       },
       onKeyPress: event => {
         const { key, shiftKey } = event;

packages/core/src/definitions/speakingActivity.js

@@ -1,3 +1,5 @@
+// If true, the activity is in the queue and needs to be spoken.
+
 export default function speakingActivity(activity) {
   return activity.channelData && activity.channelData.speak;
 }

packages/core/src/reducer.ts

@@ -10,6 +10,7 @@ import referenceGrammarID from './reducers/referenceGrammarID';
 import sendBoxValue from './reducers/sendBoxValue';
 import sendTimeout from './reducers/sendTimeout';
 import sendTypingIndicator from './reducers/sendTypingIndicator';
+import shouldSpeakIncomingActivity from './reducers/shouldSpeakIncomingActivity';
 import suggestedActions from './reducers/suggestedActions';
 
 export default combineReducers({
@@ -23,6 +24,7 @@ export default combineReducers({
   sendBoxValue,
   sendTimeout,
   sendTypingIndicator,
+  shouldSpeakIncomingActivity,
   suggestedActions,
 
   // TODO: [P3] Take this deprecation code out when releasing on or after January 13 2020

packages/core/src/reducers/shouldSpeakIncomingActivity.js

@@ -0,0 +1,19 @@
+import { START_SPEAKING_ACTIVITY } from '../../lib/actions/startSpeakingActivity';
+import { STOP_SPEAKING_ACTIVITY } from '../../lib/actions/stopSpeakingActivity';
+
+export default function(state = false, { type }) {
+  switch (type) {
+    case START_SPEAKING_ACTIVITY:
+      state = true;
+      break;
+
+    case STOP_SPEAKING_ACTIVITY:
+      state = false;
+      break;
+
+    default:
+      break;
+  }
+
+  return state;
+}

packages/core/src/sagas.js

@@ -5,17 +5,16 @@ import connectionStatusUpdateSaga from './sagas/connectionStatusUpdateSaga';
 import connectSaga from './sagas/connectSaga';
 import detectSlowConnectionSaga from './sagas/detectSlowConnectionSaga';
 import incomingActivitySaga from './sagas/incomingActivitySaga';
-import markActivityForSpeakOnIncomingActivityFromOthersSaga from './sagas/markActivityForSpeakOnIncomingActivityFromOthersSaga';
 import markAllAsSpokenOnStopSpeakActivitySaga from './sagas/markAllAsSpokenOnStopSpeakActivitySaga';
 import postActivitySaga from './sagas/postActivitySaga';
 import removeIncomingTypingAfterIntervalSaga from './sagas/removeIncomingTypingAfterIntervalSaga';
 import sendEventToPostActivitySaga from './sagas/sendEventToPostActivitySaga';
 import sendFilesToPostActivitySaga from './sagas/sendFilesToPostActivitySaga';
-import sendMessageToPostActivitySaga from './sagas/sendMessageToPostActivitySaga';
 import sendMessageBackToPostActivitySaga from './sagas/sendMessageBackToPostActivitySaga';
+import sendMessageToPostActivitySaga from './sagas/sendMessageToPostActivitySaga';
 import sendPostBackToPostActivitySaga from './sagas/sendPostBackToPostActivitySaga';
 import sendTypingIndicatorOnSetSendBoxSaga from './sagas/sendTypingIndicatorOnSetSendBoxSaga';
-import startDictateAfterSpeakActivitySaga from './sagas/startDictateAfterSpeakActivitySaga';
+import speakActivityAndStartDictateOnIncomingActivityFromOthersSaga from './sagas/speakActivityAndStartDictateOnIncomingActivityFromOthersSaga';
 import startSpeakActivityOnPostActivitySaga from './sagas/startSpeakActivityOnPostActivitySaga';
 import stopDictateOnCardActionSaga from './sagas/stopDictateOnCardActionSaga';
 import stopSpeakingActivityOnInputSaga from './sagas/stopSpeakingActivityOnInputSaga';
@@ -29,17 +28,16 @@ export default function* sagas() {
   yield fork(connectSaga);
   yield fork(detectSlowConnectionSaga);
   yield fork(incomingActivitySaga);
-  yield fork(markActivityForSpeakOnIncomingActivityFromOthersSaga);
   yield fork(markAllAsSpokenOnStopSpeakActivitySaga);
   yield fork(postActivitySaga);
   yield fork(removeIncomingTypingAfterIntervalSaga);
   yield fork(sendEventToPostActivitySaga);
   yield fork(sendFilesToPostActivitySaga);
-  yield fork(sendMessageToPostActivitySaga);
   yield fork(sendMessageBackToPostActivitySaga);
+  yield fork(sendMessageToPostActivitySaga);
   yield fork(sendPostBackToPostActivitySaga);
   yield fork(sendTypingIndicatorOnSetSendBoxSaga);
-  yield fork(startDictateAfterSpeakActivitySaga);
+  yield fork(speakActivityAndStartDictateOnIncomingActivityFromOthersSaga);
   yield fork(startSpeakActivityOnPostActivitySaga);
   yield fork(stopDictateOnCardActionSaga);
   yield fork(stopSpeakingActivityOnInputSaga);

packages/core/src/sagas/markAllAsSpokenOnStopSpeakActivitySaga.js

@@ -13,6 +13,7 @@ function* markAllAsSpoken() {
   }
 }
 
+// TODO: [P4] We should turn this into a reducer instead
 export default function* markAllAsSpokenOnStopSpeakActivitySaga() {
   yield takeEvery(STOP_SPEAKING_ACTIVITY, markAllAsSpoken);
 }

packages/core/src/sagas/speakActivityAndStartDictateOnIncomingActivityFromOthersSaga.js

@@ -0,0 +1,32 @@
+import { put, select, takeEvery } from 'redux-saga/effects';
+
+import { INCOMING_ACTIVITY } from '../actions/incomingActivity';
+import markActivity from '../actions/markActivity';
+import shouldSpeakIncomingActivitySelector from '../selectors/shouldSpeakIncomingActivity';
+import speakableActivity from '../definitions/speakableActivity';
+import startDictate from '../actions/startDictate';
+import stopDictate from '../actions/stopDictate';
+import whileConnected from './effects/whileConnected';
+
+function* speakActivityAndStartDictateOnIncomingActivityFromOthers({ userID }) {
+  yield takeEvery(({ payload, type }) => type === INCOMING_ACTIVITY && payload.activity.from.id !== userID, function*({
+    payload: { activity }
+  }) {
+    const shouldSpeakIncomingActivity = yield select(shouldSpeakIncomingActivitySelector);
+    const shouldSpeak = speakableActivity(activity) && shouldSpeakIncomingActivity;
+
+    if (shouldSpeak && (activity.speak || activity.text)) {
+      yield put(markActivity(activity, 'speak', true));
+    }
+
+    if (activity.inputHint === 'expectingInput' || (shouldSpeak && activity.inputHint !== 'ignoringInput')) {
+      yield put(startDictate());
+    } else if (activity.inputHint === 'ignoringInput') {
+      yield put(stopDictate());
+    }
+  });
+}
+
+export default function* speakActivityAndStartDictateOnIncomingActivityFromOthersSaga() {
+  yield whileConnected(speakActivityAndStartDictateOnIncomingActivityFromOthers);
+}