From 7306d15268b6a1472b7c30c085e78465fb6b04f5 Mon Sep 17 00:00:00 2001 From: Aravind Raveendran Date: Thu, 22 Feb 2024 15:14:57 +1100 Subject: [PATCH] Ignore ontrack event and adding remote track and calling project for every sourceId obtained in active event --- src/components/errorview/ErrorView.tsx | 2 + src/screens/multiview/MultiView.tsx | 272 ++++++++++-------- .../singleStreamView/SingleStreamView.tsx | 16 +- src/store/reducers/viewer.js | 65 +---- src/types/RemoteTrackSource.types.ts | 12 + 5 files changed, 178 insertions(+), 189 deletions(-) create mode 100644 src/types/RemoteTrackSource.types.ts diff --git a/src/components/errorview/ErrorView.tsx b/src/components/errorview/ErrorView.tsx index ce0fcc9..9a031f0 100644 --- a/src/components/errorview/ErrorView.tsx +++ b/src/components/errorview/ErrorView.tsx @@ -24,6 +24,8 @@ const ErrorView = (props) => { useEffect(() => { LogBox.ignoreLogs(['Error generating token.']); LogBox.ignoreLogs(['Error while getting subscriber connection path.']); + LogBox.ignoreLogs(['Reconnection failed']); + LogBox.ignoreLogs(['WebSocket not connected']); }, []); useEffect(() => { diff --git a/src/screens/multiview/MultiView.tsx b/src/screens/multiview/MultiView.tsx index 5f8d207..0607afc 100644 --- a/src/screens/multiview/MultiView.tsx +++ b/src/screens/multiview/MultiView.tsx @@ -1,4 +1,11 @@ -import { Director, View as MillicastView, Logger as MillicastLogger } from '@millicast/sdk'; +import { + Director, + View as MillicastView, + Logger as MillicastLogger, + MediaTrackInfo, + ViewProjectSourceMapping, + MediaStreamSource, +} from '@millicast/sdk'; import { useNetInfo } from '@react-native-community/netinfo'; import React, { useEffect, useRef, useState } from 'react'; import { @@ -8,15 +15,17 @@ import { Text, SafeAreaView, FlatList, - Platform, AppState, Pressable, DeviceEventEmitter, BackHandler, + Platform, + NativeModules, } from 'react-native'; import { RTCView } from 'react-native-webrtc'; import { useSelector, useDispatch } from 'react-redux'; +import { RemoteTrackSource } from '../../types/RemoteTrackSource.types'; import { Routes } from '../../types/routes.types'; window.Logger = MillicastLogger; @@ -28,11 +37,9 @@ export const MultiView = ({ navigation }) => { const streamName = useSelector((state) => state.viewerReducer.streamName); const accountId = useSelector((state) => state.viewerReducer.accountId); const isMediaSet = useSelector((state) => state.viewerReducer.isMediaSet); - const streams = useSelector((state) => state.viewerReducer.streams); - const streamsProjecting = useSelector((state) => state.viewerReducer.streamsProjecting); + const remoteTrackSources = useSelector((state) => state.viewerReducer.remoteTrackSources); const sourceIds = useSelector((state) => state.viewerReducer.sourceIds); const playing = useSelector((state) => state.viewerReducer.playing); - const millicastView = useSelector((state) => state.viewerReducer.millicastView); const selectedSource = useSelector((state) => state.viewerReducer.selectedSource); const error = useSelector((state) => state.viewerReducer.error); const dispatch = useDispatch(); @@ -40,18 +47,18 @@ export const MultiView = ({ navigation }) => { const currentRoute = routes[index].name; const playingRef = useRef(null); playingRef.current = playing; - const streamsRef = useRef(null); + const remoteTrackSourcesRef = useRef(null); const selectedSourceRef = useRef(null); - const millicastViewRef = useRef(null); const sourceIdsRef = useRef([]); const netInfo = useNetInfo(); const [isReconnectionScheduled, setIsReconnectionScheduled] = useState(false); + remoteTrackSourcesRef.current = remoteTrackSources; selectedSourceRef.current = selectedSource; - streamsRef.current = streams; - millicastViewRef.current = millicastView; sourceIdsRef.current = sourceIds; + const millicastViewRef = useRef(); + const [columnsNumber, setColumnsNumber] = useState(1); const margin = margins(columnsNumber, false); const labelLayout = margins(columnsNumber, true); @@ -95,7 +102,7 @@ export const MultiView = ({ navigation }) => { stopStream(); } dispatch({ type: 'viewer/resetAll' }); - streamsRef.current = []; + remoteTrackSourcesRef.current = []; } }; }, [handleAppStateChange, stopStream]); @@ -114,90 +121,104 @@ export const MultiView = ({ navigation }) => { const resetState = () => { dispatch({ type: 'viewer/setPlaying', payload: false }); dispatch({ type: 'viewer/setIsMediaSet', payload: true }); - dispatch({ type: 'viewer/setStreams', payload: [] }); + dispatch({ type: 'viewer/resetRemoteTrackSources' }); dispatch({ type: 'viewer/setSelectedSource', payload: { url: null, mid: null }, }); - dispatch({ type: 'viewer/removeProjectingStreams' }); dispatch({ type: 'viewer/setSourceIds', payload: [] }); }; const subscribe = async () => { + if (millicastViewRef.current?.isActive()) { + return; + } const tokenGenerator = () => Director.getSubscriber({ streamName, streamAccountId: accountId, }); + // Create a new instance const view = new MillicastView(streamName, tokenGenerator, undefined, true); - // Set track event handler to receive streams from Publisher. - view.on('track', async (event) => { - dispatch({ type: 'viewer/onTrackEvent', payload: event }); - }); + view.on('broadcastEvent', async (event) => { + // Get event name and data + const { name, data } = event; + const { sourceId, tracks } = event.data as MediaStreamSource; + const { current: viewer } = millicastViewRef; + if (!viewer) { + return; + } - // Start connection to viewer - try { - view.on('broadcastEvent', async (event) => { - // Get event name and data - const { name, data } = event; - let sourceId; - - switch (name) { - case 'active': - sourceId = data.sourceId === null || data.sourceId.length === 0 ? 'main' : data.sourceId; - - if (sourceIds?.indexOf(sourceId) === -1) { - dispatch({ - type: 'viewer/addSourceId', - payload: sourceId, - }); - if (sourceId !== 'main' && sourceId !== null) { - addRemoteTrack(sourceId); - } - } - // A source has been started on the stream - dispatch({ type: 'viewer/setError', payload: null }); - break; - case 'inactive': - // A source has been stopped on the steam - sourceId = data.sourceId === null || data.sourceId.length === 0 ? 'main' : data.sourceId; - if (sourceIdsRef.current?.indexOf(sourceId) !== -1) { - dispatch({ - type: 'viewer/removeSourceId', - payload: sourceId, - }); - - const streamToRemove = streamsRef.current.find((stream) => stream.sourceId === data.sourceId); - dispatch({ - type: 'viewer/removeStream', - payload: streamToRemove, - }); + switch (name) { + case 'active': + if (sourceIdsRef.current?.indexOf(sourceId) === -1) { + const isFirstSource = sourceIdsRef.current.length === 0; + dispatch({ + type: 'viewer/addSourceId', + payload: sourceId, + }); + const newRemoteTrackSource = await addRemoteTrack(viewer, sourceId, tracks); + + if (Platform.OS === 'ios') { + const { AudioManager } = NativeModules; + AudioManager.routeAudioThroughDefaultSpeaker(); } - break; - case 'vad': - // A new source was multiplexed over the vad tracks - break; - case 'layers': + + await unprojectFromStream(viewer, newRemoteTrackSource); + const videoMapping = newRemoteTrackSource.projectMapping.filter((mapping) => mapping.media === 'video'); + const mapping = isFirstSource ? newRemoteTrackSource.projectMapping : videoMapping; + await viewer.project(sourceId, mapping); dispatch({ - type: 'viewer/setActiveLayers', - payload: data.medias?.['0']?.active, + type: 'viewer/addRemoteTrackSource', + payload: newRemoteTrackSource, }); - // Updated layer information for each simulcast/svc video track - break; - default: - console.log('Unknown event', name); - } - }); + } + // A source has been started on the stream + dispatch({ type: 'viewer/setError', payload: null }); + break; + case 'inactive': + // A source has been stopped on the steam + if (sourceIdsRef.current?.indexOf(sourceId) !== -1) { + dispatch({ + type: 'viewer/removeSourceId', + payload: sourceId, + }); + + const remoteTrackSourceToRemove = remoteTrackSourcesRef.current.find( + (remoteTrackSource) => remoteTrackSource.sourceId === sourceId, + ); + dispatch({ + type: 'viewer/removeRemoteTrackSource', + payload: remoteTrackSourceToRemove, + }); + } + break; + case 'vad': + // A new source was multiplexed over the vad tracks + break; + case 'layers': + dispatch({ + type: 'viewer/setActiveLayers', + payload: data.medias?.['0']?.active, + }); + // Updated layer information for each simulcast/svc video track + break; + default: + console.log('Unknown event', name); + } + }); + + millicastViewRef.current = view; + dispatch({ type: 'viewer/setMillicastView', payload: view }); + + // Start connection to viewer + try { await view.connect({ events: ['active', 'inactive', 'vad', 'layers', 'viewercount'], }); - dispatch({ type: 'viewer/setMillicastView', payload: view }); dispatch({ type: 'viewer/setError', payload: null }); - - millicastViewRef.current = view; } catch (e) { - console.log('Connection failed. Reason:', e); dispatch({ type: 'viewer/setError', payload: e }); setIsReconnectionScheduled(true); } @@ -207,13 +228,24 @@ export const MultiView = ({ navigation }) => { dispatch({ type: 'viewer/setSelectedSource', payload: { url, mid } }); try { - const listVideoMids = streamsRef.current.map((track) => track.videoMid).filter((x) => x !== '0' && x !== mid); + const listVideoMids = remoteTrackSourcesRef.current + .map((remoteTrackSource) => remoteTrackSource.videoMediaId) + .filter((x) => x !== '0' && x !== mid); await millicastViewRef.current.unproject(listVideoMids); } catch (error) { console.log('unproject error', error); } }; + const unprojectFromStream = async (viewer: MillicastView, source: RemoteTrackSource) => { + const mediaIds = []; + if (source.audioMediaId) mediaIds.push(source.audioMediaId); + if (source.videoMediaId) mediaIds.push(source.videoMediaId); + if (mediaIds.length) { + await viewer.unproject(mediaIds); + } + }; + useEffect(() => { checkOrientation(); const subscription = Dimensions.addEventListener('change', () => { @@ -227,8 +259,8 @@ export const MultiView = ({ navigation }) => { /* eslint no-param-reassign: ["error", { "props": false }] */ const changeStateOfMediaTracks = (value) => { - streams?.map((s) => - s.stream?.getTracks().forEach((videoTrack) => { + remoteTrackSourcesRef.current?.map((remoteTrackSource) => + remoteTrackSource.mediaStream?.getTracks().forEach((videoTrack) => { videoTrack.enabled = value; }), ); @@ -247,55 +279,47 @@ export const MultiView = ({ navigation }) => { await subscribe(); }; - const addRemoteTrack = async (sourceId) => { - if (millicastViewRef.current == null) { - return; - } - const isAlreadyProjected = streams.some((stream) => stream.sourceId === sourceId); - const isAlreadyProjecting = streamsProjecting.some((stream) => stream.sourceId === sourceId); - if (!isAlreadyProjected && !isAlreadyProjecting) { - dispatch({ - type: 'viewer/addProjectingStream', - payload: { sourceId }, - }); - // eslint-disable-next-line no-undef - const mediaStream = new MediaStream(); - const transceiver = await millicastViewRef.current.addRemoteTrack('video', [mediaStream]); - const mediaId = transceiver.mid; - await millicastViewRef.current.project(sourceId, [ - { - media: 'video', - mediaId, - trackId: 'video', - }, - ]); - dispatch({ - type: 'viewer/addStream', - payload: { stream: mediaStream, videoMid: mediaId, sourceId }, - }); - dispatch({ - type: 'viewer/removeProjectingStream', - payload: { sourceId }, - }); + const addRemoteTrack = async ( + viewer: MillicastView, + sourceId?: string, + trackInfo?: MediaTrackInfo[], + ): Promise => { + const mapping: ViewProjectSourceMapping[] = []; + const mediaStream = new MediaStream(); + + const trackAudio = trackInfo?.find(({ media }) => media == 'audio'); + const trackVideo = trackInfo?.find(({ media }) => media == 'video'); + + let audioMediaId: string | undefined; + let videoMediaId: string | undefined; + + if (trackAudio) { + const audioTransceiver = await viewer.addRemoteTrack('audio', [mediaStream]); + audioMediaId = audioTransceiver?.mid ?? undefined; + + if (audioMediaId) { + mapping.push({ media: 'audio', mediaId: audioMediaId, trackId: 'audio' }); + } } - }; - useEffect(() => { - const initializeMultiview = async () => { - try { - await Promise.all( - sourceIds?.map(async (sourceId) => { - if (sourceId !== 'main') { - addRemoteTrack(sourceId); - } - }), - ); - } catch (e) { - console.log('error', e); + if (trackVideo) { + const videoTransceiver = await viewer.addRemoteTrack('video', [mediaStream]); + videoMediaId = videoTransceiver?.mid ?? undefined; + + if (videoMediaId) { + mapping.push({ media: 'video', mediaId: videoMediaId, trackId: 'video' }); } + } + + return { + audioMediaId, + mediaStream, + projectMapping: mapping, + quality: 'Auto', + sourceId, + videoMediaId, }; - initializeMultiview(); - }, [addRemoteTrack, navigation, sourceIds]); + }; useEffect(() => { if (currentRoute !== Routes.ErrorView && error !== null) { @@ -307,7 +331,7 @@ export const MultiView = ({ navigation }) => { useEffect(() => { if (netInfo.isConnected === false) { - dispatch({ type: 'viewer/setStreams', payload: [] }); + dispatch({ type: 'viewer/resetRemoteTrackSources' }); dispatch({ type: 'viewer/setSelectedSource', payload: { url: null, mid: null }, @@ -348,7 +372,7 @@ export const MultiView = ({ navigation }) => { > { dispatch({ type: 'viewer/setSelectedSource', payload: { - url: item?.stream.toURL(), - mid: item?.videoMid, + url: item?.mediaStream.toURL(), + mid: item?.videoMediaId, }, }); navigation.navigate(Routes.SingleStreamView); }} > { const styles = makeStyles(); - const streams = useSelector((state) => state.viewerReducer.streams); + const remoteTrackSources = useSelector((state) => state.viewerReducer.remoteTrackSources); const selectedSource = useSelector((state) => state.viewerReducer.selectedSource); const millicastView = useSelector((state) => state.viewerReducer.millicastView); const dispatch = useDispatch(); - const streamsRef = useRef(null); + const remoteTrackSourcesRef = useRef(null); const selectedSourceRef = useRef(null); const millicastViewRef = useRef(null); selectedSourceRef.current = selectedSource; - streamsRef.current = streams; + remoteTrackSourcesRef.current = remoteTrackSources; millicastViewRef.current = millicastView; const [videoTileIndex, setVideoTileIndex] = useState(-1); @@ -75,9 +75,9 @@ export const SingleStreamView = ({ navigation }) => { useEffect(()=> { const {url, mid} = selectedSourceRef.current; - const streamIndex = streamsRef.current.findIndex( (element) => element.videoMid === mid && element.stream.toURL() === url); + const streamIndex = remoteTrackSourcesRef.current.findIndex( (remoteTrackSource) => remoteTrackSource.videoMediaId === mid && remoteTrackSource.mediaStream.toURL() === url); setVideoTileIndex(streamIndex); - }, [selectedSource, streams]); + }, [selectedSource, remoteTrackSources]); const openStreamStatsModel = () => { if(!isStreamStatsModelVisible) { @@ -109,8 +109,8 @@ export const SingleStreamView = ({ navigation }) => { const renderVideoItem = ({item}) => ( @@ -128,7 +128,7 @@ export const SingleStreamView = ({ navigation }) => { numColumns={1} horizontal={true} pagingEnabled={true} - data={streamsRef.current} + data={remoteTrackSourcesRef.current} keyExtractor={(_, index) => String(index)} style={{ width: width }} renderItem={renderVideoItem} diff --git a/src/store/reducers/viewer.js b/src/store/reducers/viewer.js index 587a40a..67bc033 100644 --- a/src/store/reducers/viewer.js +++ b/src/store/reducers/viewer.js @@ -6,9 +6,8 @@ const initialState = { accountId: null, playing: false, error: null, - streams: [], + remoteTrackSources: [], sourceIds: [], - streamsProjecting: [], activeLayers: [], millicastView: null, isMediaSet: true, @@ -38,42 +37,10 @@ const viewerReducer = (state = initialState, action) => { ...state, accountId: action.payload, }; - case 'viewer/setStreams': + case 'viewer/resetRemoteTrackSources': return { ...state, - streams: action.payload, - }; - case 'viewer/onTrackEvent': - const event = action.payload; - const mediaStream = event?.streams?.[0]; - const streamsArray = [...state.streams]; - if (mediaStream) { - const audioPromise = async () => { - await Promise.all( - streamsArray?.map((stream) => { - if (stream.stream.toURL() == event.streams?.[0]?.toURL()) { - stream.audioMid = event.transceiver.mid; - - if (Platform.OS === 'ios') { - const {AudioManager} = NativeModules; - AudioManager.routeAudioThroughDefaultSpeaker(); - } - } - }), - ); - }; - if (event.track.kind == 'audio') { - audioPromise(); - } else { - streamsArray.push({ - stream: mediaStream, - videoMid: event.transceiver.mid, - }); - } - } - return { - ...state, - streams: streamsArray, + remoteTrackSources: [], }; case 'viewer/setSourceIds': return { @@ -121,32 +88,16 @@ const viewerReducer = (state = initialState, action) => { ...state, sourceIds: [...sourceIds], }; - case 'viewer/addStream': - return { - ...state, - streams: [...state.streams, action.payload], - }; - case 'viewer/removeStream': - const streams = state.streams.filter((stream) => stream !== action.payload); - return { - ...state, - streams: [...streams], - }; - case 'viewer/addProjectingStream': - return { - ...state, - streamsProjecting: [...state.streamsProjecting, action.payload], - }; - case 'viewer/removeProjectingStream': - const projectingStreams = state.streamsProjecting.filter((stream) => stream !== action.payload); + case 'viewer/addRemoteTrackSource': return { ...state, - streamsProjecting: projectingStreams, + remoteTrackSources: [...state.remoteTrackSources, action.payload], }; - case 'viewer/removeProjectingStreams': + case 'viewer/removeRemoteTrackSource': + const remoteTrackSources = state.remoteTrackSources.filter((remoteTrackSource) => remoteTrackSource !== action.payload); return { ...state, - streamsProjecting: [], + remoteTrackSources: [...remoteTrackSources], }; case 'viewer/setSelectedSource': return { diff --git a/src/types/RemoteTrackSource.types.ts b/src/types/RemoteTrackSource.types.ts new file mode 100644 index 0000000..1af3582 --- /dev/null +++ b/src/types/RemoteTrackSource.types.ts @@ -0,0 +1,12 @@ +import { ViewProjectSourceMapping } from '@millicast/sdk'; + +export type StreamQuality = 'Auto' | 'High' | 'Medium' | 'Low'; + +export interface RemoteTrackSource { + audioMediaId?: string; + mediaStream: MediaStream; + projectMapping: ViewProjectSourceMapping[]; + quality?: StreamQuality; + sourceId?: string; + videoMediaId?: string; +}