feat(flat-web): add rtc & rtm

web/flat-web/package.json

@@ -31,18 +31,36 @@
   "dependencies": {
     "@ant-design/icons": "^4.6.2",
     "@loadable/component": "^5.15.0",
+    "@netless/combine-player": "^1.1.5",
+    "@netless/cursor-tool": "^0.1.0",
+    "@netless/white-audio-plugin": "^2.0.3",
+    "@netless/white-video-plugin": "^2.0.3",
     "agora-rtc-sdk-ng": "^4.5.0",
+    "agora-rtm-sdk": "^1.4.3",
     "antd": "^4.15.4",
+    "axios": "^0.21.1",
+    "classnames": "^2.3.1",
+    "date-fns": "^2.22.1",
     "eventemitter3": "^4.0.7",
+    "i18next": "^20.3.1",
+    "i18next-browser-languagedetector": "^6.1.1",
+    "mobx": "^6.1.0",
     "mobx-react-lite": "^3.2.0",
+    "polly-js": "^1.8.2",
+    "rc-picker": "^2.5.10",
     "react": "^17.0.2",
+    "react-device-detect": "^1.17.0",
     "react-dom": "^17.0.2",
+    "react-i18next": "^11.10.0",
+    "react-router": "^5.2.0",
+    "react-router-dom": "^5.2.0",
     "react-router-last-location": "^2.0.1",
     "react-use": "^17.2.4",
+    "uuid": "^8.3.2",
     "white-web-sdk": "^2.12.15"
   },
   "scripts": {
-    "postinstall": "esbuild-dev ./scripts/white-web-sdk.ts",
+    "postinstall": "esbuild-dev ./scripts/post-install.ts",
     "start": "vite --open",
     "build": "vite build",
     "serve": "vite preview",

web/flat-web/scripts/agora-rtm-sdk.ts

@@ -0,0 +1,17 @@
+/**
+ * Get rid of "process.env.NODE_ENV" replace error in agora-rtm-sdk.
+ *
+ * @TODO: Remove this file when agora-rtm-sdk fix the code.
+ */
+
+/// <reference types="node" />
+
+// https://vitejs.dev/guide/env-and-mode.html#production-replacement
+
+import fs from "fs";
+// NOTE: `import.meta.resolve` is still experimental
+const file = require.resolve("agora-rtm-sdk");
+const code = fs.readFileSync(file, "utf-8");
+const modified = code.replace("process.env.NODE_ENV", "process\u200b.env.NODE_ENV");
+fs.writeFileSync(file, modified);
+console.log("agora-rtm-sdk: done!");

web/flat-web/scripts/post-install.ts

@@ -0,0 +1,2 @@
+import "./white-web-sdk";
+import "./agora-rtm-sdk";

web/flat-web/scripts/white-web-sdk.ts

@@ -47,7 +47,7 @@ function hackAndReplaceMainScript(script: string, main: string): void {
     // webpack will add polyfill under the hood so it is ok, but not rollup/esbuild
     script = script.replace(/=require\([^)]+\)/g, "=void 0");
     fs.writeFileSync(path.resolve(sdkPath, main), script);
-    console.log("hack: done!");
+    console.log("white-web-sdk: done!");
 }
 
 if (fs.existsSync(pkgJSON)) {

web/flat-web/src/apiMiddleware/CloudRecording.ts

@@ -76,7 +76,7 @@ export class CloudRecording {
                     height: 360,
                     fps: 15,
                     bitrate: 500,
-                    defaultUserBackgroundImage: process.env.CLOUD_RECORDING_DEFAULT_AVATAR,
+                    defaultUserBackgroundImage: import.meta.env.CLOUD_RECORDING_DEFAULT_AVATAR,
                 },
             },
         },

web/flat-web/src/apiMiddleware/Rtm.ts

@@ -2,7 +2,7 @@ import AgoraRTM, { RtmChannel, RtmClient } from "agora-rtm-sdk";
 import polly from "polly-js";
 import { v4 as uuidv4 } from "uuid";
 import { AGORA, NODE_ENV } from "../constants/Process";
-import { EventEmitter } from "events";
+import { EventEmitter } from "eventemitter3";
 import { RoomStatus } from "./flatServer/constants";
 import { generateRTMToken } from "./flatServer/agora";
 import { globalStore } from "../stores/GlobalStore";
@@ -145,7 +145,7 @@ export declare interface Rtm {
 }
 
 // eslint-disable-next-line no-redeclare
-export class Rtm extends EventEmitter {
+export class Rtm extends EventEmitter<keyof RTMEvents> {
     public static MessageType = AgoraRTM.MessageType;
 
     public client: RtmClient;

web/flat-web/src/apiMiddleware/rtc/avatar.ts

@@ -0,0 +1,154 @@
+import type {
+    IAgoraRTCClient,
+    IAgoraRTCRemoteUser,
+    ICameraVideoTrack,
+    IMicrophoneAudioTrack,
+    IRemoteAudioTrack,
+    IRemoteVideoTrack,
+    ITrack,
+} from "agora-rtc-sdk-ng";
+import AgoraRTC from "agora-rtc-sdk-ng";
+import type { User } from "../../stores/UserStore";
+import type { RtcRoom } from "./room";
+
+export interface RtcAvatarParams {
+    rtc: RtcRoom;
+    userUUID: string;
+    avatarUser: User;
+}
+
+/**
+ * @example
+ * const avatar = new RtcAvatar({ rtc, userUUID, avatarUser })
+ * avatar.element = el
+ * avatar.setCamera(true)
+ */
+export class RtcAvatar {
+    private readonly rtc: RtcRoom;
+
+    public readonly userUUID: string;
+    public readonly avatarUser: User;
+    public element?: HTMLElement;
+    public audioTrack?: ITrack;
+    public videoTrack?: ITrack;
+
+    private readonly isLocal: boolean;
+    private readonly remoteAudioTrack: Promise<IRemoteAudioTrack>;
+    private readonly remoteVideoTrack: Promise<IRemoteVideoTrack>;
+
+    private resolveRemoteAudioTrack?: (value: IRemoteAudioTrack) => void;
+    private resolveRemoteVideoTrack?: (value: IRemoteVideoTrack) => void;
+
+    constructor({ rtc, userUUID, avatarUser }: RtcAvatarParams) {
+        this.rtc = rtc;
+        this.userUUID = userUUID;
+        this.avatarUser = avatarUser;
+        this.isLocal = userUUID === avatarUser.userUUID;
+        this.remoteAudioTrack = new Promise<IRemoteAudioTrack>(resolve => {
+            this.resolveRemoteAudioTrack = resolve;
+        });
+        this.remoteVideoTrack = new Promise<IRemoteVideoTrack>(resolve => {
+            this.resolveRemoteVideoTrack = resolve;
+        });
+        if (!this.isLocal) {
+            this.setupExistingTracks();
+            this.client.on("user-published", this.onUserPublished);
+        }
+    }
+
+    private get client(): IAgoraRTCClient {
+        return this.rtc.client!;
+    }
+
+    private async setupExistingTracks(): Promise<void> {
+        const exist = this.client.remoteUsers.find(e => e.uid === this.avatarUser.rtcUID);
+        if (exist) {
+            if (exist.hasAudio) {
+                const audioTrack = await this.client.subscribe(exist, "audio");
+                this.resolveRemoteAudioTrack?.(audioTrack);
+                this.resolveRemoteAudioTrack = undefined;
+            }
+            if (exist.hasVideo) {
+                const videoTrack = await this.client.subscribe(exist, "video");
+                this.resolveRemoteVideoTrack?.(videoTrack);
+                this.resolveRemoteVideoTrack = undefined;
+            }
+        }
+    }
+
+    public destroy(): void {
+        if (!this.isLocal && this.client) {
+            this.client.off("user-published", this.onUserPublished);
+        }
+        this.resolveRemoteAudioTrack = undefined;
+        this.resolveRemoteVideoTrack = undefined;
+    }
+
+    private onUserPublished = async (
+        user: IAgoraRTCRemoteUser,
+        mediaType: "video" | "audio",
+    ): Promise<void> => {
+        if (user.uid === this.avatarUser.rtcUID) {
+            const track = await this.client.subscribe(user, mediaType);
+            if (mediaType === "audio") {
+                this.resolveRemoteAudioTrack?.(track as IRemoteAudioTrack);
+                this.resolveRemoteAudioTrack = undefined;
+            } else {
+                this.resolveRemoteVideoTrack?.(track as IRemoteVideoTrack);
+                this.resolveRemoteVideoTrack = undefined;
+            }
+        }
+    };
+
+    public async setCamera(enable: boolean): Promise<void> {
+        try {
+            if (this.isLocal) {
+                const videoTrack = this.videoTrack as ICameraVideoTrack | undefined;
+                if (videoTrack) {
+                    videoTrack.setEnabled(enable);
+                } else if (enable) {
+                    const videoTrack = await AgoraRTC.createCameraVideoTrack({
+                        encoderConfig: { width: 288, height: 216 },
+                    });
+                    this.videoTrack = videoTrack;
+                    this.element && videoTrack.play(this.element);
+                    await this.client.publish([videoTrack]);
+                }
+            } else {
+                if (!this.videoTrack && enable) {
+                    const videoTrack = await this.remoteVideoTrack;
+                    this.videoTrack = videoTrack;
+                    this.element && videoTrack.play(this.element);
+                }
+            }
+        } catch (error) {
+            console.info("setCamera failed", error);
+        }
+    }
+
+    public async setMic(enable: boolean): Promise<void> {
+        try {
+            if (this.isLocal) {
+                const audioTrack = this.audioTrack as IMicrophoneAudioTrack | undefined;
+                if (audioTrack) {
+                    audioTrack.setEnabled(enable);
+                } else if (enable) {
+                    const audioTrack = await AgoraRTC.createMicrophoneAudioTrack();
+                    this.audioTrack = audioTrack;
+                    audioTrack.play();
+                    await this.client.publish(audioTrack);
+                }
+            } else {
+                if (!this.audioTrack && enable) {
+                    const audioTrack = await this.remoteAudioTrack;
+                    this.audioTrack = audioTrack;
+                    audioTrack.play();
+                }
+            }
+        } catch (error) {
+            console.info("setMic failed", error);
+        }
+    }
+}
+
+(window as any).RtcAvatar = RtcAvatar;

web/flat-web/src/apiMiddleware/rtc/room.ts

@@ -0,0 +1,75 @@
+import AgoraRTC, { IAgoraRTCClient } from "agora-rtc-sdk-ng";
+import { AGORA } from "../../constants/Process";
+import { globalStore } from "../../stores/GlobalStore";
+import { generateRTCToken } from "../flatServer/agora";
+
+AgoraRTC.setLogLevel(/* WARNING */ 2);
+
+export enum RtcChannelType {
+    Communication = 0,
+    Broadcast = 1,
+}
+
+/**
+ * Flow:
+ * ```
+ * join() -> now it has `client`
+ *   getLatency() -> number
+ * destroy()
+ * ```
+ */
+export class RtcRoom {
+    public client?: IAgoraRTCClient;
+
+    private roomUUID?: string;
+
+    public async join({
+        roomUUID,
+        isCreator,
+        rtcUID,
+        channelType,
+    }: {
+        roomUUID: string;
+        isCreator: boolean;
+        rtcUID: number;
+        channelType: RtcChannelType;
+    }): Promise<void> {
+        if (this.client) {
+            await this.destroy();
+        }
+
+        const mode = channelType === RtcChannelType.Communication ? "rtc" : "live";
+        this.client = AgoraRTC.createClient({ mode, codec: "vp8" });
+
+        this.client.on("token-privilege-will-expire", this.renewToken);
+
+        await this.client.setClientRole(
+            channelType === RtcChannelType.Broadcast && !isCreator ? "audience" : "host",
+        );
+        const token = globalStore.rtcToken || (await generateRTCToken(roomUUID));
+        await this.client.join(AGORA.APP_ID, roomUUID, token, rtcUID);
+
+        this.roomUUID = roomUUID;
+    }
+
+    public getLatency(): number {
+        return this.client?.getRTCStats().RTT ?? NaN;
+    }
+
+    public async destroy(): Promise<void> {
+        if (this.client) {
+            this.client.off("token-privilege-will-expire", this.renewToken);
+            await this.client.leave();
+            this.client = undefined;
+        }
+    }
+
+    private renewToken = async (): Promise<void> => {
+        if (this.client && this.roomUUID) {
+            const token = await generateRTCToken(this.roomUUID);
+            await this.client.renewToken(token);
+        }
+    };
+}
+
+(window as any).RtcRoom = RtcRoom;

web/flat-web/src/constants/Process.ts

@@ -1,4 +1,4 @@
-export const NODE_ENV = process.env.NODE_ENV;
+export const NODE_ENV = import.meta.env.MODE;
 
 export const NETLESS = Object.freeze({
     APP_IDENTIFIER: import.meta.env.NETLESS_APP_IDENTIFIER,