feat(functions-load-balancing): add functions to control game load ba…

…lance

packages/functions/package.json

@@ -15,11 +15,17 @@
   "main": "lib/index.js",
   "dependencies": {
     "firebase-admin": "^9.2.0",
-    "firebase-functions": "^3.11.0"
+    "firebase-functions": "^3.11.0",
+    "axios": "^0.21.1",
+    "@pipeline/common": "^0.1.0",
+    "express": "^4.17.1",
+    "lodash": "^4.17.20"
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^3.9.1",
     "@typescript-eslint/parser": "^3.8.0",
+    "@types/axios": "^0.14.0",
+    "@types/lodash": "^4.14.168",
     "eslint": "^7.6.0",
     "eslint-plugin-import": "^2.22.0",
     "firebase-functions-test": "^0.2.0",

packages/functions/src/load-balancing/moveGameFromRTDBToFirestore.ts

@@ -0,0 +1,19 @@
+import * as admin from "firebase-admin";
+import {FirebaseCollection, Game, RTDBPaths} from "@pipeline/common";
+
+/**
+ * It moves a game from a particular RTDB instance to Firestore.
+ * Before writing to Firestore, the 'rtdbInstance' field is set to null
+ *
+ * @param gameId
+ * @param db
+ * @param rtdb
+ */
+const moveGameFromRTDBToFirestore = async (gameId: string, db: FirebaseFirestore.Firestore, rtdb: admin.database.Database) => {
+  const gameSnap = await rtdb.ref(`/${RTDBPaths.Games}/${gameId}`).get();
+  const game = gameSnap.val() as Game;
+  game.rtdbInstance = null;
+  await db.collection(FirebaseCollection.Games).doc(gameId).update({...game});
+}
+
+export {moveGameFromRTDBToFirestore};

packages/functions/src/load-balancing/onOnlineGameStatusUpdate.ts

@@ -0,0 +1,66 @@
+import * as functions from 'firebase-functions';
+import * as admin from "firebase-admin";
+import {FirebaseCollection, RTDBInstance, RTDBPaths, Status} from "@pipeline/common";
+import {PROJECT_ID} from "../utils/rtdb";
+import FieldValue = admin.firestore.FieldValue;
+import {end, RTDB_LOCATION} from "../utils/general";
+import {moveGameFromRTDBToFirestore} from "./moveGameFromRTDBToFirestore";
+
+const db = admin.firestore();
+const logger = functions.logger;
+
+const INSTANCE_ID = `${PROJECT_ID}-default-rtdb`
+
+/**
+ * It triggers when the path /statuses/{userId} of that RTDB instance is updated.
+ *
+ * If the status state is different from the previous one, then:
+ *  - if the new one is 'online', the correct document of Firestore, representing the RTDB instance, should be updated incrementing by +1
+ *  - if the new one is 'offline', the correct document of Firestore, representing the RTDB instance, should be updated incrementing by -1
+ *
+ *  Next, if the new status is 'offline', an RTDB query is performed to look for the online users for that game.
+ *  If more than one are found (because one is the one we were updating), this means there is still someone in the game
+ *  Otherwise, we can move the game from RTDB back to Firestore
+ */
+
+export const onOnlineGameStatusUpdate = functions.database.instance(INSTANCE_ID).ref(`/${RTDBPaths.Statuses}/{userId}`)
+  .onUpdate(async (snapshot, context) => {
+
+    const instanceId = INSTANCE_ID;
+    const userId = context.params.userId;
+
+    const originalStatus = snapshot.before.val() as Status;
+    const nextStatus = snapshot.after.val() as Status;
+
+
+    if (originalStatus.state !== nextStatus.state) {
+
+        const gameId = nextStatus.gameId;
+
+        logger.log(`User ${userId} from game ${gameId} going from ${originalStatus.state} to ${nextStatus.state}`);
+
+        await db.collection(FirebaseCollection.RTDBInstances).doc(instanceId)
+          .update({onlineOnGameCount: nextStatus.state === 'online' ?
+                FieldValue.increment(1) as any :
+                FieldValue.increment(-1) as any,
+          } as Partial<RTDBInstance>);
+        if (nextStatus.state === 'offline') {
+            const rtdb = admin.app().database(`https://${INSTANCE_ID}.${RTDB_LOCATION}.firebasedatabase.app`);
+            const snap = await rtdb.ref(`/${RTDBPaths.Statuses}`).orderByChild('gameId').equalTo(gameId).get();
+            const statuses: Status[] = [];
+            snap.forEach(s => {
+                statuses.push(s.val())
+            });
+            const onlineCount = statuses.filter((s: Status) => s.state === 'online').length;
+            logger.log(`Online user for game ${gameId}: ${onlineCount}`);
+            if (onlineCount <= 1) {
+                await moveGameFromRTDBToFirestore(gameId, db, rtdb);
+                logger.log(`Game ${gameId} moved from RTDB to Firestore`);
+            }
+            return end();
+        }
+        return end();
+
+    }
+    return end();
+  });

packages/functions/src/load-balancing/selectBestRTDBInstance.ts

@@ -0,0 +1,117 @@
+import * as functions from 'firebase-functions';
+import * as admin from "firebase-admin";
+import axios from 'axios';
+import FieldValue = admin.firestore.FieldValue;
+import {checkAuth} from "../utils/auth";
+import {runTransactionWithRetry} from "../utils/db";
+import {FirebaseCollection, RTDBInstance, Game, RTDBPaths} from '@pipeline/common';
+import {getRTDBInstanceName, PROJECT_ID} from "../utils/rtdb";
+import {end, RTDB_LOCATION} from "../utils/general";
+const db = admin.firestore();
+const logger = functions.logger;
+
+const getNextRTDBInstanceNum = async (): Promise<number> => {
+  const res = await axios.get(`https://firebasedatabase.googleapis.com/v1beta/projects/${PROJECT_ID}/locations/-/instances?pageSize=100`, {
+    headers: {
+      'Authorization': `Bearer `,
+    },
+  })
+  const instances = res.data.instances as any[];
+  return instances.length;
+}
+
+const createRTDBInstance = async (databaseId: string) => {
+  await axios.get(`https://firebasedatabase.googleapis.com/v1beta/projects/${PROJECT_ID}/locations/${RTDB_LOCATION}/instances?database_id=${databaseId}`, {
+    headers: {
+      'Authorization': `Bearer `,
+    },
+  })
+}
+
+const createNewRTDBInstance = async () => {
+  const nextNum = await getNextRTDBInstanceNum();
+  const newRTDBInstanceName = getRTDBInstanceName(nextNum);
+  await createRTDBInstance(newRTDBInstanceName);
+  await db.collection(FirebaseCollection.RTDBInstances).doc(newRTDBInstanceName).set({
+    createdAt: FieldValue.serverTimestamp(),
+    onlineOnGameCount: 0,
+  } as RTDBInstance)
+  admin.app().database(`https://secondary_db_url.firebaseio.com`)
+
+  return newRTDBInstanceName;
+}
+
+
+export const selectBestRTDBInstance = functions.region(
+  'europe-west1'
+).https.onRequest(async (req, res) => {
+
+  logger.log('selectBestRTDBInstance API triggered');
+
+  try {
+
+    if (req.method !== "GET") {
+      res.status(405).send("Method not allowed");
+      return end();
+    }
+
+    const gameId = req.query.gameId as string;
+    if (!gameId) {
+      res.status(400).send("Missing required parameter");
+      return end();
+    }
+
+    const decodedToken = await checkAuth(req, res);
+
+    if (!decodedToken) {
+      logger.log('User is not authenticated');
+      res.status(403).send();
+      return end();
+    }
+
+
+    logger.log('User uid', decodedToken.uid);
+
+    const bestRTDBInstanceQuery = await db.collection(FirebaseCollection.RTDBInstances)
+      .orderBy('onlineOnGameCount', "asc").limit(1).get();
+    const bestRTDBInstanceDoc = bestRTDBInstanceQuery.docs[0];
+    const bestRTDBInstance = bestRTDBInstanceDoc.data();
+    const bestRTDBInstanceId = bestRTDBInstanceDoc.id;
+
+    logger.log(`Selected instance ${bestRTDBInstanceId} with ${bestRTDBInstance.onlineOnGameCount} online on game users`);
+
+    /*
+    if (bestRTDBInstance.onlineOnGameCount >= RTDB_THRESHOLD) {
+      //create new instance
+      axi
+      admin.database()
+    }
+    */
+
+    const rtdb = admin.app().database(`https://${bestRTDBInstanceId}.${RTDB_LOCATION}.firebasedatabase.app`);
+
+    const gameRef = db.collection(FirebaseCollection.Games).doc(gameId);
+
+    await runTransactionWithRetry(db, async transaction => {
+      const gameDoc = await transaction.get(gameRef);
+      const game = gameDoc.data() as Game;
+      if (!game.rtdbInstance) {
+        transaction.update(gameRef, {
+          rtdbInstance: bestRTDBInstanceId,
+        } as Partial<Game>);
+        await rtdb.ref(`/${RTDBPaths.Games}/${gameId}`).set({
+          ...game,
+        })
+      }
+    });
+
+    res.status(200).send({bestRTDBInstanceId});
+
+  } catch (e) {
+    logger.error(e);
+    res.status(500).send("Unknown error");
+  }
+
+  return end();
+
+});

packages/functions/src/load-balancing/syncStatusJob.ts

@@ -0,0 +1,73 @@
+import * as functions from 'firebase-functions';
+import * as admin from "firebase-admin";
+import {FirebaseCollection, RTDBInstance, RTDBPaths, Status} from "@pipeline/common";
+import {PROJECT_ID} from "../utils/rtdb";
+import Timestamp = admin.firestore.Timestamp;
+import {end, RTDB_LOCATION} from "../utils/general";
+import {moveGameFromRTDBToFirestore} from "./moveGameFromRTDBToFirestore";
+import * as _ from 'lodash';
+
+const db = admin.firestore();
+const logger = functions.logger;
+
+/**
+ * It's a job scheduled every 30 mins. It's needed to synchronize disconnected users.
+ * First of all, all RTDB instances are retrieved from the proper Firestore document.
+ * Then, for each of these RTDB instances, a query is performed, looking for online status.
+ * At this point, for each users found as online, we check if the last time they updated their status
+ * is too old. Old is defined as 25s before now.
+ * These are grouped by their gameId.
+ *
+ * At this point, for each game id, it's checked if the game itself have to be moved. In order to do this,
+ * an RTDB query and then a filter are performed to obtain the online users for that game.
+ *
+ * Now, if these online users are less or equal than the disconnected ones, game is moved.
+ *
+ * After it, every disconnected user is updated to reflect the real offline status.
+ *
+ * NB: This job will trigger the onUpdate RTDB triggers, and because of this one could think that
+ * the game move of this job is useless. But, because of the async nature of the RTDB trigger,
+ * we prefer to keep the game move logic also here
+ *
+ */
+
+const syncStatusJob = async () => {
+  logger.log('syncStatusJob triggered');
+  const rtdbInstancesQuery = await db.collection(FirebaseCollection.RTDBInstances).get();
+  for (const rtdbDoc of rtdbInstancesQuery.docs) {
+    const rtdb = admin.app().database(`https://${rtdbDoc.id}.${RTDB_LOCATION}.firebasedatabase.app`);
+    const snap = await rtdb.ref(`/${RTDBPaths.Statuses}`).orderByChild('state').equalTo('online').get();
+    const disconnectedStatusesIds: string[] = [];
+    const nowMillis = Date.now();
+    snap.forEach(s => {
+      const status = s.val() as Status;
+      // if an user is online but he is not updating it's status since 25s -> disconnected
+      if (nowMillis - 25000 >= (status.updatedAt as Timestamp).toMillis()) {
+        disconnectedStatusesIds.push(s.key as string);
+      }
+    });
+    const disconnectedStatusesIdsByGameId = _.groupBy(disconnectedStatusesIds, 'gameId');
+    for (const gameIdKey of Object.keys(disconnectedStatusesIdsByGameId)) {
+
+      const disconnectedsByGameId = disconnectedStatusesIdsByGameId[gameIdKey];
+      const statusSnap = await rtdb.ref(`/${RTDBPaths.Statuses}`).orderByChild('gameId').equalTo(gameIdKey).get();
+      const statuses: Status[] = [];
+      statusSnap.forEach(s => {
+        statuses.push(s.val())
+      });
+      const onlineCount = statuses.filter((s: Status) => s.state === 'online').length;
+      if (onlineCount <= disconnectedsByGameId.length) {
+        // game must be moved
+        await moveGameFromRTDBToFirestore(gameIdKey, db, rtdb);
+      }
+      for (const disconnectedIdByGameId of disconnectedsByGameId) {
+        // status must be updated
+        await rtdb.ref(`/${RTDBPaths.Statuses}/${disconnectedIdByGameId}`).update({
+          state: 'offline',
+          updatedAt: admin.database.ServerValue.TIMESTAMP,
+          gameId: null,
+        })
+      }
+    }
+  }
+};

packages/functions/src/utils/auth.ts

@@ -0,0 +1,19 @@
+import * as admin from 'firebase-admin';
+import * as functions from 'firebase-functions';
+import * as express from 'express';
+
+const checkAuth = async (req: functions.https.Request, res: express.Response): Promise<admin.auth.DecodedIdToken | null> => {
+  if (!req.headers['authorization'] || !req.headers['authorization'].startsWith('Bearer ')) {
+    res.status(403).send();
+    return null;
+  }
+  const idToken = req.headers.authorization.split('Bearer ')[1];
+  try {
+    return await admin.auth().verifyIdToken(idToken)
+  } catch(e) {
+    res.status(403).send();
+    return null;
+  }
+};
+
+export {checkAuth}