Calls support

README.md

@@ -20,22 +20,24 @@
 
 A web app that takes chat exports from supported platforms and generates a single HTML file containing information, statistics and interactive graphs about them. Privacy is its main concern; chat data never leaves the device when generating reports.  
 
-|  💬 MESSAGES |  🅰️ LANGUAGE | 😃 EMOJI | 🔗 LINKS | 🌀 INTERACTION | 💙 SENTIMENT | 📅 TIMELINE |
-|--|--|--|--|--|--|--|
-| <img src="https://user-images.githubusercontent.com/5845105/222576038-ebcff785-1d5a-4402-ac16-5f55fe7a1a8f.png" alt="chat analytics messages tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576383-91ec15d7-0a3b-44eb-96bb-24de3886d23f.png" alt="chat analytics language tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576596-dfeb7660-808f-4b1f-905c-340282f1ed8d.png" alt="chat analytics emoji tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576676-9eac93b7-59d2-4ab6-95d4-d65bb0d32207.png" alt="chat analytics links tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576804-0d884987-6394-4435-97cd-06bbca84e391.png" alt="chat analytics interaction tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576869-f754d647-d915-4938-8acf-6c85f9315fee.png" alt="chat analytics sentiment tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576879-30461d12-2a3b-4814-a16c-b23eab263b6b.png" alt="chat analytics timeline tab" width="200"> |
+|  💬 MESSAGES |  🅰️ LANGUAGE | 😃 EMOJI | 🔗 LINKS | 📞 CALLS | 🌀 INTERACTION | 💙 SENTIMENT | 📅 TIMELINE |
+|--|--|--|--|--|--|--|--|
+| <img src="https://user-images.githubusercontent.com/5845105/222576038-ebcff785-1d5a-4402-ac16-5f55fe7a1a8f.png" alt="chat analytics messages tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576383-91ec15d7-0a3b-44eb-96bb-24de3886d23f.png" alt="chat analytics language tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576596-dfeb7660-808f-4b1f-905c-340282f1ed8d.png" alt="chat analytics emoji tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576676-9eac93b7-59d2-4ab6-95d4-d65bb0d32207.png" alt="chat analytics links tab" width="200"> | <img src="https://github.com/mlomb/chat-analytics/assets/5845105/644c41ee-767b-4554-9bf5-9c79e7c37bce" alt="chat analytics calls tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576804-0d884987-6394-4435-97cd-06bbca84e391.png" alt="chat analytics interaction tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576869-f754d647-d915-4938-8acf-6c85f9315fee.png" alt="chat analytics sentiment tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576879-30461d12-2a3b-4814-a16c-b23eab263b6b.png" alt="chat analytics timeline tab" width="200"> |
+
+
 
 You can interact with [the demo here](https://chatanalytics.app/demo)!
 
 ## Chat platform support
 
 You can generate reports from the following platforms:
 
-| Platform  | Formats supported                                                                | Text content | Edits & Replies  | Attachment Types                                                                    | Reactions        | Profile picture        | Mentions    |
-|-----------|----------------------------------------------------------------------------------|--------------|------------------|-------------------------------------------------------------------------------------|------------------|------------------------|-------------|
-| Discord   | `json` from [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) | ✅            | ✅                | ✅                                                                                   | ✅                | ✅ (until link expires) | ✅ (as text) |
-| Messenger | `json` from [Facebook DYI export](https://www.facebook.com/dyi)                  | ✅            | ❌                | ✅                                                                                   | ❌                | ❌                      | ✅ (as text) |
-| Telegram  | `json` from [Telegram Desktop](https://desktop.telegram.org/)                    | ✅            | ✅                | ✅                                                                                   | ❌ (not provided) | ❌                      | ✅ (as text) |
-| WhatsApp  | `txt` or `zip` exported from a phone                                             | ✅            | ❌ (not provided) | ✅<strong>*</strong> (if exported from iOS)<br>🟦 (generic if exported from Android) | ❌ (not provided) | ❌                      | ✅ (as text) |
+| Platform  | Formats supported                                                                | Text content | Edits & Replies  | Attachment Types                                                                    | Reactions        | Profile picture        | Mentions    | Calls |
+|-----------|----------------------------------------------------------------------------------|--------------|------------------|-------------------------------------------------------------------------------------|------------------|------------------------|-------------|-------|
+| Discord   | `json` from [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) | ✅            | ✅                | ✅                                                                                   | ✅                | ✅ (until link expires) | ✅ (as text) | ✅ |
+| Messenger | `json` from [Facebook DYI export](https://www.facebook.com/dyi)                  | ✅            | ❌                | ✅                                                                                   | ❌                | ❌                      | ✅ (as text) | ❌ |
+| Telegram  | `json` from [Telegram Desktop](https://desktop.telegram.org/)                    | ✅            | ✅                | ✅                                                                                   | ❌ (not provided) | ❌                      | ✅ (as text) | ✅ |
+| WhatsApp  | `txt` or `zip` exported from a phone                                             | ✅            | ❌ (not provided) | ✅<strong>*</strong> (if exported from iOS)<br>🟦 (generic if exported from Android) | ❌ (not provided) | ❌                      | ✅ (as text) | ❌ |
 
 <strong>*</strong> not all languages are supported, check [WhatsApp.ts](/pipeline/parse/parsers/WhatsApp.ts).
 

pipeline/Platforms.ts

@@ -10,6 +10,7 @@ interface PlatformInformation {
         reactions: boolean;
         replies: boolean;
         edits: boolean;
+        calls: boolean;
     };
 }
 
@@ -25,6 +26,7 @@ export const PlatformsInfo: {
             reactions: true,
             replies: true,
             edits: true,
+            calls: true,
         },
     },
     messenger: {
@@ -36,6 +38,7 @@ export const PlatformsInfo: {
             reactions: false,
             replies: false,
             edits: false,
+            calls: false,
         },
     },
     telegram: {
@@ -47,6 +50,7 @@ export const PlatformsInfo: {
             reactions: false,
             replies: true,
             edits: true,
+            calls: true,
         },
     },
     whatsapp: {
@@ -58,6 +62,7 @@ export const PlatformsInfo: {
             reactions: false,
             replies: false,
             edits: false,
+            calls: false,
         },
     },
 };

pipeline/Time.ts

@@ -273,3 +273,16 @@ export const formatDatetime = (format: TimeFormat, datetime?: Datetime) => {
 
     return formatTime(format, Day.fromKey(datetime.key), datetime.secondOfDay);
 };
+
+/** Finds the time difference in seconds between two Datetimes */
+export const diffDatetime = (a: Datetime, b: Datetime): number => {
+    // Probably this can be done more efficient and be reused (also see formatTime)
+
+    const aDate = Day.fromKey(a.key).toDate();
+    if (a.secondOfDay !== undefined) aDate.setSeconds(a.secondOfDay);
+
+    const bDate = Day.fromKey(b.key).toDate();
+    if (b.secondOfDay !== undefined) bDate.setSeconds(b.secondOfDay);
+
+    return (bDate.getTime() - aDate.getTime()) / 1000;
+};

pipeline/aggregate/Blocks.ts

@@ -1,5 +1,8 @@
 import { CommonBlockData } from "@pipeline/aggregate/Common";
 import { Filters } from "@pipeline/aggregate/Filters";
+import CallsActivity from "@pipeline/aggregate/blocks/calls/CallsActivity";
+import CallsPerPeriod from "@pipeline/aggregate/blocks/calls/CallsPerPeriod";
+import CallsStats from "@pipeline/aggregate/blocks/calls/CallsStats";
 import DomainsStats from "@pipeline/aggregate/blocks/domains/DomainsStats";
 import EmojiStats from "@pipeline/aggregate/blocks/emojis/EmojiStats";
 import ConversationStats from "@pipeline/aggregate/blocks/interaction/ConversationStats";
@@ -37,6 +40,9 @@ export type BlockDescription<K, Data, Args = undefined> = {
 /** All existing blocks must be defined here, so the UI can dynamically load them */
 export const Blocks = {
     [ActiveAuthors.key]: ActiveAuthors,
+    [CallsActivity.key]: CallsActivity,
+    [CallsPerPeriod.key]: CallsPerPeriod,
+    [CallsStats.key]: CallsStats,
     [ConversationsDuration.key]: ConversationsDuration,
     [ConversationStats.key]: ConversationStats,
     [DomainsStats.key]: DomainsStats,

pipeline/aggregate/Common.ts

@@ -47,9 +47,10 @@ export const computeCommonBlockData = (database: Database): CommonBlockData => {
 
 export interface VariableDistribution {
     total: number;
+    sum: number;
+    average: number;
     /** Aggregation in `count.length` buckets of `(whiskerMax-whiskerMin)/count.length` */
     count: number[];
-    /** Boxplot */
     boxplot: {
         min: number;
         whiskerMin: number;
@@ -65,6 +66,8 @@ export interface VariableDistribution {
 export const computeVariableDistribution = (values: Uint32Array, count: number): VariableDistribution => {
     const res: VariableDistribution = {
         total: count,
+        sum: 0,
+        average: 0,
         count: [],
         boxplot: {
             min: 0,
@@ -114,15 +117,18 @@ export const computeVariableDistribution = (values: Uint32Array, count: number):
     };
 
     for (let i = 0; i < count; i++) {
-        const time = values[i];
-        if (time >= lower && time < upper) {
+        const value = values[i];
+        if (value >= lower && value < upper) {
             // Order of operations is critical to avoid rounding issues
-            res.count[Math.floor((buckets / (upper - lower)) * (time - lower))]++;
+            res.count[Math.floor((buckets / (upper - lower)) * (value - lower))]++;
         } else {
             res.boxplot.outliers++;
         }
+        res.sum += value;
     }
 
+    res.average = res.sum / count;
+
     return res;
 };
 
@@ -137,3 +143,10 @@ export interface DateItem {
     ts: number; // timestamp
     v: number; // value
 }
+
+/** Activity entry for one hour of a weekday */
+export interface WeekdayHourEntry {
+    value: number;
+    hour: `${number}hs`;
+    weekday: "Sun" | "Mon" | "Tue" | "Wed" | "Thu" | "Fri" | "Sat";
+}

pipeline/aggregate/blocks/calls/CallsActivity.ts

@@ -0,0 +1,45 @@
+import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";
+import { WeekdayHourEntry } from "@pipeline/aggregate/Common";
+
+import { iterateHoursInCall } from "./CallsUtils";
+
+export interface CallsActivity {
+    /** Each entry contains the total amount of seconds spent in calls for that hour of the week */
+    weekdayHourActivity: WeekdayHourEntry[];
+}
+
+const fn: BlockFn<CallsActivity> = (database, filters, common) => {
+    const weekdayHourDurations: number[] = new Array(7 * 24).fill(0);
+
+    for (const call of database.calls) {
+        // Note: time filtered below, we have to filter each hour individually
+        if (!filters.hasChannel(call.channelIndex)) continue;
+        if (!filters.hasAuthor(call.authorIndex)) continue;
+
+        iterateHoursInCall(call, (dayIndex, hourInDay, secondsInCall) => {
+            if (filters.inTime(dayIndex)) {
+                weekdayHourDurations[common.dayOfWeek[dayIndex] * 24 + hourInDay] += secondsInCall;
+            }
+        });
+    }
+
+    const weekdayHourActivity: WeekdayHourEntry[] = weekdayHourDurations.map((count, i) => {
+        const weekday = Math.floor(i / 24);
+        const hour = i % 24;
+        return {
+            value: count,
+            hour: `${hour}hs`,
+            weekday: (["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] as const)[weekday],
+        };
+    });
+
+    return {
+        weekdayHourActivity,
+    };
+};
+
+export default {
+    key: "calls/activty",
+    triggers: ["time", "authors", "channels"],
+    fn,
+} as BlockDescription<"calls/activty", CallsActivity>;

pipeline/aggregate/blocks/calls/CallsPerPeriod.ts

@@ -0,0 +1,79 @@
+import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";
+
+import { iterateHoursInCall } from "./CallsUtils";
+
+export type CallsInDate = {
+    ts: number; // timestamp
+
+    n: number; // number of calls
+    t: number; // total time in calls (seconds)
+};
+
+/**
+ * Number of calls per different time cycles.
+ * It ignores the time filter completely, all cycles are included.
+ */
+export interface CallsPerPeriod {
+    perDay: CallsInDate[];
+    perWeek: CallsInDate[];
+    perMonth: CallsInDate[];
+}
+
+const fn: BlockFn<CallsPerPeriod> = (database, filters, common) => {
+    const res: CallsPerPeriod = {
+        perDay: [],
+        perWeek: [],
+        perMonth: [],
+    };
+
+    const { keyToTimestamp } = common;
+    const { dateToWeekIndex, dateToMonthIndex } = common.timeKeys;
+
+    // fill empty
+    for (const ts of keyToTimestamp.date) {
+        res.perDay.push({
+            ts,
+            n: 0,
+            t: 0,
+        });
+    }
+    for (const ts of keyToTimestamp.week) {
+        res.perWeek.push({
+            ts,
+            n: 0,
+            t: 0,
+        });
+    }
+    for (const ts of keyToTimestamp.month) {
+        res.perMonth.push({
+            ts,
+            n: 0,
+            t: 0,
+        });
+    }
+
+    for (const call of database.calls) {
+        // check filters
+        // if (!filters.inTime(call.start.dayIndex)) continue; // don't filter by time, UI scrolls the time natively
+        if (!filters.hasChannel(call.channelIndex)) continue;
+        if (!filters.hasAuthor(call.authorIndex)) continue;
+
+        iterateHoursInCall(call, (dayIndex, _, secondsInCall) => {
+            res.perDay[dayIndex].t += secondsInCall;
+            res.perWeek[dateToWeekIndex[dayIndex]].t += secondsInCall;
+            res.perMonth[dateToMonthIndex[dayIndex]].t += secondsInCall;
+        });
+
+        res.perDay[call.start.dayIndex].n += 1;
+        res.perWeek[dateToWeekIndex[call.start.dayIndex]].n += 1;
+        res.perMonth[dateToMonthIndex[call.start.dayIndex]].n += 1;
+    }
+
+    return res;
+};
+
+export default {
+    key: "calls/per-period",
+    triggers: ["authors", "channels"],
+    fn,
+} as BlockDescription<"calls/per-period", CallsPerPeriod>;

pipeline/aggregate/blocks/calls/CallsStats.ts

@@ -0,0 +1,95 @@
+import { Datetime, diffDatetime } from "@pipeline/Time";
+import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";
+import { VariableDistribution, computeVariableDistribution } from "@pipeline/aggregate/Common";
+import { filterMessages } from "@pipeline/aggregate/Helpers";
+import { MessageView } from "@pipeline/serialization/MessageView";
+
+interface CallDuration {
+    duration: number;
+    start: Datetime;
+}
+
+export interface CallsStats {
+    /** Total number of calls */
+    total: number;
+    /** Total number of seconds spent in calls */
+    secondsInCall: number;
+
+    longestCall?: CallDuration;
+
+    /** Call duration distribution in seconds */
+    durationDistribution: VariableDistribution;
+    /** Time between calls distribution in seconds */
+    timesBetweenDistribution: VariableDistribution;
+
+    /** Number of calls made by each author */
+    authorsCount: number[];
+}
+
+const fn: BlockFn<CallsStats> = (database, filters, common, args) => {
+    const { dateKeys } = common.timeKeys;
+
+    let total = 0;
+    let secondsInCall = 0;
+    let longestCall: CallDuration | undefined = undefined;
+    let lastCall: Datetime | undefined = undefined;
+
+    const authorsCount = new Array(database.authors.length).fill(0);
+
+    const durations = new Uint32Array(database.calls.length).fill(0xfffffff0);
+    const timesBetween = new Uint32Array(database.calls.length).fill(0xfffffff0);
+
+    for (const call of database.calls) {
+        if (!filters.inTime(call.start.dayIndex)) continue;
+        if (!filters.hasChannel(call.channelIndex)) continue;
+        if (!filters.hasAuthor(call.authorIndex)) continue;
+
+        const startDatetime = {
+            key: dateKeys[call.start.dayIndex],
+            secondOfDay: call.start.secondOfDay,
+        };
+        const endDatetime = {
+            key: dateKeys[call.end.dayIndex],
+            secondOfDay: call.end.secondOfDay,
+        };
+
+        durations[total] = call.duration;
+        authorsCount[call.authorIndex]++;
+
+        if (longestCall === undefined || call.duration > longestCall.duration) {
+            longestCall = {
+                duration: call.duration,
+                start: startDatetime,
+            };
+        }
+
+        if (lastCall !== undefined) {
+            // compute time difference between calls
+            const diff = diffDatetime(lastCall, startDatetime);
+            if (diff < 0) throw new Error("Time difference between calls is negative, diff=" + diff);
+            timesBetween[total - 1] = diff;
+        }
+        lastCall = endDatetime;
+
+        secondsInCall += call.duration;
+        total++;
+    }
+
+    return {
+        total,
+        secondsInCall,
+        averageDuration: secondsInCall / total,
+        longestCall,
+
+        durationDistribution: computeVariableDistribution(durations, total),
+        timesBetweenDistribution: computeVariableDistribution(timesBetween, total - 1),
+
+        authorsCount,
+    };
+};
+
+export default {
+    key: "calls/stats",
+    triggers: ["authors", "channels", "time"],
+    fn,
+} as BlockDescription<"calls/stats", CallsStats>;