Skip to content

Commit

Permalink
Merge pull request #88 from mlomb/calls-support
Browse files Browse the repository at this point in the history
Merge calls support
  • Loading branch information
mlomb authored Aug 17, 2023
2 parents 65b524d + eafb341 commit 6378e92
Show file tree
Hide file tree
Showing 46 changed files with 1,085 additions and 199 deletions.
20 changes: 11 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,24 @@

A web app that takes chat exports from supported platforms and generates a single HTML file containing information, statistics and interactive graphs about them. Privacy is its main concern; chat data never leaves the device when generating reports.

| 💬 MESSAGES | 🅰️ LANGUAGE | 😃 EMOJI | 🔗 LINKS | 🌀 INTERACTION | 💙 SENTIMENT | 📅 TIMELINE |
|--|--|--|--|--|--|--|
| <img src="https://user-images.githubusercontent.com/5845105/222576038-ebcff785-1d5a-4402-ac16-5f55fe7a1a8f.png" alt="chat analytics messages tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576383-91ec15d7-0a3b-44eb-96bb-24de3886d23f.png" alt="chat analytics language tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576596-dfeb7660-808f-4b1f-905c-340282f1ed8d.png" alt="chat analytics emoji tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576676-9eac93b7-59d2-4ab6-95d4-d65bb0d32207.png" alt="chat analytics links tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576804-0d884987-6394-4435-97cd-06bbca84e391.png" alt="chat analytics interaction tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576869-f754d647-d915-4938-8acf-6c85f9315fee.png" alt="chat analytics sentiment tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576879-30461d12-2a3b-4814-a16c-b23eab263b6b.png" alt="chat analytics timeline tab" width="200"> |
| 💬 MESSAGES | 🅰️ LANGUAGE | 😃 EMOJI | 🔗 LINKS | 📞 CALLS | 🌀 INTERACTION | 💙 SENTIMENT | 📅 TIMELINE |
|--|--|--|--|--|--|--|--|
| <img src="https://user-images.githubusercontent.com/5845105/222576038-ebcff785-1d5a-4402-ac16-5f55fe7a1a8f.png" alt="chat analytics messages tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576383-91ec15d7-0a3b-44eb-96bb-24de3886d23f.png" alt="chat analytics language tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576596-dfeb7660-808f-4b1f-905c-340282f1ed8d.png" alt="chat analytics emoji tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576676-9eac93b7-59d2-4ab6-95d4-d65bb0d32207.png" alt="chat analytics links tab" width="200"> | <img src="https://github.com/mlomb/chat-analytics/assets/5845105/644c41ee-767b-4554-9bf5-9c79e7c37bce" alt="chat analytics calls tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576804-0d884987-6394-4435-97cd-06bbca84e391.png" alt="chat analytics interaction tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576869-f754d647-d915-4938-8acf-6c85f9315fee.png" alt="chat analytics sentiment tab" width="200"> | <img src="https://user-images.githubusercontent.com/5845105/222576879-30461d12-2a3b-4814-a16c-b23eab263b6b.png" alt="chat analytics timeline tab" width="200"> |



You can interact with [the demo here](https://chatanalytics.app/demo)!

## Chat platform support

You can generate reports from the following platforms:

| Platform | Formats supported | Text content | Edits & Replies | Attachment Types | Reactions | Profile picture | Mentions |
|-----------|----------------------------------------------------------------------------------|--------------|------------------|-------------------------------------------------------------------------------------|------------------|------------------------|-------------|
| Discord | `json` from [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) ||||| ✅ (until link expires) | ✅ (as text) |
| Messenger | `json` from [Facebook DYI export](https://www.facebook.com/dyi) |||||| ✅ (as text) |
| Telegram | `json` from [Telegram Desktop](https://desktop.telegram.org/) |||| ❌ (not provided) || ✅ (as text) |
| WhatsApp | `txt` or `zip` exported from a phone || ❌ (not provided) | ✅<strong>*</strong> (if exported from iOS)<br>🟦 (generic if exported from Android) | ❌ (not provided) || ✅ (as text) |
| Platform | Formats supported | Text content | Edits & Replies | Attachment Types | Reactions | Profile picture | Mentions | Calls |
|-----------|----------------------------------------------------------------------------------|--------------|------------------|-------------------------------------------------------------------------------------|------------------|------------------------|-------------|-------|
| Discord | `json` from [DiscordChatExporter](https://github.com/Tyrrrz/DiscordChatExporter) ||||| ✅ (until link expires) | ✅ (as text) ||
| Messenger | `json` from [Facebook DYI export](https://www.facebook.com/dyi) |||||| ✅ (as text) ||
| Telegram | `json` from [Telegram Desktop](https://desktop.telegram.org/) |||| ❌ (not provided) || ✅ (as text) ||
| WhatsApp | `txt` or `zip` exported from a phone || ❌ (not provided) | ✅<strong>*</strong> (if exported from iOS)<br>🟦 (generic if exported from Android) | ❌ (not provided) || ✅ (as text) ||

<strong>*</strong> not all languages are supported, check [WhatsApp.ts](/pipeline/parse/parsers/WhatsApp.ts).

Expand Down
5 changes: 5 additions & 0 deletions pipeline/Platforms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ interface PlatformInformation {
reactions: boolean;
replies: boolean;
edits: boolean;
calls: boolean;
};
}

Expand All @@ -25,6 +26,7 @@ export const PlatformsInfo: {
reactions: true,
replies: true,
edits: true,
calls: true,
},
},
messenger: {
Expand All @@ -36,6 +38,7 @@ export const PlatformsInfo: {
reactions: false,
replies: false,
edits: false,
calls: false,
},
},
telegram: {
Expand All @@ -47,6 +50,7 @@ export const PlatformsInfo: {
reactions: false,
replies: true,
edits: true,
calls: true,
},
},
whatsapp: {
Expand All @@ -58,6 +62,7 @@ export const PlatformsInfo: {
reactions: false,
replies: false,
edits: false,
calls: false,
},
},
};
13 changes: 13 additions & 0 deletions pipeline/Time.ts
Original file line number Diff line number Diff line change
Expand Up @@ -273,3 +273,16 @@ export const formatDatetime = (format: TimeFormat, datetime?: Datetime) => {

return formatTime(format, Day.fromKey(datetime.key), datetime.secondOfDay);
};

/** Finds the time difference in seconds between two Datetimes */
export const diffDatetime = (a: Datetime, b: Datetime): number => {
// Probably this can be done more efficient and be reused (also see formatTime)

const aDate = Day.fromKey(a.key).toDate();
if (a.secondOfDay !== undefined) aDate.setSeconds(a.secondOfDay);

const bDate = Day.fromKey(b.key).toDate();
if (b.secondOfDay !== undefined) bDate.setSeconds(b.secondOfDay);

return (bDate.getTime() - aDate.getTime()) / 1000;
};
6 changes: 6 additions & 0 deletions pipeline/aggregate/Blocks.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { CommonBlockData } from "@pipeline/aggregate/Common";
import { Filters } from "@pipeline/aggregate/Filters";
import CallsActivity from "@pipeline/aggregate/blocks/calls/CallsActivity";
import CallsPerPeriod from "@pipeline/aggregate/blocks/calls/CallsPerPeriod";
import CallsStats from "@pipeline/aggregate/blocks/calls/CallsStats";
import DomainsStats from "@pipeline/aggregate/blocks/domains/DomainsStats";
import EmojiStats from "@pipeline/aggregate/blocks/emojis/EmojiStats";
import ConversationStats from "@pipeline/aggregate/blocks/interaction/ConversationStats";
Expand Down Expand Up @@ -37,6 +40,9 @@ export type BlockDescription<K, Data, Args = undefined> = {
/** All existing blocks must be defined here, so the UI can dynamically load them */
export const Blocks = {
[ActiveAuthors.key]: ActiveAuthors,
[CallsActivity.key]: CallsActivity,
[CallsPerPeriod.key]: CallsPerPeriod,
[CallsStats.key]: CallsStats,
[ConversationsDuration.key]: ConversationsDuration,
[ConversationStats.key]: ConversationStats,
[DomainsStats.key]: DomainsStats,
Expand Down
21 changes: 17 additions & 4 deletions pipeline/aggregate/Common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ export const computeCommonBlockData = (database: Database): CommonBlockData => {

export interface VariableDistribution {
total: number;
sum: number;
average: number;
/** Aggregation in `count.length` buckets of `(whiskerMax-whiskerMin)/count.length` */
count: number[];
/** Boxplot */
boxplot: {
min: number;
whiskerMin: number;
Expand All @@ -65,6 +66,8 @@ export interface VariableDistribution {
export const computeVariableDistribution = (values: Uint32Array, count: number): VariableDistribution => {
const res: VariableDistribution = {
total: count,
sum: 0,
average: 0,
count: [],
boxplot: {
min: 0,
Expand Down Expand Up @@ -114,15 +117,18 @@ export const computeVariableDistribution = (values: Uint32Array, count: number):
};

for (let i = 0; i < count; i++) {
const time = values[i];
if (time >= lower && time < upper) {
const value = values[i];
if (value >= lower && value < upper) {
// Order of operations is critical to avoid rounding issues
res.count[Math.floor((buckets / (upper - lower)) * (time - lower))]++;
res.count[Math.floor((buckets / (upper - lower)) * (value - lower))]++;
} else {
res.boxplot.outliers++;
}
res.sum += value;
}

res.average = res.sum / count;

return res;
};

Expand All @@ -137,3 +143,10 @@ export interface DateItem {
ts: number; // timestamp
v: number; // value
}

/** Activity entry for one hour of a weekday */
export interface WeekdayHourEntry {
value: number;
hour: `${number}hs`;
weekday: "Sun" | "Mon" | "Tue" | "Wed" | "Thu" | "Fri" | "Sat";
}
45 changes: 45 additions & 0 deletions pipeline/aggregate/blocks/calls/CallsActivity.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";
import { WeekdayHourEntry } from "@pipeline/aggregate/Common";

import { iterateHoursInCall } from "./CallsUtils";

export interface CallsActivity {
/** Each entry contains the total amount of seconds spent in calls for that hour of the week */
weekdayHourActivity: WeekdayHourEntry[];
}

const fn: BlockFn<CallsActivity> = (database, filters, common) => {
const weekdayHourDurations: number[] = new Array(7 * 24).fill(0);

for (const call of database.calls) {
// Note: time filtered below, we have to filter each hour individually
if (!filters.hasChannel(call.channelIndex)) continue;
if (!filters.hasAuthor(call.authorIndex)) continue;

iterateHoursInCall(call, (dayIndex, hourInDay, secondsInCall) => {
if (filters.inTime(dayIndex)) {
weekdayHourDurations[common.dayOfWeek[dayIndex] * 24 + hourInDay] += secondsInCall;
}
});
}

const weekdayHourActivity: WeekdayHourEntry[] = weekdayHourDurations.map((count, i) => {
const weekday = Math.floor(i / 24);
const hour = i % 24;
return {
value: count,
hour: `${hour}hs`,
weekday: (["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"] as const)[weekday],
};
});

return {
weekdayHourActivity,
};
};

export default {
key: "calls/activty",
triggers: ["time", "authors", "channels"],
fn,
} as BlockDescription<"calls/activty", CallsActivity>;
79 changes: 79 additions & 0 deletions pipeline/aggregate/blocks/calls/CallsPerPeriod.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";

import { iterateHoursInCall } from "./CallsUtils";

export type CallsInDate = {
ts: number; // timestamp

n: number; // number of calls
t: number; // total time in calls (seconds)
};

/**
* Number of calls per different time cycles.
* It ignores the time filter completely, all cycles are included.
*/
export interface CallsPerPeriod {
perDay: CallsInDate[];
perWeek: CallsInDate[];
perMonth: CallsInDate[];
}

const fn: BlockFn<CallsPerPeriod> = (database, filters, common) => {
const res: CallsPerPeriod = {
perDay: [],
perWeek: [],
perMonth: [],
};

const { keyToTimestamp } = common;
const { dateToWeekIndex, dateToMonthIndex } = common.timeKeys;

// fill empty
for (const ts of keyToTimestamp.date) {
res.perDay.push({
ts,
n: 0,
t: 0,
});
}
for (const ts of keyToTimestamp.week) {
res.perWeek.push({
ts,
n: 0,
t: 0,
});
}
for (const ts of keyToTimestamp.month) {
res.perMonth.push({
ts,
n: 0,
t: 0,
});
}

for (const call of database.calls) {
// check filters
// if (!filters.inTime(call.start.dayIndex)) continue; // don't filter by time, UI scrolls the time natively
if (!filters.hasChannel(call.channelIndex)) continue;
if (!filters.hasAuthor(call.authorIndex)) continue;

iterateHoursInCall(call, (dayIndex, _, secondsInCall) => {
res.perDay[dayIndex].t += secondsInCall;
res.perWeek[dateToWeekIndex[dayIndex]].t += secondsInCall;
res.perMonth[dateToMonthIndex[dayIndex]].t += secondsInCall;
});

res.perDay[call.start.dayIndex].n += 1;
res.perWeek[dateToWeekIndex[call.start.dayIndex]].n += 1;
res.perMonth[dateToMonthIndex[call.start.dayIndex]].n += 1;
}

return res;
};

export default {
key: "calls/per-period",
triggers: ["authors", "channels"],
fn,
} as BlockDescription<"calls/per-period", CallsPerPeriod>;
95 changes: 95 additions & 0 deletions pipeline/aggregate/blocks/calls/CallsStats.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import { Datetime, diffDatetime } from "@pipeline/Time";
import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";
import { VariableDistribution, computeVariableDistribution } from "@pipeline/aggregate/Common";
import { filterMessages } from "@pipeline/aggregate/Helpers";
import { MessageView } from "@pipeline/serialization/MessageView";

interface CallDuration {
duration: number;
start: Datetime;
}

export interface CallsStats {
/** Total number of calls */
total: number;
/** Total number of seconds spent in calls */
secondsInCall: number;

longestCall?: CallDuration;

/** Call duration distribution in seconds */
durationDistribution: VariableDistribution;
/** Time between calls distribution in seconds */
timesBetweenDistribution: VariableDistribution;

/** Number of calls made by each author */
authorsCount: number[];
}

const fn: BlockFn<CallsStats> = (database, filters, common, args) => {
const { dateKeys } = common.timeKeys;

let total = 0;
let secondsInCall = 0;
let longestCall: CallDuration | undefined = undefined;
let lastCall: Datetime | undefined = undefined;

const authorsCount = new Array(database.authors.length).fill(0);

const durations = new Uint32Array(database.calls.length).fill(0xfffffff0);
const timesBetween = new Uint32Array(database.calls.length).fill(0xfffffff0);

for (const call of database.calls) {
if (!filters.inTime(call.start.dayIndex)) continue;
if (!filters.hasChannel(call.channelIndex)) continue;
if (!filters.hasAuthor(call.authorIndex)) continue;

const startDatetime = {
key: dateKeys[call.start.dayIndex],
secondOfDay: call.start.secondOfDay,
};
const endDatetime = {
key: dateKeys[call.end.dayIndex],
secondOfDay: call.end.secondOfDay,
};

durations[total] = call.duration;
authorsCount[call.authorIndex]++;

if (longestCall === undefined || call.duration > longestCall.duration) {
longestCall = {
duration: call.duration,
start: startDatetime,
};
}

if (lastCall !== undefined) {
// compute time difference between calls
const diff = diffDatetime(lastCall, startDatetime);
if (diff < 0) throw new Error("Time difference between calls is negative, diff=" + diff);
timesBetween[total - 1] = diff;
}
lastCall = endDatetime;

secondsInCall += call.duration;
total++;
}

return {
total,
secondsInCall,
averageDuration: secondsInCall / total,
longestCall,

durationDistribution: computeVariableDistribution(durations, total),
timesBetweenDistribution: computeVariableDistribution(timesBetween, total - 1),

authorsCount,
};
};

export default {
key: "calls/stats",
triggers: ["authors", "channels", "time"],
fn,
} as BlockDescription<"calls/stats", CallsStats>;
Loading

0 comments on commit 6378e92

Please sign in to comment.