diff --git a/pipeline/aggregate/Blocks.ts b/pipeline/aggregate/Blocks.ts index 4b8c4bed..7c7d34db 100644 --- a/pipeline/aggregate/Blocks.ts +++ b/pipeline/aggregate/Blocks.ts @@ -7,10 +7,10 @@ import ExternalStats from "@pipeline/aggregate/blocks/ExternalStats"; import Growth from "@pipeline/aggregate/blocks/Growth"; import InteractionStats from "@pipeline/aggregate/blocks/InteractionStats"; import LanguageStats from "@pipeline/aggregate/blocks/LanguageStats"; -import MessagesPerCycle from "@pipeline/aggregate/blocks/MessagesPerCycle"; import MessagesStats from "@pipeline/aggregate/blocks/MessagesStats"; import SentimentPerCycle from "@pipeline/aggregate/blocks/SentimentPerCycle"; import SentimentStats from "@pipeline/aggregate/blocks/SentimentStats"; +import MessagesPerCycle from "@pipeline/aggregate/blocks/messages/MessagesPerCycle"; import { Database } from "@pipeline/process/Types"; import WordStats from "./blocks/WordStats"; diff --git a/pipeline/aggregate/blocks/MessagesPerCycle.ts b/pipeline/aggregate/blocks/messages/MessagesPerCycle.ts similarity index 93% rename from pipeline/aggregate/blocks/MessagesPerCycle.ts rename to pipeline/aggregate/blocks/messages/MessagesPerCycle.ts index d4d890f1..fac29eff 100644 --- a/pipeline/aggregate/blocks/MessagesPerCycle.ts +++ b/pipeline/aggregate/blocks/messages/MessagesPerCycle.ts @@ -8,13 +8,17 @@ type MessagesInDate = { m: number; // messages }; +/** + * Number of messages per different time cycles. + * It ignores the time filter completely, all cycles are included. + */ export interface MessagesPerCycle { perDay: MessagesInDate[]; perWeek: MessagesInDate[]; perMonth: MessagesInDate[]; } -const fn: BlockFn = (database, filters, common, args) => { +const fn: BlockFn = (database, filters, common) => { const res: MessagesPerCycle = { perDay: [], perWeek: [], diff --git a/report/components/viz/MessagesOverTime.tsx b/report/components/viz/MessagesOverTime.tsx index 137e6290..0efa6600 100644 --- a/report/components/viz/MessagesOverTime.tsx +++ b/report/components/viz/MessagesOverTime.tsx @@ -14,7 +14,7 @@ import { XYSeries, } from "@amcharts/amcharts5/xy"; import { Filter } from "@pipeline/aggregate/Blocks"; -import { MessagesPerCycle } from "@pipeline/aggregate/blocks/MessagesPerCycle"; +import { MessagesPerCycle } from "@pipeline/aggregate/blocks/messages/MessagesPerCycle"; import { getWorker } from "@report/WorkerWrapper"; import { Themes } from "./AmCharts5"; diff --git a/tests/aggregate/Common.ts b/tests/aggregate/Common.ts new file mode 100644 index 00000000..33439ab6 --- /dev/null +++ b/tests/aggregate/Common.ts @@ -0,0 +1,13 @@ +import { generateDatabase } from "@pipeline/index"; + +import { TestEnv, loadSamples } from "@tests/samples"; + +export const loadTestDatabase = async () => { + const samples = await loadSamples(["discord/GC_3A_5M.json", "discord/SV_5A_5M.json"]); + const db = await generateDatabase( + samples.map((s) => s.input), + { platform: "discord" }, + TestEnv + ); + return db; +}; diff --git a/tests/aggregate/Helper.test.ts b/tests/aggregate/Helper.test.ts new file mode 100644 index 00000000..38a7f9b6 --- /dev/null +++ b/tests/aggregate/Helper.test.ts @@ -0,0 +1,83 @@ +import { Index } from "@pipeline/Types"; +import { computeCommonBlockData } from "@pipeline/aggregate/Common"; +import { Filters } from "@pipeline/aggregate/Filters"; +import { filterMessages } from "@pipeline/aggregate/Helpers"; +import { Database, Message } from "@pipeline/process/Types"; +import { BitStream } from "@pipeline/serialization/BitStream"; +import { MessagesArray } from "@pipeline/serialization/MessagesArray"; + +import { loadTestDatabase } from "@tests/aggregate/Common"; + +let db: Database; +let fn: jest.Mock; +let filters: Filters; + +let allowedAuthors: Index[]; +let allowedChannels: Index[]; +let allowedDays: Index[]; + +let allMessages: { msg: Message; channelIndex: Index }[]; + +beforeEach(async () => { + fn = jest.fn(); + db = await loadTestDatabase(); + + allMessages = []; + for (const channel of db.channels) { + const channelIndex = db.channels.indexOf(channel); + const stream = new BitStream(db.messages.buffer); + if (channel.msgAddr !== undefined) { + stream.offset = channel.msgAddr; + const arr = new MessagesArray(db.bitConfig, stream, channel.msgCount); + for (const msg of arr) { + allMessages.push({ msg, channelIndex }); + } + } + } + + const dateKeys = computeCommonBlockData(db).timeKeys.dateKeys; + allowedAuthors = db.authors.slice(0, db.authors.length / 2).map((_, i) => i); + allowedChannels = db.channels.slice(0, db.channels.length / 2).map((_, i) => i); + const days = dateKeys.slice(0, dateKeys.length / 2); + allowedDays = days.map((_, i) => i); + + filters = new Filters(db); + filters.updateAuthors(allowedAuthors); + filters.updateChannels(allowedChannels); + filters.updateEndDate(days[days.length - 1]); +}); + +describe("filterMessages", () => { + // prettier-ignore + test.each([ + { authors: false, channels: false, time: false, }, + { authors: false, channels: false, time: true, }, + { authors: false, channels: true, time: false, }, + { authors: false, channels: true, time: true, }, + { authors: true, channels: false, time: false, }, + { authors: true, channels: false, time: true, }, + { authors: true, channels: true, time: false, }, + { authors: true, channels: true, time: true, }, + ])(`filters with active %p`, (activeFilters) => { + filterMessages(fn, db, filters, activeFilters); + + const expectedMessages = allMessages.filter(({channelIndex,msg}) => { + return ( + (!activeFilters.authors || allowedAuthors.includes(msg.authorIndex)) && + (!activeFilters.channels || allowedChannels.includes(channelIndex)) && + (!activeFilters.time || allowedDays.includes(msg.dayIndex)) + ); + }); + + expect(fn).toHaveBeenCalledTimes(expectedMessages.length); + + for (const { msg } of expectedMessages) { + expect(fn).toHaveBeenCalledWith(expect.objectContaining({ + authorIndex: msg.authorIndex, + dayIndex: msg.dayIndex, + secondOfDay: msg.secondOfDay, + langIndex: msg.langIndex, + })); + } + }); +}); diff --git a/tests/aggregate/messages/MessagesPerCycle.test.ts b/tests/aggregate/messages/MessagesPerCycle.test.ts new file mode 100644 index 00000000..ddfb30d8 --- /dev/null +++ b/tests/aggregate/messages/MessagesPerCycle.test.ts @@ -0,0 +1 @@ +test("MessagesPerCycle", async () => {});