From 7148f66d1af901b7d55e93b005261484461a9ec1 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 4 Aug 2023 13:18:30 -0700 Subject: [PATCH] Refactored query parameter handling. --- lib/clickhouse.ts | 23 ++++-- lib/constants.ts | 3 + lib/prisma.ts | 19 ++++- lib/types.ts | 36 +++++---- pages/api/websites/[id]/metrics.ts | 17 ++-- pages/api/websites/[id]/pageviews.ts | 52 +++++-------- pages/api/websites/[id]/stats.ts | 46 ++++------- .../analytics/eventData/getEventDataEvents.ts | 77 ++++++++----------- .../analytics/eventData/getEventDataFields.ts | 28 +++---- queries/analytics/events/getEventMetrics.ts | 59 +++++--------- .../analytics/pageviews/getPageviewMetrics.ts | 67 ++++------------ .../analytics/pageviews/getPageviewStats.ts | 65 ++++------------ queries/analytics/reports/getInsights.ts | 54 ++++--------- .../analytics/sessions/getSessionMetrics.ts | 48 ++++-------- queries/analytics/sessions/getSessionStats.ts | 65 ++++------------ .../analytics/stats/getWebsiteDateRange.ts | 14 ++-- queries/analytics/stats/getWebsiteStats.ts | 56 ++++---------- 17 files changed, 260 insertions(+), 469 deletions(-) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index 19d0940543..6d5bcf428b 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -2,8 +2,10 @@ import { ClickHouse } from 'clickhouse'; import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; -import { WebsiteMetricFilter } from './types'; -import { FILTER_COLUMNS } from './constants'; +import { QueryFilters } from './types'; +import { FILTER_COLUMNS, IGNORED_FILTERS } from './constants'; +import { loadWebsite } from './load'; +import { maxDate } from './date'; export const CLICKHOUSE_DATE_FORMATS = { minute: '%Y-%m-%d %H:%M:00', @@ -65,13 +67,13 @@ function getFilterQuery(filters = {}) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; - if (filter !== undefined) { + if (filter !== undefined && !IGNORED_FILTERS.includes(key)) { const column = FILTER_COLUMNS[key] || key; arr.push(`and ${column} = {${key}:String}`); } if (key === 'referrer') { - arr.push('and referrer_domain != {domain:String}'); + arr.push('and referrer_domain != {websiteDomain:String}'); } return arr; @@ -80,9 +82,20 @@ function getFilterQuery(filters = {}) { return query.join('\n'); } -function parseFilters(filters: WebsiteMetricFilter = {}) { +async function parseFilters( + websiteId: string, + filters: QueryFilters & { [key: string]: any } = {}, +) { + const website = await loadWebsite(websiteId); + return { filterQuery: getFilterQuery(filters), + params: { + ...filters, + websiteId, + startDate: maxDate(filters.startDate, website.resetAt), + websiteDomain: website.domain, + }, }; } diff --git a/lib/constants.ts b/lib/constants.ts index 2b3da8754a..9362b45601 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -50,8 +50,11 @@ export const FILTER_COLUMNS = { query: 'url_query', event: 'event_name', region: 'subdivision1', + type: 'event_type', }; +export const IGNORED_FILTERS = ['startDate', 'endDate', 'timezone', 'unit', 'eventType']; + export const COLLECTION_TYPE = { event: 'event', identify: 'identify', diff --git a/lib/prisma.ts b/lib/prisma.ts index a6f1ff8898..d1b9d0e5d1 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -1,7 +1,10 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; -import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants'; +import { FILTER_COLUMNS, IGNORED_FILTERS, SESSION_COLUMNS } from './constants'; +import { loadWebsite } from './load'; +import { maxDate } from './date'; +import { QueryFilters } from './types'; const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -68,14 +71,14 @@ function getFilterQuery(filters = {}): string { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; - if (filter !== undefined) { + if (filter !== undefined && !IGNORED_FILTERS.includes(key)) { const column = FILTER_COLUMNS[key] || key; arr.push(`and ${column}={{${key}}}`); } if (key === 'referrer') { arr.push( - 'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)', + 'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)', ); } @@ -85,12 +88,20 @@ function getFilterQuery(filters = {}): string { return query.join('\n'); } -function parseFilters(filters: { [key: string]: any } = {}) { +async function parseFilters(websiteId, filters: QueryFilters & { [key: string]: any } = {}) { + const website = await loadWebsite(websiteId); + return { joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key]) ? `inner join session on website_event.session_id = session.session_id` : '', filterQuery: getFilterQuery(filters), + params: { + ...filters, + websiteId, + startDate: maxDate(filters.startDate, website.resetAt), + websiteDomain: website.domain, + }, }; } diff --git a/lib/types.ts b/lib/types.ts index 7c91ec4f79..131740f887 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -73,21 +73,6 @@ export interface WebsiteMetric { y: number; } -export interface WebsiteMetricFilter { - domain?: string; - url?: string; - referrer?: string; - title?: string; - query?: string; - event?: string; - os?: string; - browser?: string; - device?: string; - country?: string; - region?: string; - city?: string; -} - export interface WebsiteEventMetric { x: string; t: string; @@ -144,3 +129,24 @@ export interface DateRange { unit: string; value: string; } + +export interface QueryFilters { + startDate?: Date; + endDate?: Date; + timezone?: string; + unit?: string; + domain?: string; + eventType?: number; + url?: string; + referrer?: string; + title?: string; + query?: string; + event?: string; + os?: string; + browser?: string; + device?: string; + country?: string; + region?: string; + city?: string; + language?: string; +} diff --git a/pages/api/websites/[id]/metrics.ts b/pages/api/websites/[id]/metrics.ts index 15389e3ee6..7c84583c2a 100644 --- a/pages/api/websites/[id]/metrics.ts +++ b/pages/api/websites/[id]/metrics.ts @@ -23,6 +23,7 @@ export interface WebsiteMetricsRequestQuery { country: string; region: string; city: string; + language: string; } export default async ( @@ -57,6 +58,8 @@ export default async ( const { startDate, endDate } = await parseDateRangeQuery(req); const filters = { + startDate, + endDate, url, referrer, title, @@ -76,12 +79,7 @@ export default async ( const column = FILTER_COLUMNS[type] || type; if (SESSION_COLUMNS.includes(type)) { - const data = await getSessionMetrics(websiteId, { - startDate, - endDate, - column, - filters, - }); + const data = await getSessionMetrics(websiteId, column, filters); if (type === 'language') { const combined = {}; @@ -103,12 +101,7 @@ export default async ( } if (EVENT_COLUMNS.includes(type)) { - const data = await getPageviewMetrics(websiteId, { - startDate, - endDate, - column, - filters, - }); + const data = await getPageviewMetrics(websiteId, column, filters); return ok(res, data); } diff --git a/pages/api/websites/[id]/pageviews.ts b/pages/api/websites/[id]/pageviews.ts index 810854a752..87c60d58e4 100644 --- a/pages/api/websites/[id]/pageviews.ts +++ b/pages/api/websites/[id]/pageviews.ts @@ -57,41 +57,25 @@ export default async ( return badRequest(res); } + const filters = { + startDate, + endDate, + timezone, + unit, + url, + referrer, + title, + os, + browser, + device, + country, + region, + city, + }; + const [pageviews, sessions] = await Promise.all([ - getPageviewStats(websiteId, { - startDate, - endDate, - timezone, - unit, - filters: { - url, - referrer, - title, - os, - browser, - device, - country, - region, - city, - }, - }), - getSessionStats(websiteId, { - startDate, - endDate, - timezone, - unit, - filters: { - url, - referrer, - title, - os, - browser, - device, - country, - region, - city, - }, - }), + getPageviewStats(websiteId, filters), + getSessionStats(websiteId, filters), ]); return ok(res, { pageviews, sessions }); diff --git a/pages/api/websites/[id]/stats.ts b/pages/api/websites/[id]/stats.ts index 3164913d96..a77c7eaf88 100644 --- a/pages/api/websites/[id]/stats.ts +++ b/pages/api/websites/[id]/stats.ts @@ -56,40 +56,26 @@ export default async ( const prevStartDate = subMinutes(startDate, diff); const prevEndDate = subMinutes(endDate, diff); - const metrics = await getWebsiteStats(websiteId, { - startDate, - endDate, - filters: { - url, - referrer, - title, - query, - event, - os, - browser, - device, - country, - region, - city, - }, - }); + const filters = { + url, + referrer, + title, + query, + event, + os, + browser, + device, + country, + region, + city, + }; + + const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate }); const prevPeriod = await getWebsiteStats(websiteId, { + ...filters, startDate: prevStartDate, endDate: prevEndDate, - filters: { - url, - referrer, - title, - query, - event, - os, - browser, - device, - country, - region, - city, - }, }); const stats = Object.keys(metrics[0]).reduce((obj, key) => { diff --git a/queries/analytics/eventData/getEventDataEvents.ts b/queries/analytics/eventData/getEventDataEvents.ts index 634a28a26a..fae46db107 100644 --- a/queries/analytics/eventData/getEventDataEvents.ts +++ b/queries/analytics/eventData/getEventDataEvents.ts @@ -1,17 +1,10 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import { WebsiteEventDataFields } from 'lib/types'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters, WebsiteEventDataFields } from 'lib/types'; export async function getEventDataEvents( - ...args: [ - websiteId: string, - startDate: Date, - endDate: Date, - filters: { field?: string; event?: string }, - ] + ...args: [websiteId: string, filters: QueryFilters & { field?: string; event?: string }] ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -21,64 +14,60 @@ export async function getEventDataEvents( async function relationalQuery( websiteId: string, - startDate: Date, - endDate: Date, - filters: { field?: string; event?: string }, + filters: QueryFilters & { field?: string; event?: string }, ) { - const { rawQuery } = prisma; - const website = await loadWebsite(websiteId); - const { event } = filters; + const { rawQuery, parseFilters } = prisma; + const { params } = await parseFilters(websiteId, filters); if (event) { return rawQuery( ` select - we.event_name as event, - ed.event_key as field, - ed.data_type as type, - ed.string_value as value, + website_event.event_name as event, + event_data.event_key as field, + event_data.data_type as type, + event_data.string_value as value, count(*) as total - from event_data as ed - inner join website_event as we - on we.event_id = ed.website_event_id - where ed.website_id = {{websiteId::uuid}} - and ed.created_at between {{startDate}} and {{endDate}} - and we.event_name = {{event}} - group by we.event_name, ed.event_key, ed.data_type, ed.string_value + from event_data + inner join website_event + on website_event.event_id = event_data.website_event_id + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + and websit_event.event_name = {{event}} + group by website_event.event_name, event_data.event_key, event_data.data_type, event_data.string_value order by 1 asc, 2 asc, 3 asc, 4 desc `, - { websiteId, startDate: maxDate(startDate, website.resetAt), endDate, ...filters }, + params, ); } + return rawQuery( ` select - we.event_name as event, - ed.event_key as field, - ed.data_type as type, + website_event.event_name as event, + event_data.event_key as field, + event_data.data_type as type, count(*) as total - from event_data as ed - inner join website_event as we - on we.event_id = ed.website_event_id - where ed.website_id = {{websiteId::uuid}} - and ed.created_at between {{startDate}} and {{endDate}} - group by we.event_name, ed.event_key, ed.data_type + from event_data + inner join website_event + on website_event.event_id = event_data.website_event_id + where event_data.website_id = {{websiteId::uuid}} + and event_data.created_at between {{startDate}} and {{endDate}} + group by website_event.event_name, event_data.event_key, event_data.data_type order by 1 asc, 2 asc limit 100 `, - { websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } async function clickhouseQuery( websiteId: string, - startDate: Date, - endDate: Date, - filters: { field?: string; event?: string }, + filters: QueryFilters & { field?: string; event?: string }, ) { - const { rawQuery } = clickhouse; - const website = await loadWebsite(websiteId); + const { rawQuery, parseFilters } = clickhouse; const { event } = filters; + const { params } = await parseFilters(websiteId, filters); if (event) { return rawQuery( @@ -97,7 +86,7 @@ async function clickhouseQuery( order by 1 asc, 2 asc, 3 asc, 4 desc limit 100 `, - { ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } @@ -115,6 +104,6 @@ async function clickhouseQuery( order by 1 asc, 2 asc limit 100 `, - { websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } diff --git a/queries/analytics/eventData/getEventDataFields.ts b/queries/analytics/eventData/getEventDataFields.ts index 516c58d0c2..a27f22810a 100644 --- a/queries/analytics/eventData/getEventDataFields.ts +++ b/queries/analytics/eventData/getEventDataFields.ts @@ -1,12 +1,10 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import { WebsiteEventDataFields } from 'lib/types'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters, WebsiteEventDataFields } from 'lib/types'; export async function getEventDataFields( - ...args: [websiteId: string, startDate: Date, endDate: Date, field?: string] + ...args: [websiteId: string, filters: QueryFilters & { field?: string }] ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -14,9 +12,10 @@ export async function getEventDataFields( }); } -async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) { - const { rawQuery } = prisma; - const website = await loadWebsite(websiteId); +async function relationalQuery(websiteId: string, filters: QueryFilters & { field?: string }) { + const { rawQuery, parseFilters } = prisma; + const { field } = filters; + const { params } = await parseFilters(websiteId, filters); if (field) { return rawQuery( @@ -33,7 +32,7 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date order by 3 desc, 2 desc, 1 asc limit 100 `, - { websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } @@ -50,13 +49,14 @@ async function relationalQuery(websiteId: string, startDate: Date, endDate: Date order by 3 desc, 2 asc, 1 asc limit 100 `, - { websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } -async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) { - const { rawQuery } = clickhouse; - const website = await loadWebsite(websiteId); +async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) { + const { rawQuery, parseFilters } = clickhouse; + const { field } = filters; + const { params } = await parseFilters(websiteId, filters); if (field) { return rawQuery( @@ -73,7 +73,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date order by 3 desc, 2 desc, 1 asc limit 100 `, - { websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } @@ -90,6 +90,6 @@ async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date order by 3 desc, 2 asc, 1 asc limit 100 `, - { websiteId, startDate: maxDate(startDate, website.resetAt), endDate }, + params, ); } diff --git a/queries/analytics/events/getEventMetrics.ts b/queries/analytics/events/getEventMetrics.ts index 03b252b796..cf862f4a7b 100644 --- a/queries/analytics/events/getEventMetrics.ts +++ b/queries/analytics/events/getEventMetrics.ts @@ -1,24 +1,11 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; -import { WebsiteEventMetric } from 'lib/types'; +import { WebsiteEventMetric, QueryFilters } from 'lib/types'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; - -export interface GetEventMetricsCriteria { - startDate: Date; - endDate: Date; - timezone: string; - unit: string; - filters: { - url: string; - eventName: string; - }; -} export async function getEventMetrics( - ...args: [websiteId: string, criteria: GetEventMetricsCriteria] + ...args: [websiteId: string, criteria: QueryFilters] ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -26,11 +13,13 @@ export async function getEventMetrics( }); } -async function relationalQuery(websiteId: string, criteria: GetEventMetricsCriteria) { - const { startDate, endDate, timezone = 'utc', unit = 'day', filters } = criteria; - const { rawQuery, getDateQuery, getFilterQuery } = prisma; - const website = await loadWebsite(websiteId); - const filterQuery = getFilterQuery(filters); +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; + const { rawQuery, getDateQuery, parseFilters } = prisma; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.customEvent, + }); return rawQuery( ` @@ -46,22 +35,17 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite group by 1, 2 order by 2 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.customEvent, - domain: website.domain, - }, + params, ); } -async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCriteria) { - const { startDate, endDate, timezone = 'utc', unit = 'day', filters } = criteria; - const { rawQuery, getDateQuery, getFilterQuery } = clickhouse; - const website = await loadWebsite(websiteId); - const filterQuery = getFilterQuery(filters); +async function clickhouseQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; + const { rawQuery, getDateQuery, parseFilters } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.customEvent, + }); return rawQuery( ` @@ -77,13 +61,6 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite group by x, t order by t `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.customEvent, - domain: website.domain, - }, + params, ); } diff --git a/queries/analytics/pageviews/getPageviewMetrics.ts b/queries/analytics/pageviews/getPageviewMetrics.ts index 8e4460e611..bbbd77052d 100644 --- a/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/queries/analytics/pageviews/getPageviewMetrics.ts @@ -2,19 +2,10 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters } from 'lib/types'; export async function getPageviewMetrics( - ...args: [ - websiteId: string, - criteria: { - startDate: Date; - endDate: Date; - column: string; - filters: object; - }, - ] + ...args: [websiteId: string, columns: string, filters: QueryFilters] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -22,20 +13,12 @@ export async function getPageviewMetrics( }); } -async function relationalQuery( - websiteId: string, - criteria: { - startDate: Date; - endDate: Date; - column: string; - filters: object; - }, -) { - const { startDate, endDate, filters = {}, column } = criteria; +async function relationalQuery(websiteId: string, column: string, filters: QueryFilters) { const { rawQuery, parseFilters } = prisma; - const website = await loadWebsite(websiteId); - - const { filterQuery, joinSession } = parseFilters(filters); + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -50,31 +33,16 @@ async function relationalQuery( order by 2 desc limit 100 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } -async function clickhouseQuery( - websiteId: string, - criteria: { - startDate: Date; - endDate: Date; - column: string; - filters: object; - }, -) { - const { startDate, endDate, filters = {}, column } = criteria; +async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) { const { rawQuery, parseFilters } = clickhouse; - const website = await loadWebsite(websiteId); - - const { filterQuery } = parseFilters(filters); + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -88,13 +56,6 @@ async function clickhouseQuery( order by y desc limit 100 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } diff --git a/queries/analytics/pageviews/getPageviewStats.ts b/queries/analytics/pageviews/getPageviewStats.ts index cdbd6442be..d6a980ef7a 100644 --- a/queries/analytics/pageviews/getPageviewStats.ts +++ b/queries/analytics/pageviews/getPageviewStats.ts @@ -2,43 +2,22 @@ import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters } from 'lib/types'; -export interface PageviewStatsCriteria { - startDate: Date; - endDate: Date; - timezone?: string; - unit?: string; - filters: { - url?: string; - referrer?: string; - title?: string; - browser?: string; - os?: string; - device?: string; - screen?: string; - language?: string; - country?: string; - region?: string; - city?: string; - }; -} - -export async function getPageviewStats( - ...args: [websiteId: string, criteria: PageviewStatsCriteria] -) { +export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } -async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) { - const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria; +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; const { getDateQuery, parseFilters, rawQuery } = prisma; - const website = await loadWebsite(websiteId); - const { filterQuery, joinSession } = parseFilters(filters); + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -53,22 +32,17 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri ${filterQuery} group by 1 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } -async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) { - const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria; +async function clickhouseQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'UTC', unit = 'day' } = filters; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; - const website = await loadWebsite(websiteId); - const { filterQuery } = parseFilters(filters); + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -88,13 +62,6 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri ) as g order by t `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } diff --git a/queries/analytics/reports/getInsights.ts b/queries/analytics/reports/getInsights.ts index ff1399317f..dfe7c39754 100644 --- a/queries/analytics/reports/getInsights.ts +++ b/queries/analytics/reports/getInsights.ts @@ -1,19 +1,10 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; -import { maxDate } from 'lib/date'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; +import { QueryFilters } from 'lib/types'; -export interface GetInsightsCriteria { - startDate: Date; - endDate: Date; - fields: { name: string; type: string; value: string }[]; - filters: string[]; - groups: string[]; -} - -export async function getInsights(...args: [websiteId: string, criteria: GetInsightsCriteria]) { +export async function getInsights(...args: [websiteId: string, filters: QueryFilters]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), @@ -22,18 +13,18 @@ export async function getInsights(...args: [websiteId: string, criteria: GetInsi async function relationalQuery( websiteId: string, - criteria: GetInsightsCriteria, + filters: QueryFilters, ): Promise< { x: string; y: number; }[] > { - const { startDate, endDate, filters = [] } = criteria; const { parseFilters, rawQuery } = prisma; - const website = await loadWebsite(websiteId); - const params = {}; - const { filterQuery, joinSession } = parseFilters(params); + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -48,37 +39,30 @@ async function relationalQuery( ${filterQuery} group by 1 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - }, + params, ); } async function clickhouseQuery( websiteId: string, - criteria: GetInsightsCriteria, + filters: QueryFilters, ): Promise< { x: string; y: number; }[] > { - const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria; const { parseFilters, rawQuery } = clickhouse; - const website = await loadWebsite(websiteId); - const params = {}; - const { filterQuery } = parseFilters(params); - - const fieldsQuery = parseFields(fields); + const { fields } = filters; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` select - ${fieldsQuery} + ${parseFields(fields)} from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime} and {endDate:DateTime} @@ -88,13 +72,7 @@ async function clickhouseQuery( order by total desc limit 500 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - }, + params, ); } diff --git a/queries/analytics/sessions/getSessionMetrics.ts b/queries/analytics/sessions/getSessionMetrics.ts index a9b49ec81c..910c9785a4 100644 --- a/queries/analytics/sessions/getSessionMetrics.ts +++ b/queries/analytics/sessions/getSessionMetrics.ts @@ -2,14 +2,10 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters } from 'lib/types'; export async function getSessionMetrics( - ...args: [ - websiteId: string, - criteria: { startDate: Date; endDate: Date; column: string; filters: object }, - ] + ...args: [websiteId: string, column: string, filters: QueryFilters] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -17,14 +13,12 @@ export async function getSessionMetrics( }); } -async function relationalQuery( - websiteId: string, - criteria: { startDate: Date; endDate: Date; column: string; filters: object }, -) { - const website = await loadWebsite(websiteId); - const { startDate, endDate, column, filters = {} } = criteria; +async function relationalQuery(websiteId: string, column: string, filters: QueryFilters) { const { parseFilters, rawQuery } = prisma; - const { filterQuery, joinSession } = parseFilters(filters); + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( `select ${column} x, count(*) y @@ -32,28 +26,22 @@ async function relationalQuery( ${joinSession} where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = {{eventType}} ${filterQuery} ) as t group by 1 order by 2 desc limit 100`, - { - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - ...filters, - }, + params, ); } -async function clickhouseQuery( - websiteId: string, - data: { startDate: Date; endDate: Date; column: string; filters: object }, -) { - const { startDate, endDate, column, filters = {} } = data; +async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) { const { parseFilters, rawQuery } = clickhouse; - const website = await loadWebsite(websiteId); - const { filterQuery } = parseFilters(filters); + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -68,12 +56,6 @@ async function clickhouseQuery( order by y desc limit 100 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - }, + params, ); } diff --git a/queries/analytics/sessions/getSessionStats.ts b/queries/analytics/sessions/getSessionStats.ts index 7633f242e4..9ed01a5948 100644 --- a/queries/analytics/sessions/getSessionStats.ts +++ b/queries/analytics/sessions/getSessionStats.ts @@ -2,43 +2,22 @@ import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters } from 'lib/types'; -export interface SessionStatsCriteria { - startDate: Date; - endDate: Date; - timezone?: string; - unit?: string; - filters: { - url?: string; - referrer?: string; - title?: string; - browser?: string; - os?: string; - device?: string; - screen?: string; - language?: string; - country?: string; - region?: string; - city?: string; - }; -} - -export async function getSessionStats( - ...args: [websiteId: string, criteria: SessionStatsCriteria] -) { +export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } -async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria) { - const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria; +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; const { getDateQuery, parseFilters, rawQuery } = prisma; - const website = await loadWebsite(websiteId); - const { filterQuery, joinSession } = parseFilters(filters); + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -53,22 +32,17 @@ async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria ${filterQuery} group by 1 `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } -async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria) { - const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria; +async function clickhouseQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'UTC', unit = 'day' } = filters; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; - const website = await loadWebsite(websiteId); - const { filterQuery } = parseFilters(filters); + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -88,13 +62,6 @@ async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria ) as g order by t `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } diff --git a/queries/analytics/stats/getWebsiteDateRange.ts b/queries/analytics/stats/getWebsiteDateRange.ts index 45885e45c6..4fb24733a2 100644 --- a/queries/analytics/stats/getWebsiteDateRange.ts +++ b/queries/analytics/stats/getWebsiteDateRange.ts @@ -1,9 +1,7 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; -import { loadWebsite } from 'lib/load'; import { DEFAULT_RESET_DATE } from 'lib/constants'; -import { maxDate } from 'lib/date'; export async function getWebsiteDateRange(...args: [websiteId: string]) { return runQuery({ @@ -13,8 +11,8 @@ export async function getWebsiteDateRange(...args: [websiteId: string]) { } async function relationalQuery(websiteId: string) { - const { rawQuery } = prisma; - const website = await loadWebsite(websiteId); + const { rawQuery, parseFilters } = prisma; + const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) }); const result = await rawQuery( ` @@ -25,15 +23,15 @@ async function relationalQuery(websiteId: string) { where website_id = {{websiteId::uuid}} and created_at >= {{startDate}} `, - { websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) }, + params, ); return result[0] ?? null; } async function clickhouseQuery(websiteId: string) { - const { rawQuery } = clickhouse; - const website = await loadWebsite(websiteId); + const { rawQuery, parseFilters } = clickhouse; + const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) }); const result = await rawQuery( ` @@ -44,7 +42,7 @@ async function clickhouseQuery(websiteId: string) { where website_id = {websiteId:UUID} and created_at >= {startDate:DateTime} `, - { websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) }, + params, ); return result[0] ?? null; diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index e048fc8f0e..1651951199 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -2,29 +2,21 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { EVENT_TYPE } from 'lib/constants'; -import { loadWebsite } from 'lib/load'; -import { maxDate } from 'lib/date'; +import { QueryFilters } from 'lib/types'; -export async function getWebsiteStats( - ...args: [ - websiteId: string, - data: { startDate: Date; endDate: Date; type?: string; filters: object }, - ] -) { +export async function getWebsiteStats(...args: [websiteId: string, filters: QueryFilters]) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } -async function relationalQuery( - websiteId: string, - criteria: { startDate: Date; endDate: Date; filters: object }, -) { - const { startDate, endDate, filters = {} } = criteria; +async function relationalQuery(websiteId: string, filters: QueryFilters) { const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma; - const website = await loadWebsite(websiteId); - const { filterQuery, joinSession } = parseFilters(filters); + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -43,32 +35,23 @@ async function relationalQuery( join website on website_event.website_id = website.website_id ${joinSession} - where event_type = {{eventType}} - and website.website_id = {{websiteId::uuid}} + where website.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} ${filterQuery} group by 1, 2 ) as t `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); } -async function clickhouseQuery( - websiteId: string, - criteria: { startDate: Date; endDate: Date; filters: object }, -) { - const { startDate, endDate, filters = {} } = criteria; +async function clickhouseQuery(websiteId: string, filters: QueryFilters) { const { rawQuery, getDateQuery, parseFilters } = clickhouse; - const website = await loadWebsite(websiteId); - const { filterQuery } = parseFilters(filters); + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); return rawQuery( ` @@ -92,13 +75,6 @@ async function clickhouseQuery( group by session_id, time_series ) as t; `, - { - ...filters, - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: EVENT_TYPE.pageView, - domain: website.domain, - }, + params, ); }