Merge branch 'dev' of https://github.com/umami-software/umami into fe…

…at/um-376-retention-report
vgtom · Aug 4, 2023 · 74b5b14 · 74b5b14
2 parents 8fff657 + 7148f66
commit 74b5b14
Show file tree

Hide file tree

Showing 17 changed files with 260 additions and 469 deletions.
diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts
@@ -2,8 +2,10 @@ import { ClickHouse } from 'clickhouse';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
-import { WebsiteMetricFilter } from './types';
-import { FILTER_COLUMNS } from './constants';
+import { QueryFilters } from './types';
+import { FILTER_COLUMNS, IGNORED_FILTERS } from './constants';
+import { loadWebsite } from './load';
+import { maxDate } from './date';
 
 export const CLICKHOUSE_DATE_FORMATS = {
   minute: '%Y-%m-%d %H:%M:00',
@@ -65,13 +67,13 @@ function getFilterQuery(filters = {}) {
   const query = Object.keys(filters).reduce((arr, key) => {
     const filter = filters[key];
 
-    if (filter !== undefined) {
+    if (filter !== undefined && !IGNORED_FILTERS.includes(key)) {
       const column = FILTER_COLUMNS[key] || key;
       arr.push(`and ${column} = {${key}:String}`);
     }
 
     if (key === 'referrer') {
-      arr.push('and referrer_domain != {domain:String}');
+      arr.push('and referrer_domain != {websiteDomain:String}');
     }
 
     return arr;
@@ -80,9 +82,20 @@ function getFilterQuery(filters = {}) {
   return query.join('\n');
 }
 
-function parseFilters(filters: WebsiteMetricFilter = {}) {
+async function parseFilters(
+  websiteId: string,
+  filters: QueryFilters & { [key: string]: any } = {},
+) {
+  const website = await loadWebsite(websiteId);
+
   return {
     filterQuery: getFilterQuery(filters),
+    params: {
+      ...filters,
+      websiteId,
+      startDate: maxDate(filters.startDate, website.resetAt),
+      websiteDomain: website.domain,
+    },
   };
 }
 

diff --git a/lib/constants.ts b/lib/constants.ts
@@ -50,8 +50,11 @@ export const FILTER_COLUMNS = {
   query: 'url_query',
   event: 'event_name',
   region: 'subdivision1',
+  type: 'event_type',
 };
 
+export const IGNORED_FILTERS = ['startDate', 'endDate', 'timezone', 'unit', 'eventType'];
+
 export const COLLECTION_TYPE = {
   event: 'event',
   identify: 'identify',

diff --git a/lib/prisma.ts b/lib/prisma.ts
@@ -1,7 +1,10 @@
 import prisma from '@umami/prisma-client';
 import moment from 'moment-timezone';
 import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
-import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants';
+import { FILTER_COLUMNS, IGNORED_FILTERS, SESSION_COLUMNS } from './constants';
+import { loadWebsite } from './load';
+import { maxDate } from './date';
+import { QueryFilters } from './types';
 
 const MYSQL_DATE_FORMATS = {
   minute: '%Y-%m-%d %H:%i:00',
@@ -68,14 +71,14 @@ function getFilterQuery(filters = {}): string {
   const query = Object.keys(filters).reduce((arr, key) => {
     const filter = filters[key];
 
-    if (filter !== undefined) {
+    if (filter !== undefined && !IGNORED_FILTERS.includes(key)) {
       const column = FILTER_COLUMNS[key] || key;
       arr.push(`and ${column}={{${key}}}`);
     }
 
     if (key === 'referrer') {
       arr.push(
-        'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)',
+        'and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',
       );
     }
 
@@ -85,12 +88,20 @@ function getFilterQuery(filters = {}): string {
   return query.join('\n');
 }
 
-function parseFilters(filters: { [key: string]: any } = {}) {
+async function parseFilters(websiteId, filters: QueryFilters & { [key: string]: any } = {}) {
+  const website = await loadWebsite(websiteId);
+
   return {
     joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key])
       ? `inner join session on website_event.session_id = session.session_id`
       : '',
     filterQuery: getFilterQuery(filters),
+    params: {
+      ...filters,
+      websiteId,
+      startDate: maxDate(filters.startDate, website.resetAt),
+      websiteDomain: website.domain,
+    },
   };
 }
 

diff --git a/lib/types.ts b/lib/types.ts
@@ -73,21 +73,6 @@ export interface WebsiteMetric {
   y: number;
 }
 
-export interface WebsiteMetricFilter {
-  domain?: string;
-  url?: string;
-  referrer?: string;
-  title?: string;
-  query?: string;
-  event?: string;
-  os?: string;
-  browser?: string;
-  device?: string;
-  country?: string;
-  region?: string;
-  city?: string;
-}
-
 export interface WebsiteEventMetric {
   x: string;
   t: string;
@@ -144,3 +129,24 @@ export interface DateRange {
   unit: string;
   value: string;
 }
+
+export interface QueryFilters {
+  startDate?: Date;
+  endDate?: Date;
+  timezone?: string;
+  unit?: string;
+  domain?: string;
+  eventType?: number;
+  url?: string;
+  referrer?: string;
+  title?: string;
+  query?: string;
+  event?: string;
+  os?: string;
+  browser?: string;
+  device?: string;
+  country?: string;
+  region?: string;
+  city?: string;
+  language?: string;
+}
diff --git a/pages/api/websites/[id]/metrics.ts b/pages/api/websites/[id]/metrics.ts
@@ -23,6 +23,7 @@ export interface WebsiteMetricsRequestQuery {
   country: string;
   region: string;
   city: string;
+  language: string;
 }
 
 export default async (
@@ -57,6 +58,8 @@ export default async (
     const { startDate, endDate } = await parseDateRangeQuery(req);
 
     const filters = {
+      startDate,
+      endDate,
       url,
       referrer,
       title,
@@ -76,12 +79,7 @@ export default async (
     const column = FILTER_COLUMNS[type] || type;
 
     if (SESSION_COLUMNS.includes(type)) {
-      const data = await getSessionMetrics(websiteId, {
-        startDate,
-        endDate,
-        column,
-        filters,
-      });
+      const data = await getSessionMetrics(websiteId, column, filters);
 
       if (type === 'language') {
         const combined = {};
@@ -103,12 +101,7 @@ export default async (
     }
 
     if (EVENT_COLUMNS.includes(type)) {
-      const data = await getPageviewMetrics(websiteId, {
-        startDate,
-        endDate,
-        column,
-        filters,
-      });
+      const data = await getPageviewMetrics(websiteId, column, filters);
 
       return ok(res, data);
     }

diff --git a/pages/api/websites/[id]/pageviews.ts b/pages/api/websites/[id]/pageviews.ts
@@ -57,41 +57,25 @@ export default async (
       return badRequest(res);
     }
 
+    const filters = {
+      startDate,
+      endDate,
+      timezone,
+      unit,
+      url,
+      referrer,
+      title,
+      os,
+      browser,
+      device,
+      country,
+      region,
+      city,
+    };
+
     const [pageviews, sessions] = await Promise.all([
-      getPageviewStats(websiteId, {
-        startDate,
-        endDate,
-        timezone,
-        unit,
-        filters: {
-          url,
-          referrer,
-          title,
-          os,
-          browser,
-          device,
-          country,
-          region,
-          city,
-        },
-      }),
-      getSessionStats(websiteId, {
-        startDate,
-        endDate,
-        timezone,
-        unit,
-        filters: {
-          url,
-          referrer,
-          title,
-          os,
-          browser,
-          device,
-          country,
-          region,
-          city,
-        },
-      }),
+      getPageviewStats(websiteId, filters),
+      getSessionStats(websiteId, filters),
     ]);
 
     return ok(res, { pageviews, sessions });

diff --git a/pages/api/websites/[id]/stats.ts b/pages/api/websites/[id]/stats.ts
@@ -56,40 +56,26 @@ export default async (
     const prevStartDate = subMinutes(startDate, diff);
     const prevEndDate = subMinutes(endDate, diff);
 
-    const metrics = await getWebsiteStats(websiteId, {
-      startDate,
-      endDate,
-      filters: {
-        url,
-        referrer,
-        title,
-        query,
-        event,
-        os,
-        browser,
-        device,
-        country,
-        region,
-        city,
-      },
-    });
+    const filters = {
+      url,
+      referrer,
+      title,
+      query,
+      event,
+      os,
+      browser,
+      device,
+      country,
+      region,
+      city,
+    };
+
+    const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate });
 
     const prevPeriod = await getWebsiteStats(websiteId, {
+      ...filters,
       startDate: prevStartDate,
       endDate: prevEndDate,
-      filters: {
-        url,
-        referrer,
-        title,
-        query,
-        event,
-        os,
-        browser,
-        device,
-        country,
-        region,
-        city,
-      },
     });
 
     const stats = Object.keys(metrics[0]).reduce((obj, key) => {