From 941f5c2cd0e89e91f6e07b848a86dc6da28a3166 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Fri, 1 May 2026 14:48:14 +0100 Subject: [PATCH 01/22] Unify analytics tables into a single 'events' table --- backend/apps/cloud/src/ai/ai.service.ts | 103 ++- .../src/analytics/analytics.controller.ts | 74 +- .../cloud/src/analytics/analytics.service.ts | 841 +++++++----------- .../cloud/src/analytics/utils/transformers.ts | 444 ++++----- .../apps/cloud/src/captcha/captcha.service.ts | 17 +- .../cloud/src/captcha/utils/transformers.ts | 22 - .../src/data-import/data-import.processor.ts | 44 +- .../src/data-import/data-import.service.ts | 8 +- .../src/data-import/mappers/fathom.mapper.ts | 6 +- .../src/data-import/mappers/ga4.mapper.ts | 6 +- .../data-import/mappers/mapper.interface.ts | 2 +- .../data-import/mappers/plausible.mapper.ts | 6 +- .../mappers/simple-analytics.mapper.ts | 6 +- .../src/data-import/mappers/umami.mapper.ts | 6 +- .../src/experiment/experiment.controller.ts | 14 +- .../apps/cloud/src/goal/goal.controller.ts | 63 +- .../cloud/src/project/project.controller.ts | 12 +- .../apps/cloud/src/project/project.service.ts | 233 ++--- .../src/task-manager/task-manager.service.ts | 75 +- .../src/analytics/analytics.controller.ts | 74 +- .../src/analytics/analytics.service.ts | 778 +++++++--------- .../src/analytics/utils/transformers.ts | 444 ++++----- .../community/src/captcha/captcha.service.ts | 17 +- .../src/captcha/utils/transformers.ts | 22 - .../src/data-import/data-import.processor.ts | 44 +- .../src/data-import/data-import.service.ts | 8 +- .../src/data-import/mappers/fathom.mapper.ts | 6 +- .../data-import/mappers/mapper.interface.ts | 2 +- .../data-import/mappers/plausible.mapper.ts | 6 +- .../mappers/simple-analytics.mapper.ts | 6 +- .../src/data-import/mappers/umami.mapper.ts | 6 +- .../community/src/goal/goal.controller.ts | 63 +- .../src/project/project.controller.ts | 12 +- .../community/src/project/project.service.ts | 81 +- .../community/src/user/user.controller.ts | 5 +- .../meta/clickhouse/generate-dummy-data.js | 18 +- .../clickhouse/2026_05_01_unify_events.js | 94 ++ .../clickhouse/initialise_database.js | 127 +-- 38 files changed, 1427 insertions(+), 2368 deletions(-) delete mode 100644 backend/apps/cloud/src/captcha/utils/transformers.ts delete mode 100644 backend/apps/community/src/captcha/utils/transformers.ts create mode 100644 backend/migrations/clickhouse/2026_05_01_unify_events.js diff --git a/backend/apps/cloud/src/ai/ai.service.ts b/backend/apps/cloud/src/ai/ai.service.ts index 8671e63fd..35aa6114e 100644 --- a/backend/apps/cloud/src/ai/ai.service.ts +++ b/backend/apps/cloud/src/ai/ai.service.ts @@ -1218,8 +1218,9 @@ Filter modifiers: SELECT count(*) as pageviews, count(DISTINCT psid) as sessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1242,8 +1243,9 @@ Filter modifiers: ${this.getTimeBucketSelect(timeBucket, timezone)} as date, count(*) as pageviews, count(DISTINCT psid) as sessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -1266,8 +1268,9 @@ Filter modifiers: // Get top pages const pagesQuery = ` SELECT pg as name, count(*) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY pg @@ -1285,8 +1288,9 @@ Filter modifiers: // Get top countries const countriesQuery = ` SELECT cc as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND cc IS NOT NULL AND cc != '' ${filterConditions.where} @@ -1305,8 +1309,9 @@ Filter modifiers: // Get top referrers const referrersQuery = ` SELECT ref as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND ref IS NOT NULL AND ref != '' ${filterConditions.where} @@ -1325,8 +1330,9 @@ Filter modifiers: // Get browsers const browsersQuery = ` SELECT br as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND br IS NOT NULL AND br != '' ${filterConditions.where} @@ -1345,8 +1351,9 @@ Filter modifiers: // Get devices const devicesQuery = ` SELECT dv as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND dv IS NOT NULL AND dv != '' ${filterConditions.where} @@ -1416,8 +1423,9 @@ Filter modifiers: round(${measureFn}(render) / 1000, 2) as render, round(${measureFn}(domLoad) / 1000, 2) as domLoad, round(${measureFn}(ttfb) / 1000, 2) as ttfb - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1440,8 +1448,9 @@ Filter modifiers: ${this.getTimeBucketSelect(timeBucket, timezone)} as date, round(${measureFn}(pageLoad) / 1000, 2) as pageLoad, round(${measureFn}(ttfb) / 1000, 2) as ttfb - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -1497,8 +1506,9 @@ Filter modifiers: SELECT count(*) as totalErrors, uniqExact(eid) as uniqueErrors - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1512,15 +1522,16 @@ Filter modifiers: const topErrorsQuery = ` SELECT - name, - message, + error_name AS name, + error_message AS message, count(*) as count, max(created) as lastSeen - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} - GROUP BY name, message + GROUP BY error_name, error_message ORDER BY count DESC LIMIT 10 ` @@ -1571,22 +1582,24 @@ Filter modifiers: } // Query specific goal stats - const table = goal.type === 'custom_event' ? 'customEV' : 'analytics' + const goalType = + goal.type === 'custom_event' ? 'custom_event' : 'pageview' const matchCondition = goal.matchType === 'exact' ? goal.type === 'custom_event' - ? `ev = {goalValue:String}` + ? `event_name = {goalValue:String}` : `pg = {goalValue:String}` : goal.type === 'custom_event' - ? `ev ILIKE concat('%', {goalValue:String}, '%')` + ? `event_name ILIKE concat('%', {goalValue:String}, '%')` : `pg ILIKE concat('%', {goalValue:String}, '%')` const query = ` SELECT count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${goalType}' AND ${matchCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -1617,20 +1630,22 @@ Filter modifiers: const results = await Promise.all( goals.map(async (goal) => { - const table = goal.type === 'custom_event' ? 'customEV' : 'analytics' + const goalType = + goal.type === 'custom_event' ? 'custom_event' : 'pageview' const matchCondition = goal.matchType === 'exact' ? goal.type === 'custom_event' - ? `ev = {goalValue:String}` + ? `event_name = {goalValue:String}` : `pg = {goalValue:String}` : goal.type === 'custom_event' - ? `ev ILIKE concat('%', {goalValue:String}, '%')` + ? `event_name ILIKE concat('%', {goalValue:String}, '%')` : `pg ILIKE concat('%', {goalValue:String}, '%')` const query = ` SELECT count(*) as conversions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${goalType}' AND ${matchCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -1747,8 +1762,9 @@ Filter modifiers: const overallQuery = ` SELECT count(*) as total - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1757,8 +1773,9 @@ Filter modifiers: SELECT ${this.getTimeBucketSelect(timeBucket, timezone)} as date, count(*) as challenges - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -1767,8 +1784,9 @@ Filter modifiers: const breakdownQuery = (column: string) => ` SELECT ${column} as name, count(*) as count - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND ${column} IS NOT NULL AND ${column} != '' ${filterConditions.where} @@ -1844,21 +1862,23 @@ Filter modifiers: const overallQuery = ` SELECT count(*) as totalEvents, - uniqExact(ev) as uniqueEvents, + uniqExact(event_name) as uniqueEvents, uniqExact(psid) as sessions - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` const topEventsQuery = ` - SELECT ev as name, count(*) as count, uniqExact(psid) as sessions - FROM customEV + SELECT event_name AS name, count(*) as count, uniqExact(psid) as sessions + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} - GROUP BY ev + GROUP BY event_name ORDER BY count DESC LIMIT 25 ` @@ -1868,8 +1888,9 @@ Filter modifiers: ${this.getTimeBucketSelect(timeBucket, timezone)} as date, count(*) as events, uniqExact(psid) as sessions - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -2137,10 +2158,10 @@ Filter modifiers: let conversions: { variantKey: string; conversions: number }[] = [] if (experiment.goal) { - const table = - experiment.goal.type === 'custom_event' ? 'customEV' : 'analytics' + const eventType = + experiment.goal.type === 'custom_event' ? 'custom_event' : 'pageview' const matchColumn = - experiment.goal.type === 'custom_event' ? 'ev' : 'pg' + experiment.goal.type === 'custom_event' ? 'event_name' : 'pg' const matchCondition = experiment.goal.matchType === 'exact' ? `c.${matchColumn} = {goalValue:String}` @@ -2149,7 +2170,7 @@ Filter modifiers: const conversionsQuery = ` SELECT e.variantKey, uniqExact(e.profileId) as conversions FROM experiment_exposures e - INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) + INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}' WHERE e.pid = {pid:FixedString(12)} AND e.experimentId = {experimentId:String} AND e.created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2291,8 +2312,9 @@ Filter modifiers: min(created) as startedAt, max(created) as endedAt, count(*) as pageviews - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IS NOT NULL ${extraWhere.join(' ')} @@ -2356,8 +2378,9 @@ Filter modifiers: uniqExactIf(profileId, profileId LIKE 'usr_%') as identifiedProfiles, uniqExact(psid) as sessions, count(*) as pageviews - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL ` @@ -2368,8 +2391,9 @@ Filter modifiers: uniqExact(psid) as sessions, count(*) as pageviews, max(created) as lastSeen - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL GROUP BY profileId @@ -2379,8 +2403,9 @@ Filter modifiers: const topPagesQuery = ` SELECT pg as name, uniqExact(profileId) as profiles - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND pg IS NOT NULL AND pg != '' diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index 8fa6767c0..a64f0d2e6 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -81,12 +81,7 @@ import { GetErrorOverviewOptions, } from './dto/get-error-overview.dto' import { PatchStatusDto } from './dto/patch-status.dto' -import { - customEventTransformer, - errorEventTransformer, - performanceTransformer, - trafficTransformer, -} from './utils/transformers' +import { eventTransformer } from './utils/transformers' import { enrichTrafficSource } from './utils/clickIdSources' import { MAX_METRICS_IN_VIEW, @@ -294,12 +289,12 @@ export class AnalyticsController { diff, ) - let subQuery = `FROM ${ - isCaptcha ? 'captcha' : 'analytics' - } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + let subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${ + isCaptcha ? 'captcha' : 'pageview' + }' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` if (customEVFilterApplied && !isCaptcha) { - subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` } const paramsData = { @@ -804,7 +799,7 @@ export class AnalyticsController { diff, ) - const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } @@ -1332,32 +1327,12 @@ export class AnalyticsController { any(os) AS os, any(cc) AS cc, toString(psid) AS psid - FROM - ( - SELECT - psid, - dv, - br, - os, - cc - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT - psid, - dv, - br, - os, - cc - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL GROUP BY psid ` @@ -1464,7 +1439,8 @@ export class AnalyticsController { const { name, message, lineno, colno, filename, stackTrace, meta } = errorDTO - const transformed = errorEventTransformer({ + const transformed = eventTransformer({ + type: 'error', psid, profileId, eid: this.analyticsService.getErrorID(errorDTO), @@ -1496,7 +1472,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'errors', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1618,7 +1594,8 @@ export class AnalyticsController { enrichTrafficSource(eventsDTO) - const transformed = customEventTransformer({ + const transformed = eventTransformer({ + type: 'custom_event', psid, profileId, pid: eventsDTO.pid, @@ -1650,7 +1627,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'customEV', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1789,7 +1766,8 @@ export class AnalyticsController { enrichTrafficSource(logDTO) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1832,7 +1810,8 @@ export class AnalyticsController { ttfb, } = logDTO.perf - perfTransformed = performanceTransformer({ + perfTransformed = eventTransformer({ + type: 'performance', pid: logDTO.pid, host: this.analyticsService.getHostFromOrigin(headers.origin), pg: logDTO.pg, @@ -1860,7 +1839,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1868,7 +1847,7 @@ export class AnalyticsController { if (!_isEmpty(perfTransformed)) { await clickhouse.insert({ - table: 'performance', + table: 'events', format: 'JSONEachRow', values: [perfTransformed], clickhouse_settings: { async_insert: 1 }, @@ -1955,7 +1934,8 @@ export class AnalyticsController { const { deviceType, browserName, browserVersion, osName, osVersion } = await this.analyticsService.getRequestInformation(headers) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1986,7 +1966,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 80c4a8b6a..70e85e154 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -857,9 +857,10 @@ export class AnalyticsService { pg, created, lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND pg IS NOT NULL ${filtersQuery} @@ -975,10 +976,17 @@ export class AnalyticsService { timeBucket: TimeBucketType[] diff: number }> { + const tableToType: Record = { + analytics: 'pageview', + customEV: 'custom_event', + performance: 'performance', + errors: 'error', + } + const { data: fromData } = await clickhouse .query({ - query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`, - query_params: { pid }, + query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type = {type:String}`, + query_params: { pid, type: tableToType[table] }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -1212,8 +1220,8 @@ export class AnalyticsService { ? 'argMin(pg, created)' : 'argMax(pg, created)' const subQueryForPages = isContains - ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` - : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}` + ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` + : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page = {${param}:String}` // For exclusive filter (isNot) we exclude sessions with matching entry/exit page query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})` @@ -1254,6 +1262,9 @@ export class AnalyticsService { ) { sqlColumn = 'meta.value' isArrayDataset = true + } else if (column === 'ev') { + // Public filter contract `ev` maps to the renamed `event_name` column + sqlColumn = 'event_name' } const isNullFilter = @@ -1631,21 +1642,11 @@ export class AnalyticsService { FROM ( SELECT psid, - pg AS value, - created - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - - UNION ALL - - SELECT - psid, - ev AS value, + if(type = 'pageview', pg, event_name) AS value, created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -1693,14 +1694,14 @@ export class AnalyticsService { psid, CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1715,17 +1716,11 @@ export class AnalyticsService { psid, argMin(cc, created) AS cc, argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source - FROM ( - SELECT psid, cc, ref, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, cc, ref, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 + AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid ) SELECT step, type, val, cnt FROM ( @@ -1803,14 +1798,14 @@ export class AnalyticsService { psid, windowFunnel(86400)(created, ${pagesStr}) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1826,19 +1821,12 @@ export class AnalyticsService { any(br) AS br, min(toTimeZone(created, {timezone:String})) AS sessionStart, max(toTimeZone(created, {timezone:String})) AS lastActivity - FROM ( - SELECT psid, pid, cc, os, br, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND psid IN (SELECT psid FROM funnel_qualified) - UNION ALL - SELECT psid, pid, cc, os, br, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) - ) GROUP BY psidCasted, pid ), pageview_counts AS ( @@ -1846,8 +1834,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1857,8 +1845,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1868,8 +1856,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1947,8 +1935,9 @@ export class AnalyticsService { const query = ` SELECT count() as c - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -2047,9 +2036,10 @@ export class AnalyticsService { SELECT cc, count() as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND cc IS NOT NULL AND cc != '' @@ -2083,9 +2073,10 @@ export class AnalyticsService { pid, cc, count() as cnt - FROM analytics + FROM events WHERE pid IN {pids:Array(FixedString(12))} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND cc IS NOT NULL AND cc != '' @@ -2131,9 +2122,10 @@ export class AnalyticsService { SELECT count() as count, count(DISTINCT eid) as uniqueErrors - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -2163,9 +2155,10 @@ export class AnalyticsService { pid, count() as count, count(DISTINCT eid) as uniqueErrors - FROM errors + FROM events WHERE pid IN {pids:Array(FixedString(12))} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid ` @@ -2210,9 +2203,10 @@ export class AnalyticsService { SELECT pid, uniqExact(psid) as totalSessions - FROM analytics + FROM events WHERE pid IN {pids:Array(FixedString(12))} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid ` @@ -2277,15 +2271,20 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - let queryAll = ` + const allTimeType = customEVFilterApplied + ? 'custom_event' + : 'pageview' + + const queryAll = ` WITH analytics_counts AS ( SELECT count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' ${filtersQuery} ), duration_avg AS ( @@ -2305,36 +2304,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryAll = ` - WITH analytics_counts AS ( - SELECT - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const { data } = await clickhouse .query({ query: queryAll, @@ -2419,16 +2388,19 @@ export class AnalyticsService { ) .format('YYYY-MM-DD HH:mm:ss') - let queryCurrent = ` + const periodType = customEVFilterApplied ? 'custom_event' : 'pageview' + + const queryCurrent = ` WITH analytics_counts AS ( SELECT 1 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} ), @@ -2442,8 +2414,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} @@ -2457,16 +2430,17 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - let queryPrevious = ` + const queryPrevious = ` WITH analytics_counts AS ( SELECT 2 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} ), @@ -2480,8 +2454,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} @@ -2495,83 +2470,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryCurrent = ` - WITH analytics_counts AS ( - SELECT - 1 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - queryPrevious = ` - WITH analytics_counts AS ( - SELECT - 2 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const query = `${queryCurrent} UNION ALL ${queryPrevious}` let { data } = await clickhouse @@ -2779,7 +2677,7 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}` + const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}` const { data } = await clickhouse .query({ @@ -2842,8 +2740,8 @@ export class AnalyticsService { .format('YYYY-MM-DD HH:mm:ss') } - const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` - const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` + const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` + const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` const query = `${queryCurrent} UNION ALL ${queryPrevious}` @@ -2940,8 +2838,8 @@ export class AnalyticsService { const query = ` WITH session_pages AS ( SELECT psid, ${selector} as page - FROM analytics - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid ) SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page @@ -2957,11 +2855,11 @@ export class AnalyticsService { return _map(data, 'page') } - let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` - - if (type === 'ev') { - query = `SELECT ${type} FROM customEV WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` - } + // Public filter contract `ev` reads from the renamed event_name column. + let query = + type === 'ev' + ? `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name` + : `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND ${type} IS NOT NULL GROUP BY ${type}` const { data } = await clickhouse .query({ @@ -2982,7 +2880,15 @@ export class AnalyticsService { ) } - const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` + // Public ERROR_COLUMNS use legacy short names; map to new column names. + const errorColMap: Record = { + name: 'error_name', + message: 'error_message', + filename: 'error_filename', + } + const sqlCol = errorColMap[type] || type + + const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}` const { data } = await clickhouse .query({ @@ -3001,13 +2907,13 @@ export class AnalyticsService { type: 'traffic' | 'errors', column: 'br' | 'os', ): Promise> { - const safeTable = type === 'errors' ? 'errors' : 'analytics' + const safeType = type === 'errors' ? 'error' : 'pageview' const safeVersionCol = column === 'br' ? 'brv' : 'osv' const query = ` SELECT ${column}, ${safeVersionCol} - FROM ${safeTable} - WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL + FROM events + WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL GROUP BY ${column}, ${safeVersionCol} ` @@ -3385,8 +3291,8 @@ export class AnalyticsService { pid, psid, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3434,8 +3340,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3462,8 +3368,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3488,8 +3394,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'error' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3515,8 +3421,9 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3559,9 +3466,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3594,9 +3502,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3622,9 +3531,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3661,9 +3571,10 @@ export class AnalyticsService { FROM ( SELECT pg, dv, br, os, lc, cc, rg, ct, profileId, ${timeBucketFunc}(created) as tz_created - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) as subquery @@ -4431,7 +4342,7 @@ export class AnalyticsService { filtersQuery: string, params: any, ): Promise { - const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` + const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name` const result = {} const { data } = await clickhouse @@ -4461,10 +4372,10 @@ export class AnalyticsService { FROM ( SELECT meta.key AS key - FROM - analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -4546,12 +4457,12 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND ev = {event:String} + AND event_name = {event:String} ${filtersQuery} ) ARRAY JOIN meta.key, meta.value @@ -4647,10 +4558,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND indexOf(meta.key, {property:String}) > 0 ${filtersQuery} @@ -4700,17 +4611,11 @@ export class AnalyticsService { const query = ` SELECT uniqExact(psid) as count - FROM ( - SELECT psid FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT psid FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL ` try { @@ -4748,25 +4653,26 @@ export class AnalyticsService { const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})` const customEventsFilter = customEvents && customEvents.length > 0 - ? 'AND ev IN {customEvents:Array(String)}' + ? 'AND event_name IN {customEvents:Array(String)}' : '' const query = ` SELECT ${selector}, - ev, + event_name AS ev, count() as count FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ${customEventsFilter} ) as subquery - GROUP BY ${groupBy}, ev + GROUP BY ${groupBy}, event_name ORDER BY ${groupBy} ` @@ -4864,52 +4770,31 @@ export class AnalyticsService { const queryPages = ` WITH events_with_meta AS ( SELECT - 'pageview' AS type, - toString(pg) AS value, - toTimeZone(analytics.created, {timezone:String}) AS created, - pid, - toString(analytics.psid) AS psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND toString(analytics.psid) = {psid:String} - - UNION ALL - - SELECT - 'event' AS type, - toString(ev) AS value, - toTimeZone(customEV.created, {timezone:String}) AS created, - pid, - toString(customEV.psid) AS psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND toString(customEV.psid) = {psid:String} - - UNION ALL - - SELECT - 'error' AS type, - toString(errors.name) AS value, - toTimeZone(errors.created, {timezone:String}) AS created, + multiIf(type = 'pageview', 'pageview', type = 'custom_event', 'event', 'error') AS type, + multiIf( + type = 'pageview', toString(pg), + type = 'custom_event', toString(event_name), + toString(error_name) + ) AS value, + toTimeZone(created, {timezone:String}) AS created, pid, - toString(errors.psid) AS psid, - [ - tuple('message', COALESCE(errors.message, '')), - tuple('lineno', toString(COALESCE(errors.lineno, 0))), - tuple('colno', toString(COALESCE(errors.colno, 0))), - tuple('filename', COALESCE(errors.filename, '')) - ] AS metadata - FROM errors + toString(psid) AS psid, + if( + type = 'error', + [ + tuple('message', COALESCE(error_message, '')), + tuple('lineno', toString(COALESCE(lineno, 0))), + tuple('colno', toString(COALESCE(colno, 0))), + tuple('filename', COALESCE(error_filename, '')) + ], + arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) + ) AS metadata + FROM events WHERE pid = {pid:FixedString(12)} - AND errors.psid IS NOT NULL - AND toString(errors.psid) = {psid:String} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} UNION ALL @@ -4961,9 +4846,10 @@ export class AnalyticsService { const querySessionDetails = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC @@ -5057,9 +4943,10 @@ export class AnalyticsService { const querySessionDetailsFromCustomEV = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC @@ -5152,82 +5039,49 @@ export class AnalyticsService { let primaryEventsSubquery: string if (customEVFilterApplied) { - // When filtering by custom events, we need to: - // 1. Identify sessions (psids) that have matching custom events - // 2. Calculate session boundaries from ALL analytics events for those sessions + // When filtering by custom events, identify sessions (psids) that have + // matching custom events, then calculate session boundaries from ALL + // pageview/custom_event rows for those sessions. primaryEventsSubquery = ` SELECT - all_events.psidCasted, - all_events.pid, - all_events.cc, - all_events.os, - all_events.br, - all_events.created_for_grouping - FROM ( - SELECT - CAST(analytics.psid, 'String') AS psidCasted, - analytics.pid, - analytics.cc, - analytics.os, - analytics.br, - toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping - FROM analytics - WHERE - analytics.pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ) AS all_events - WHERE all_events.psidCasted IN ( - SELECT DISTINCT CAST(customEV.psid, 'String') - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) ` } else { primaryEventsSubquery = ` SELECT - CAST(analytics.psid, 'String') AS psidCasted, - analytics.pid, - analytics.cc, - analytics.os, - analytics.br, - toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping - FROM analytics - WHERE - analytics.pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ` } @@ -5250,8 +5104,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -5260,8 +5114,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -5270,8 +5124,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -5390,42 +5244,26 @@ export class AnalyticsService { br, dv, created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics + if(type = 'pageview', 1, 0) AS isPageview, + if(type = 'custom_event', 1, 0) AS isEvent + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} AND profileId IN ( SELECT DISTINCT profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} ${filtersQuery} ) - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} )` } else { allProfileDataCTE = ` @@ -5438,28 +5276,11 @@ export class AnalyticsService { br, dv, created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV + if(type = 'pageview', 1, 0) AS isPageview, + if(type = 'custom_event', 1, 0) AS isEvent + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' @@ -5489,9 +5310,10 @@ export class AnalyticsService { SELECT profileId, count() AS errorsCount - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} + AND type = 'error' + AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} @@ -5547,7 +5369,7 @@ export class AnalyticsService { AND profileId = {profileId:String} ` - // Query avg duration from analytics table (more accurate than sessions table) + // Query avg duration from pageview events (more accurate than sessions table) const queryAvgDuration = ` SELECT avg(session_duration) AS avgDuration @@ -5555,8 +5377,9 @@ export class AnalyticsService { SELECT psid, dateDiff('second', min(created), max(created)) AS session_duration - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND psid IS NOT NULL GROUP BY psid @@ -5564,23 +5387,22 @@ export class AnalyticsService { ) ` - // Query analytics for accurate pageview count const queryPageviews = ` SELECT count() AS pageviewsCount - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} ` - // Query customEV for accurate events count const queryEvents = ` SELECT count() AS eventsCount - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} ` - // Query for device/location details const queryDetails = ` SELECT any(cc) AS cc, @@ -5592,8 +5414,9 @@ export class AnalyticsService { any(brv) AS brv, any(dv) AS dv, any(lc) AS lc - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} ` @@ -5679,8 +5502,9 @@ export class AnalyticsService { SELECT pg AS page, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} GROUP BY pg ORDER BY count DESC @@ -5709,30 +5533,15 @@ export class AnalyticsService { .format('YYYY-MM-DD') const query = ` - WITH all_activity AS ( - SELECT - toDate(created) AS date, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - UNION ALL - SELECT - toDate(created) AS date, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - ) SELECT - date, - sum(isPageview) AS pageviews, - sum(isEvent) AS events - FROM all_activity + toDate(created) AS date, + countIf(type = 'pageview') AS pageviews, + countIf(type = 'custom_event') AS events + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND profileId = {profileId:String} + AND created >= {startDate:Date} GROUP BY date ORDER BY date ASC ` @@ -5767,8 +5576,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5779,8 +5589,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5791,8 +5602,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5866,56 +5678,6 @@ export class AnalyticsService { let allProfileEventsCTE: string if (customEVFilterApplied) { - allProfileEventsCTE = ` - all_profile_events AS ( - SELECT - all_events.psidCasted, - all_events.pid, - all_events.profileId, - all_events.cc, - all_events.os, - all_events.br, - all_events.created_tz - FROM ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) AS all_events - WHERE all_events.psidCasted IN ( - SELECT DISTINCT CAST(psid, 'String') - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - )` - } else { allProfileEventsCTE = ` all_profile_events AS ( SELECT @@ -5926,13 +5688,26 @@ export class AnalyticsService { os, br, toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND profileId = {profileId:String} + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + )` + } else { + allProfileEventsCTE = ` + all_profile_events AS ( SELECT CAST(psid, 'String') AS psidCasted, pid, @@ -5941,8 +5716,9 @@ export class AnalyticsService { os, br, toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -5970,8 +5746,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -5982,8 +5759,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -5994,8 +5772,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6081,10 +5860,18 @@ export class AnalyticsService { count(DISTINCT psid) as sessions, status.status FROM ( - SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created - FROM errors + SELECT + eid, + error_name AS name, + error_message AS message, + error_filename AS filename, + psid, + profileId, + toTimeZone(created, {timezone:String}) AS created + FROM events WHERE pid = {pid:FixedString(12)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} + AND type = 'error' + AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) AS errors LEFT JOIN ( @@ -6143,14 +5930,15 @@ export class AnalyticsService { FROM ( SELECT eid, - name, - message, - filename, + error_name AS name, + error_message AS message, + error_filename AS filename, colno, lineno, stackTrace - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ORDER BY created DESC @@ -6173,9 +5961,10 @@ export class AnalyticsService { max(created) AS last_seen, min(created) AS first_seen, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)}; ` @@ -6188,9 +5977,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -6237,7 +6027,7 @@ export class AnalyticsService { timeBucket, groupFrom, groupTo, - `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, + `FROM events WHERE type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, 'AND eid = {eid:FixedString(32)}', paramsData, safeTimezone, @@ -6269,11 +6059,12 @@ export class AnalyticsService { ? '' : "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)" - // Get total sessions from analytics table for the time range + // Get total sessions from pageview events for the time range const queryTotalSessions = ` SELECT count(DISTINCT psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -6284,7 +6075,7 @@ export class AnalyticsService { count(DISTINCT eid) as uniqueErrors, count(DISTINCT psid) as affectedSessions, count(DISTINCT profileId) as affectedUsers - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -6292,6 +6083,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -6301,12 +6093,12 @@ export class AnalyticsService { const queryMostFrequentError = ` SELECT errors.eid as eid, - any(errors.name) as name, - any(errors.message) as message, + any(errors.error_name) as name, + any(errors.error_message) as message, count(*) as count, count(DISTINCT errors.profileId) as usersAffected, max(errors.created) as lastSeen - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -6314,6 +6106,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -6338,7 +6131,7 @@ export class AnalyticsService { SELECT profileId, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -6346,6 +6139,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -6457,8 +6251,9 @@ export class AnalyticsService { ): Promise<{ sessions: any[]; total: number }> { const queryCount = ` SELECT count(DISTINCT psid) as total - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -6473,9 +6268,13 @@ export class AnalyticsService { min(errors.created) as firstErrorAt, max(errors.created) as lastErrorAt, count(*) as errorCount - FROM errors - LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid + FROM events AS errors + LEFT JOIN events AS analytics + ON errors.psid = analytics.psid + AND errors.pid = analytics.pid + AND analytics.type = 'pageview' WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.eid = {eid:FixedString(32)} AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY errors.psid @@ -6522,7 +6321,7 @@ export class AnalyticsService { {}, ) - const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map( + const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map( params, (val, key) => `{${key}:FixedString(32)}`, ).join(', ')})` @@ -6604,15 +6403,7 @@ export class AnalyticsService { events: number }> { const query = ` - SELECT 'traffic' AS type, count(*) AS count FROM analytics - UNION ALL - SELECT 'customEV' AS type, count(*) AS count FROM customEV - UNION ALL - SELECT 'performance' AS type, count(*) AS count FROM performance - UNION ALL - SELECT 'captcha' AS type, count(*) AS count FROM captcha - UNION ALL - SELECT 'errors' AS type, count(*) AS count FROM errors + SELECT count(*) AS count FROM events ` const users = await this.userService.count({ @@ -6732,12 +6523,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodFormatted:String} AND {now:String} ${kvQuery} ${filtersQuery} @@ -6750,12 +6542,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${kvQuery} ${filtersQuery} diff --git a/backend/apps/cloud/src/analytics/utils/transformers.ts b/backend/apps/cloud/src/analytics/utils/transformers.ts index b420cdac8..687fb0c3e 100644 --- a/backend/apps/cloud/src/analytics/utils/transformers.ts +++ b/backend/apps/cloud/src/analytics/utils/transformers.ts @@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc' dayjs.extend(utc) const processMetaKV = ( - rawMeta: Record, + rawMeta: Record | null | undefined, ): { 'meta.key': string[] | null; 'meta.value': string[] | null } => { if (!rawMeta || _isEmpty(rawMeta)) { return { @@ -23,10 +23,10 @@ const processMetaKV = ( } } -interface TrafficTransformerOptions { - psid: string - profileId: string +interface CommonOptions { pid: string + psid?: string | null + profileId?: string | null host?: string | null pg?: string | null dv?: string | null @@ -52,262 +52,42 @@ interface TrafficTransformerOptions { meta?: Record | null } -export const trafficTransformer = ({ - psid, - profileId, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: TrafficTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } +interface PageviewOptions extends CommonOptions { + type: 'pageview' } -interface CustomEventTransformerOptions { - psid: string - profileId: string - pid: string - host?: string | null +interface CustomEventOptions extends CommonOptions { + type: 'custom_event' ev: string - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - ref?: string | null - so?: string | null - me?: string | null - ca?: string | null - te?: string | null - co?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null - meta?: Record | null -} - -export const customEventTransformer = ({ - psid, - profileId, - pid, - host, - ev, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: CustomEventTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - ev, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface ErrorEventTransformerOptions { - psid: string - profileId: string +interface ErrorEventOptions extends CommonOptions { + type: 'error' eid: string - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null name?: string | null message?: string | null lineno?: number | null colno?: number | null filename?: string | null stackTrace?: string | null - meta?: Record | null -} - -export const errorEventTransformer = ({ - psid, - profileId, - eid, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - name, - message, - lineno, - colno, - filename, - stackTrace, - meta, -}: ErrorEventTransformerOptions) => { - return { - psid, - profileId, - eid, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - name: name || null, - message: message || null, - lineno: lineno || null, - colno: colno || null, - filename: filename || null, - stackTrace: stackTrace || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface PerformanceTransformerOptions { - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null +interface PerformanceOptions extends Omit< + CommonOptions, + | 'psid' + | 'profileId' + | 'os' + | 'osv' + | 'lc' + | 'ref' + | 'so' + | 'me' + | 'ca' + | 'te' + | 'co' + | 'meta' +> { + type: 'performance' dns: number tls: number conn: number @@ -318,53 +98,131 @@ interface PerformanceTransformerOptions { ttfb: number } -export const performanceTransformer = ({ - pid, - host, - pg, - dv, - br, - brv, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - dns, - tls, - conn, - response, - render, - domLoad, - pageLoad, - ttfb, -}: PerformanceTransformerOptions) => { +interface CaptchaOptions { + type: 'captcha' + pid: string + dv?: string | null + br?: string | null + os?: string | null + cc?: string | null + timestamp?: number +} + +type EventTransformerOptions = + | PageviewOptions + | CustomEventOptions + | ErrorEventOptions + | PerformanceOptions + | CaptchaOptions + +const buildCommon = (opts: CommonOptions) => ({ + pid: opts.pid, + psid: opts.psid ?? null, + profileId: opts.profileId ?? null, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + os: opts.os || null, + osv: opts.osv || null, + lc: opts.lc || null, + ref: opts.ref || null, + so: opts.so || null, + me: opts.me || null, + ca: opts.ca || null, + te: opts.te || null, + co: opts.co || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + ...processMetaKV(opts.meta), +}) + +// Single transformer for all event kinds; the discriminator is `type`. +// Renames at the storage boundary: ev -> event_name, name -> error_name, +// message -> error_message, filename -> error_filename. DTO and API field +// names stay as the legacy short forms. +export const eventTransformer = (opts: EventTransformerOptions) => { + const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss') + + if (opts.type === 'pageview') { + return { + type: 'pageview' as const, + ...buildCommon(opts), + created, + } + } + + if (opts.type === 'custom_event') { + return { + type: 'custom_event' as const, + ...buildCommon(opts), + event_name: opts.ev, + created, + } + } + + if (opts.type === 'error') { + return { + type: 'error' as const, + ...buildCommon(opts), + eid: opts.eid, + error_name: opts.name || null, + error_message: opts.message || null, + stackTrace: opts.stackTrace || null, + lineno: opts.lineno || null, + colno: opts.colno || null, + error_filename: opts.filename || null, + created, + } + } + + if (opts.type === 'performance') { + return { + type: 'performance' as const, + pid: opts.pid, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + dns: _round(opts.dns), + tls: _round(opts.tls), + conn: _round(opts.conn), + response: _round(opts.response), + render: _round(opts.render), + domLoad: _round(opts.domLoad), + pageLoad: _round(opts.pageLoad), + ttfb: _round(opts.ttfb), + created, + } + } + + // captcha return { - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - dns: _round(dns), - tls: _round(tls), - conn: _round(conn), - response: _round(response), - render: _round(render), - domLoad: _round(domLoad), - pageLoad: _round(pageLoad), - ttfb: _round(ttfb), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), + type: 'captcha' as const, + pid: opts.pid, + dv: opts.dv || null, + br: opts.br || null, + os: opts.os || null, + cc: opts.cc || null, + created: + typeof opts.timestamp === 'number' + ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss') + : created, } } diff --git a/backend/apps/cloud/src/captcha/captcha.service.ts b/backend/apps/cloud/src/captcha/captcha.service.ts index 921e6d7b7..99ac3516c 100644 --- a/backend/apps/cloud/src/captcha/captcha.service.ts +++ b/backend/apps/cloud/src/captcha/captcha.service.ts @@ -17,7 +17,7 @@ import { } from '../common/constants' import { getIPDetails } from '../common/utils' import { GeneratedChallenge } from './interfaces/generated-captcha' -import { captchaTransformer } from './utils/transformers' +import { eventTransformer } from '../analytics/utils/transformers' import { clickhouse } from '../common/integrations/clickhouse' dayjs.extend(utc) @@ -152,18 +152,19 @@ export class CaptchaService { const osName = ua.os.name const { country } = getIPDetails(ip) - const transformed = captchaTransformer( + const transformed = eventTransformer({ + type: 'captcha', pid, - deviceType, - browserName, - osName, - country, + dv: deviceType, + br: browserName, + os: osName, + cc: country, timestamp, - ) + }) try { await clickhouse.insert({ - table: 'captcha', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { diff --git a/backend/apps/cloud/src/captcha/utils/transformers.ts b/backend/apps/cloud/src/captcha/utils/transformers.ts deleted file mode 100644 index 3a0d21d43..000000000 --- a/backend/apps/cloud/src/captcha/utils/transformers.ts +++ /dev/null @@ -1,22 +0,0 @@ -import dayjs from 'dayjs' -import utc from 'dayjs/plugin/utc' - -dayjs.extend(utc) - -export const captchaTransformer = ( - pid: string, - dv: string | null, - br: string | null, - os: string | null, - cc: string | null, - timestamp: number, -) => { - return { - pid, - dv: dv || null, - br: br || null, - os: os || null, - cc: cc || null, - created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'), - } -} diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts index 18799910d..019945a24 100644 --- a/backend/apps/cloud/src/data-import/data-import.processor.ts +++ b/backend/apps/cloud/src/data-import/data-import.processor.ts @@ -107,8 +107,7 @@ export class DataImportProcessor extends WorkerHost { let minDate: string | null = null let maxDate: string | null = null - const analyticsBatch: Record[] = [] - const customEVBatch: Record[] = [] + const eventsBatch: Record[] = [] const context = isApiBased ? { @@ -142,28 +141,17 @@ export class DataImportProcessor extends WorkerHost { if (!maxDate || created > maxDate) maxDate = created } - if (row.table === 'analytics') { - analyticsBatch.push(row.data) - } else { - customEVBatch.push(row.data) - } - - if (analyticsBatch.length >= BATCH_SIZE) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - analyticsBatch.length = 0 - } + eventsBatch.push({ type: row.type, ...row.data }) - if (customEVBatch.length >= BATCH_SIZE) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length - customEVBatch.length = 0 + if (eventsBatch.length >= BATCH_SIZE) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length + eventsBatch.length = 0 } if (totalRows % 10000 === 0) { const progress = { - importedRows: - importedRows + analyticsBatch.length + customEVBatch.length, + importedRows: importedRows + eventsBatch.length, totalRows, } @@ -181,14 +169,9 @@ export class DataImportProcessor extends WorkerHost { } } - if (analyticsBatch.length > 0) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - } - - if (customEVBatch.length > 0) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length + if (eventsBatch.length > 0) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length } await this.dataImportService.markCompleted(id, { @@ -235,12 +218,9 @@ export class DataImportProcessor extends WorkerHost { } } - private async flushBatch( - table: 'analytics' | 'customEV', - batch: Record[], - ): Promise { + private async flushBatch(batch: Record[]): Promise { await clickhouse.insert({ - table: `${CLICKHOUSE_DB}.${table}`, + table: `${CLICKHOUSE_DB}.events`, values: batch, format: 'JSONEachRow', }) diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.ts index 63f60b59e..834b53086 100644 --- a/backend/apps/cloud/src/data-import/data-import.service.ts +++ b/backend/apps/cloud/src/data-import/data-import.service.ts @@ -161,11 +161,7 @@ export class DataImportService { importId: number, ): Promise { await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, - query_params: { pid: projectId, importID: importId }, - }) - await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`, query_params: { pid: projectId, importID: importId }, }) } @@ -207,7 +203,7 @@ export class DataImportService { ): Promise { const result = await clickhouse .query({ - query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, + query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, query_params: { pid: projectId, from, to }, }) .then((resultSet) => resultSet.json<{ cnt: string }>()) diff --git a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts index f74c72122..b9dce040d 100644 --- a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts @@ -206,13 +206,13 @@ export class FathomMapper implements ImportMapper { if (fileType === 'pageviews') { for (let i = 0; i < count; i++) { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' - const evData = { ...baseData, ev: eventName } + const evData = { ...baseData, event_name: eventName } for (let i = 0; i < count; i++) { - yield { table: 'customEV', data: evData } + yield { type: 'custom_event', data: evData } } } } diff --git a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts index 5cd625c58..2ee657aa9 100644 --- a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts @@ -357,7 +357,7 @@ export class Ga4Mapper implements ImportMapper { } for (let i = 0; i < count; i++) { - yield { table: 'analytics', data } + yield { type: 'pageview', data } } } @@ -441,7 +441,7 @@ export class Ga4Mapper implements ImportMapper { pid, host: truncate(host, 253), pg: truncate(pagePath, 2048), - ev: truncate(eventName, 256), + event_name: truncate(eventName, 256), dv: device, br: truncate(browser, 30), brv: null, @@ -465,7 +465,7 @@ export class Ga4Mapper implements ImportMapper { } for (let i = 0; i < count; i++) { - yield { table: 'customEV', data } + yield { type: 'custom_event', data } } } diff --git a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts index 0351b00b0..cf19c69c8 100644 --- a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts +++ b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts @@ -6,7 +6,7 @@ export class ImportError extends Error { } export interface AnalyticsImportRow { - table: 'analytics' | 'customEV' + type: 'pageview' | 'custom_event' data: Record } diff --git a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts index 99141bb58..f0c5926d3 100644 --- a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts @@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper { created, }) - yield { table: 'analytics', data } + yield { type: 'pageview', data } } } @@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper { page: null, created, }) - data.ev = truncate(ev.name, 256) || '' - yield { table: 'customEV', data } + data.event_name = truncate(ev.name, 256) || '' + yield { type: 'custom_event', data } } } } diff --git a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts index 880b863b9..b95e37380 100644 --- a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts @@ -176,11 +176,11 @@ export class SimpleAnalyticsMapper implements ImportMapper { const datapoint = normalizeNull(row.datapoint) if (!datapoint || datapoint === 'pageview') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { yield { - table: 'customEV', - data: { ...baseData, ev: truncate(datapoint, 256) || '' }, + type: 'custom_event', + data: { ...baseData, event_name: truncate(datapoint, 256) || '' }, } } } diff --git a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts index 192a9638b..3d8c9d9b8 100644 --- a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts @@ -372,12 +372,12 @@ export class UmamiMapper implements ImportMapper { } if (eventType === '1') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' yield { - table: 'customEV', - data: { ...baseData, ev: eventName }, + type: 'custom_event', + data: { ...baseData, event_name: eventName }, } } } diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts index 52f230267..14e7d58a2 100644 --- a/backend/apps/cloud/src/experiment/experiment.controller.ts +++ b/backend/apps/cloud/src/experiment/experiment.controller.ts @@ -940,8 +940,9 @@ export class ExperimentController { let conversionsData: { variantKey: string; conversions: number }[] = [] if (experiment.goal) { const goalType = experiment.goal.type - const table = goalType === 'custom_event' ? 'customEV' : 'analytics' - const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg' + const eventType = + goalType === 'custom_event' ? 'custom_event' : 'pageview' + const matchColumn = goalType === 'custom_event' ? 'event_name' : 'pg' const goalValue = experiment.goal.value || '' let matchCondition = '' @@ -974,7 +975,7 @@ export class ExperimentController { e.variantKey, uniqExact(e.profileId) as conversions FROM experiment_exposures e - INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) + INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}' WHERE e.pid = {pid:FixedString(12)} AND e.experimentId = {experimentId:String} @@ -1178,8 +1179,9 @@ export class ExperimentController { let conversionsData: any[] = [] if (experiment.goal) { const goalType = experiment.goal.type - const table = goalType === 'custom_event' ? 'customEV' : 'analytics' - const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg' + const eventType = + goalType === 'custom_event' ? 'custom_event' : 'pageview' + const matchColumn = goalType === 'custom_event' ? 'event_name' : 'pg' const goalValue = experiment.goal.value || '' const conversionsDateColumnsSelect = this.getTimeBucketDateColumnsSelect( timeBucket, @@ -1221,7 +1223,7 @@ export class ExperimentController { e.profileId as profileId, min(c.created) as firstConversion FROM experiment_exposures e - INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) + INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}' WHERE e.pid = {pid:FixedString(12)} AND e.experimentId = {experimentId:String} diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.ts index 657554968..59c897c49 100644 --- a/backend/apps/cloud/src/goal/goal.controller.ts +++ b/backend/apps/cloud/src/goal/goal.controller.ts @@ -332,39 +332,33 @@ export class GoalController { }) } - private buildGoalMatchCondition( - goal: Goal, - _table: 'analytics' | 'customEV', - ): { condition: string; params: Record } { + private buildGoalMatchCondition(goal: Goal): { + condition: string + params: Record + } { const params: Record = {} if (goal.type === GoalType.CUSTOM_EVENT) { - // For custom events, match the event name if (goal.matchType === GoalMatchType.EXACT) { params.goalValue = goal.value || '' - return { condition: `ev = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `ev ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + return { condition: `event_name = {goalValue:String}`, params } } - } else { - // For pageview goals, match the page path - if (goal.matchType === GoalMatchType.EXACT) { - params.goalValue = goal.value || '' - return { condition: `pg = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + params.goalValue = goal.value || '' + return { + condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`, + params, } } + + if (goal.matchType === GoalMatchType.EXACT) { + params.goalValue = goal.value || '' + return { condition: `pg = {goalValue:String}`, params } + } + params.goalValue = goal.value || '' + return { + condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, + params, + } } private buildMetadataCondition(goal: Goal): { @@ -429,9 +423,10 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -440,9 +435,10 @@ export class GoalController { SELECT count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${goalType}' AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -468,9 +464,10 @@ export class GoalController { // Get total unique sessions for conversion rate const totalSessionsQuery = ` SELECT uniqExact(psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -649,7 +646,8 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const timeBucketFunc = Object.prototype.hasOwnProperty.call( timeBucketConversion, resolvedTimeBucket, @@ -659,7 +657,7 @@ export class GoalController { const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket) const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -671,9 +669,10 @@ export class GoalController { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${goalType}' AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.ts index 5e54aab17..13711340a 100644 --- a/backend/apps/cloud/src/project/project.controller.ts +++ b/backend/apps/cloud/src/project/project.controller.ts @@ -745,12 +745,8 @@ export class ProjectController { } const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}', 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}', ] try { @@ -1607,12 +1603,8 @@ export class ProjectController { } const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}', 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}', ] try { diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts index 5e5c29c63..11fe2c715 100644 --- a/backend/apps/cloud/src/project/project.service.ts +++ b/backend/apps/cloud/src/project/project.service.ts @@ -489,12 +489,8 @@ export class ProjectService { const projectIds = _map(projects, 'id') const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})', 'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE captcha DELETE WHERE pid IN ({pids:Array(FixedString(12))})', ] await Promise.all( @@ -654,7 +650,7 @@ export class ProjectService { ) } - let query = `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)} AND (` + let query = `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND (` const params = { pid, @@ -684,10 +680,7 @@ export class ProjectService { to: string, ): Promise { const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', + "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error') AND created BETWEEN {from:String} AND {to:String}", 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', ] @@ -820,52 +813,37 @@ export class ProjectService { monthEnd, } - const selector = ` + const query = ` + SELECT + countIf(type = 'pageview') AS pageviews, + countIf(type = 'custom_event') AS customEvents, + countIf(type = 'captcha') AS captcha, + countIf(type = 'error') AS errors + FROM events WHERE created BETWEEN {monthStart:String} AND {monthEnd:String} - AND pid IN ({pids:Array(FixedString(12))}) + AND pid IN ({pids:Array(FixedString(12))}) + AND type IN ('pageview', 'custom_event', 'captcha', 'error') ` - const countEVQuery = `SELECT count() FROM analytics ${selector}` - const countCustomEVQuery = `SELECT count() FROM customEV ${selector}` - const countCaptchaQuery = `SELECT count() FROM captcha ${selector}` - const countErrorsQuery = `SELECT count() FROM errors ${selector}` - - const [ - { data: pageviews }, - { data: customEvents }, - { data: captcha }, - { data: errors }, - ] = await Promise.all([ - clickhouse - .query({ - query: countEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCustomEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCaptchaQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countErrorsQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - ]) + const { data: counts } = await clickhouse + .query({ + query, + query_params: params, + }) + .then((resultSet) => + resultSet.json<{ + pageviews: string | number + customEvents: string | number + captcha: string | number + errors: string | number + }>(), + ) - totalPageviews += pageviews[0]['count()'] - totalCustomEvents += customEvents[0]['count()'] - totalCaptcha += captcha[0]['count()'] - totalErrors += errors[0]['count()'] + const row = counts[0] + totalPageviews += Number(row?.pageviews) || 0 + totalCustomEvents += Number(row?.customEvents) || 0 + totalCaptcha += Number(row?.captcha) || 0 + totalErrors += Number(row?.errors) || 0 } count = totalPageviews + totalCustomEvents + totalCaptcha + totalErrors @@ -918,52 +896,37 @@ export class ProjectService { const pidChunk = pids.slice(i, i + CHUNK_SIZE) const params = { pids: pidChunk } - const selector = ` - WHERE pid IN ({pids:Array(FixedString(12))}) - AND created BETWEEN '${monthStart}' AND '${monthEnd}' - ` - - const countEVQuery = `SELECT count() FROM analytics ${selector}` - const countCustomEVQuery = `SELECT count() FROM customEV ${selector}` - const countCaptchaQuery = `SELECT count() FROM captcha ${selector}` - const countErrorsQuery = `SELECT count() FROM errors ${selector}` - - const [ - { data: rawTraffic }, - { data: rawCustomEvents }, - { data: rawCaptcha }, - { data: rawErrors }, - ] = await Promise.all([ - clickhouse - .query({ - query: countEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCustomEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCaptchaQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countErrorsQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - ]) + const query = ` + SELECT + countIf(type = 'pageview') AS traffic, + countIf(type = 'custom_event') AS customEvents, + countIf(type = 'captcha') AS captcha, + countIf(type = 'error') AS errors + FROM events + WHERE pid IN ({pids:Array(FixedString(12))}) + AND created BETWEEN '${monthStart}' AND '${monthEnd}' + AND type IN ('pageview', 'custom_event', 'captcha', 'error') + ` + + const { data: counts } = await clickhouse + .query({ + query, + query_params: params, + }) + .then((resultSet) => + resultSet.json<{ + traffic: string | number + customEvents: string | number + captcha: string | number + errors: string | number + }>(), + ) - totalTraffic += rawTraffic[0]['count()'] - totalCustomEvents += rawCustomEvents[0]['count()'] - totalCaptcha += rawCaptcha[0]['count()'] - totalErrors += rawErrors[0]['count()'] + const row = counts[0] + totalTraffic += Number(row?.traffic) || 0 + totalCustomEvents += Number(row?.customEvents) || 0 + totalCaptcha += Number(row?.captcha) || 0 + totalErrors += Number(row?.errors) || 0 } const total = @@ -1043,36 +1006,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM analytics - WHERE pid IN (${pids}) - ) - OR EXISTS ( - SELECT 1 - FROM customEV - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM - ( - SELECT pid - FROM analytics - WHERE pid IN (${pids}) - UNION ALL - SELECT pid - FROM customEV - WHERE pid IN (${pids}) - ) AS t - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type IN ('pageview', 'custom_event') ` const { data } = await clickhouse @@ -1105,23 +1042,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM errors - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM errors - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'error' ` const { data } = await clickhouse @@ -1154,23 +1078,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM captcha - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM captcha - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'captcha' ` const { data } = await clickhouse diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 83b53f2e3..1d7b475b2 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -143,13 +143,15 @@ const mapLimit = async ( return results } -// TODO: Count for other tables like captcha, errors, customEV too const generatePlanUsageQueryForUser = (): string => { // NOTE: keep all values parameterized to avoid injection and formatting issues. + // Counts every billable event kind (pageview, custom_event, error, captcha) + // in a single scan over the unified events table. return ` SELECT {uid:String} AS id, count(*) AS "count" - FROM analytics + FROM events WHERE pid IN ({pids:Array(FixedString(12))}) + AND type IN ('pageview', 'custom_event', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String} ` } @@ -391,7 +393,7 @@ export class TaskManagerService { } const params: Record = {} - const column = goal.type === GoalType.CUSTOM_EVENT ? 'ev' : 'pg' + const column = goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg' if (goal.matchType === GoalMatchType.EXACT) { params[paramKey] = goalValue @@ -427,7 +429,7 @@ export class TaskManagerService { const total = Number(totalSessions) || 0 const buildUnionQuery = ( - table: 'analytics' | 'customEV', + eventType: 'pageview' | 'custom_event', goals: Goal[], ): { query: string; params: Record } | null => { if (_isEmpty(goals)) { @@ -459,9 +461,10 @@ export class TaskManagerService { {${goalIdKey}:String} as goalId, count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${eventType}' AND ${condition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} `) @@ -488,8 +491,8 @@ export class TaskManagerService { (g) => g.type === GoalType.CUSTOM_EVENT, ) - const analyticsQuery = buildUnionQuery('analytics', analyticsGoals) - const customEventQuery = buildUnionQuery('customEV', customEventGoals) + const analyticsQuery = buildUnionQuery('pageview', analyticsGoals) + const customEventQuery = buildUnionQuery('custom_event', customEventGoals) const [analyticsRes, customEventRes] = await Promise.all([ analyticsQuery @@ -573,24 +576,10 @@ export class TaskManagerService { for (let i = 0; i < pids.length; i += CHUNK_SIZE) { const pidChunk = pids.slice(i, i + CHUNK_SIZE) const query = ` - SELECT sum(cnt) AS totalEvents - FROM ( - SELECT count() AS cnt - FROM analytics - WHERE pid IN ({pids:Array(FixedString(12))}) - UNION ALL - SELECT count() AS cnt - FROM customEV - WHERE pid IN ({pids:Array(FixedString(12))}) - UNION ALL - SELECT count() AS cnt - FROM captcha - WHERE pid IN ({pids:Array(FixedString(12))}) - UNION ALL - SELECT count() AS cnt - FROM errors - WHERE pid IN ({pids:Array(FixedString(12))}) - ) + SELECT count() AS totalEvents + FROM events + WHERE pid IN ({pids:Array(FixedString(12))}) + AND type IN ('pageview', 'custom_event', 'error', 'captcha') ` const { data } = await clickhouse @@ -1717,8 +1706,8 @@ export class TaskManagerService { count() FROM ( SELECT eid, min(created) as first_seen - FROM errors - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' GROUP BY eid ) WHERE first_seen >= now() - ${subtractSecondsTimeframe} @@ -1727,9 +1716,10 @@ export class TaskManagerService { query = ` SELECT count() - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created >= now() - ${subtractSecondsTimeframe} ` } @@ -1737,10 +1727,11 @@ export class TaskManagerService { query = ` SELECT count() - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} - AND ev = {ev:String} + AND type = 'custom_event' + AND event_name = {ev:String} AND created >= now() - ${subtractSecondsTimeframe} ` queryParams.ev = alert.queryCustomEvent @@ -1749,9 +1740,10 @@ export class TaskManagerService { query = ` SELECT count(${isUnique ? 'DISTINCT psid' : '*'}) - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created >= now() - ${subtractSecondsTimeframe} ` } @@ -1798,14 +1790,14 @@ export class TaskManagerService { if (alert.alertOnNewErrorsOnly) { detailQuery = ` - SELECT eid, name, message, lineno, colno, filename - FROM errors - WHERE pid = {pid:FixedString(12)} AND eid IN ( + SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN ( SELECT eid FROM ( SELECT eid, min(created) AS first_seen_for_eid - FROM errors - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' GROUP BY eid ) WHERE first_seen_for_eid >= now() - ${subtractSecondsTimeframe} @@ -1815,10 +1807,11 @@ export class TaskManagerService { ` } else { detailQuery = ` - SELECT eid, name, message, lineno, colno, filename - FROM errors + SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created >= now() - ${subtractSecondsTimeframe} ORDER BY created DESC LIMIT 1 @@ -2003,9 +1996,9 @@ export class TaskManagerService { } // No need to check for performance activity because it's not tracked without tracking analytics - const queryAnalytics = `SELECT count() FROM analytics WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryAnalytics = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type = 'pageview' AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryCaptcha = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type = 'captcha' AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryCustomEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type = 'custom_event' AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` // Process project IDs in chunks to avoid ClickHouse field value limit let totalAnalytics = 0 diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts index c6313fad7..4816abb97 100644 --- a/backend/apps/community/src/analytics/analytics.controller.ts +++ b/backend/apps/community/src/analytics/analytics.controller.ts @@ -73,12 +73,7 @@ import { GetErrorOverviewOptions, } from './dto/get-error-overview.dto' import { PatchStatusDto } from './dto/patch-status.dto' -import { - customEventTransformer, - errorEventTransformer, - performanceTransformer, - trafficTransformer, -} from './utils/transformers' +import { eventTransformer } from './utils/transformers' import { enrichTrafficSource } from './utils/clickIdSources' import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto' import { GetOverallStatsDto } from './dto/get-overall-stats.dto' @@ -278,12 +273,12 @@ export class AnalyticsController { diff, ) - let subQuery = `FROM ${ - isCaptcha ? 'captcha' : 'analytics' - } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + let subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${ + isCaptcha ? 'captcha' : 'pageview' + }' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` if (customEVFilterApplied && !isCaptcha) { - subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` } const paramsData = { @@ -732,7 +727,7 @@ export class AnalyticsController { diff, ) - const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } @@ -1026,32 +1021,12 @@ export class AnalyticsController { any(os) AS os, any(cc) AS cc, toString(psid) AS psid - FROM - ( - SELECT - psid, - dv, - br, - os, - cc - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT - psid, - dv, - br, - os, - cc - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL GROUP BY psid ` @@ -1151,7 +1126,8 @@ export class AnalyticsController { enrichTrafficSource(eventsDTO) - const transformed = customEventTransformer({ + const transformed = eventTransformer({ + type: 'custom_event', psid, profileId, pid: eventsDTO.pid, @@ -1183,7 +1159,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'customEV', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1321,7 +1297,8 @@ export class AnalyticsController { enrichTrafficSource(logDTO) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1364,7 +1341,8 @@ export class AnalyticsController { ttfb, } = logDTO.perf - perfTransformed = performanceTransformer({ + perfTransformed = eventTransformer({ + type: 'performance', pid: logDTO.pid, host: this.analyticsService.getHostFromOrigin(headers.origin), pg: logDTO.pg, @@ -1392,7 +1370,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1400,7 +1378,7 @@ export class AnalyticsController { if (!_isEmpty(perfTransformed)) { await clickhouse.insert({ - table: 'performance', + table: 'events', format: 'JSONEachRow', values: [perfTransformed], clickhouse_settings: { async_insert: 1 }, @@ -1489,7 +1467,8 @@ export class AnalyticsController { const { deviceType, browserName, browserVersion, osName, osVersion } = await this.analyticsService.getRequestInformation(headers) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1520,7 +1499,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1869,7 +1848,8 @@ export class AnalyticsController { const { name, message, lineno, colno, filename, stackTrace, meta } = errorDTO - const transformed = errorEventTransformer({ + const transformed = eventTransformer({ + type: 'error', psid, profileId, eid: this.analyticsService.getErrorID(errorDTO), @@ -1901,7 +1881,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'errors', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index f63a48d82..2a0ec2231 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -665,7 +665,15 @@ export class AnalyticsService { ) } - const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` + // Public ERROR_COLUMNS use legacy short names; map to new column names. + const errorColMap: Record = { + name: 'error_name', + message: 'error_message', + filename: 'error_filename', + } + const sqlCol = errorColMap[type] || type + + const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}` const { data } = await clickhouse .query({ @@ -684,13 +692,13 @@ export class AnalyticsService { type: 'traffic' | 'errors', column: 'br' | 'os', ): Promise> { - const safeTable = type === 'errors' ? 'errors' : 'analytics' + const safeType = type === 'errors' ? 'error' : 'pageview' const safeVersionCol = column === 'br' ? 'brv' : 'osv' const query = ` - SELECT ${column}, ${safeVersionCol} - FROM ${safeTable} - WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL + SELECT ${column}, ${safeVersionCol} + FROM events + WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL GROUP BY ${column}, ${safeVersionCol} ` @@ -871,9 +879,10 @@ export class AnalyticsService { pg, created, lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND pg IS NOT NULL ${filtersQuery} @@ -991,10 +1000,18 @@ export class AnalyticsService { timeBucket: TimeBucketType[] diff: number }> { + const tableToType: Record = { + analytics: 'pageview', + customEV: 'custom_event', + performance: 'performance', + errors: 'error', + captcha: 'captcha', + } + const { data: fromData } = await clickhouse .query({ - query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`, - query_params: { pid }, + query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type = {type:String}`, + query_params: { pid, type: tableToType[table] }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -1228,8 +1245,8 @@ export class AnalyticsService { ? 'argMin(pg, created)' : 'argMax(pg, created)' const subQueryForPages = isContains - ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` - : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}` + ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` + : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page = {${param}:String}` // For exclusive filter (isNot) we exclude sessions with matching entry/exit page query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})` @@ -1270,6 +1287,9 @@ export class AnalyticsService { ) { sqlColumn = 'meta.value' isArrayDataset = true + } else if (column === 'ev') { + // Public filter contract `ev` maps to the renamed `event_name` column + sqlColumn = 'event_name' } const isNullFilter = @@ -1586,21 +1606,11 @@ export class AnalyticsService { FROM ( SELECT psid, - pg AS value, + if(type = 'pageview', pg, event_name) AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - - UNION ALL - - SELECT - psid, - ev AS value, - created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -1648,14 +1658,14 @@ export class AnalyticsService { psid, CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1670,17 +1680,11 @@ export class AnalyticsService { psid, argMin(cc, created) AS cc, argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source - FROM ( - SELECT psid, cc, ref, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, cc, ref, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 + AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid ) SELECT step, type, val, cnt FROM ( @@ -1758,14 +1762,14 @@ export class AnalyticsService { psid, windowFunnel(86400)(created, ${pagesStr}) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1781,19 +1785,12 @@ export class AnalyticsService { any(br) AS br, min(toTimeZone(created, {timezone:String})) AS sessionStart, max(toTimeZone(created, {timezone:String})) AS lastActivity - FROM ( - SELECT psid, pid, cc, os, br, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND psid IN (SELECT psid FROM funnel_qualified) - UNION ALL - SELECT psid, pid, cc, os, br, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) - ) GROUP BY psidCasted, pid ), pageview_counts AS ( @@ -1801,8 +1798,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1812,8 +1809,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1823,8 +1820,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1902,8 +1899,9 @@ export class AnalyticsService { const query = ` SELECT count() as c - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` const { data } = await clickhouse @@ -1954,15 +1952,20 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - let queryAll = ` + const allTimeType = customEVFilterApplied + ? 'custom_event' + : 'pageview' + + const queryAll = ` WITH analytics_counts AS ( SELECT count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' ${filtersQuery} ), duration_avg AS ( @@ -1982,36 +1985,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryAll = ` - WITH analytics_counts AS ( - SELECT - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const { data } = await clickhouse .query({ query: queryAll, @@ -2096,16 +2069,19 @@ export class AnalyticsService { ) .format('YYYY-MM-DD HH:mm:ss') - let queryCurrent = ` + const periodType = customEVFilterApplied ? 'custom_event' : 'pageview' + + const queryCurrent = ` WITH analytics_counts AS ( SELECT 1 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} ), @@ -2119,8 +2095,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} @@ -2134,16 +2111,17 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - let queryPrevious = ` + const queryPrevious = ` WITH analytics_counts AS ( SELECT 2 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} ), @@ -2157,8 +2135,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} @@ -2172,83 +2151,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryCurrent = ` - WITH analytics_counts AS ( - SELECT - 1 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - queryPrevious = ` - WITH analytics_counts AS ( - SELECT - 2 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const query = `${queryCurrent} UNION ALL ${queryPrevious}` let { data } = await clickhouse @@ -2456,7 +2358,7 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}` + const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}` const { data } = await clickhouse .query({ query: queryAll, @@ -2518,8 +2420,8 @@ export class AnalyticsService { .format('YYYY-MM-DD HH:mm:ss') } - const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` - const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` + const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` + const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` const query = `${queryCurrent} UNION ALL ${queryPrevious}` @@ -2616,8 +2518,8 @@ export class AnalyticsService { const query = ` WITH session_pages AS ( SELECT psid, ${selector} as page - FROM analytics - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid ) SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page @@ -2633,11 +2535,10 @@ export class AnalyticsService { return _map(data, 'page') } - let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` + let query = `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND ${type} IS NOT NULL GROUP BY ${type}` if (type === 'ev') { - query = - 'SELECT ev FROM customEV WHERE pid={pid:FixedString(12)} AND ev IS NOT NULL GROUP BY ev' + query = `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name` } const { data } = await clickhouse @@ -3005,8 +2906,8 @@ export class AnalyticsService { pid, psid, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3054,8 +2955,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3082,8 +2983,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3108,8 +3009,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'error' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3135,8 +3036,9 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3179,9 +3081,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3214,9 +3117,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3274,9 +3178,10 @@ export class AnalyticsService { FROM ( SELECT pg, dv, br, os, lc, cc, rg, ct, profileId, ${timeBucketFunc}(created) as tz_created - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) as subquery @@ -3798,9 +3703,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -4050,7 +3956,7 @@ export class AnalyticsService { filtersQuery: string, params: any, ): Promise { - const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` + const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name` const result = {} const { data } = await clickhouse @@ -4081,9 +3987,10 @@ export class AnalyticsService { SELECT meta.key AS key FROM - analytics + events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -4165,12 +4072,12 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND ev = {event:String} + AND event_name = {event:String} ${filtersQuery} ) ARRAY JOIN meta.key, meta.value @@ -4266,10 +4173,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND indexOf(meta.key, {property:String}) > 0 ${filtersQuery} @@ -4319,17 +4226,11 @@ export class AnalyticsService { const query = ` SELECT uniqExact(psid) as count - FROM ( - SELECT psid FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT psid FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL ` try { @@ -4367,25 +4268,26 @@ export class AnalyticsService { const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})` const customEventsFilter = customEvents && customEvents.length > 0 - ? 'AND ev IN {customEvents:Array(String)}' + ? 'AND event_name IN {customEvents:Array(String)}' : '' const query = ` SELECT ${selector}, - ev, + event_name AS ev, count() as count FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ${customEventsFilter} ) as subquery - GROUP BY ${groupBy}, ev + GROUP BY ${groupBy}, event_name ORDER BY ${groupBy} ` @@ -4452,52 +4354,31 @@ export class AnalyticsService { const queryPages = ` WITH events_with_meta AS ( SELECT - 'pageview' AS type, - pg AS value, - toTimeZone(analytics.created, {timezone:String}) AS created, - pid, - psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND CAST(analytics.psid AS String) = {psid:String} - - UNION ALL - - SELECT - 'event' AS type, - ev AS value, - toTimeZone(customEV.created, {timezone:String}) AS created, + multiIf(type = 'pageview', 'pageview', type = 'custom_event', 'event', 'error') AS type, + multiIf( + type = 'pageview', toString(pg), + type = 'custom_event', toString(event_name), + toString(error_name) + ) AS value, + toTimeZone(created, {timezone:String}) AS created, pid, - psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM customEV + toString(psid) AS psid, + if( + type = 'error', + [ + tuple('message', COALESCE(error_message, '')), + tuple('lineno', toString(COALESCE(lineno, 0))), + tuple('colno', toString(COALESCE(colno, 0))), + tuple('filename', COALESCE(error_filename, '')) + ], + arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) + ) AS metadata + FROM events WHERE pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND CAST(customEV.psid AS String) = {psid:String} - - UNION ALL - - SELECT - 'error' AS type, - errors.name AS value, - toTimeZone(errors.created, {timezone:String}) AS created, - pid, - psid, - [ - tuple('message', COALESCE(errors.message, '')), - tuple('lineno', CAST(COALESCE(errors.lineno, 0), 'String')), - tuple('colno', CAST(COALESCE(errors.colno, 0), 'String')), - tuple('filename', COALESCE(errors.filename, '')) - ] AS metadata - FROM errors - WHERE - pid = {pid:FixedString(12)} - AND errors.psid IS NOT NULL - AND CAST(errors.psid AS String) = {psid:String} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} ) SELECT @@ -4512,10 +4393,12 @@ export class AnalyticsService { const querySessionDetails = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} - AND CAST(psid, 'String') = {psid:String} + AND type = 'pageview' + AND psid IS NOT NULL + AND toString(psid) = {psid:String} ORDER BY created ASC LIMIT 1; ` @@ -4601,11 +4484,12 @@ export class AnalyticsService { const querySessionDetailsFromCustomEV = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND psid IS NOT NULL - AND CAST(psid, 'String') = {psid:String} + AND toString(psid) = {psid:String} ORDER BY created ASC LIMIT 1; ` @@ -4698,47 +4582,44 @@ export class AnalyticsService { if (customEVFilterApplied) { primaryEventsSubquery = ` SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) ` } else { primaryEventsSubquery = ` SELECT - CAST(analytics.psid, 'String') AS psidCasted, - analytics.pid, - analytics.cc, - analytics.os, - analytics.br, - toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping - FROM analytics - WHERE - analytics.pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ` } @@ -4761,8 +4642,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -4771,8 +4652,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -4781,8 +4662,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -4878,10 +4759,18 @@ export class AnalyticsService { count(DISTINCT psid) as sessions, status.status FROM ( - SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created - FROM errors + SELECT + eid, + error_name AS name, + error_message AS message, + error_filename AS filename, + psid, + profileId, + toTimeZone(created, {timezone:String}) AS created + FROM events WHERE pid = {pid:FixedString(12)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} + AND type = 'error' + AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) AS errors LEFT JOIN ( @@ -4940,14 +4829,15 @@ export class AnalyticsService { FROM ( SELECT eid, - name, - message, - filename, + error_name AS name, + error_message AS message, + error_filename AS filename, colno, lineno, stackTrace - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ORDER BY created DESC @@ -4970,9 +4860,10 @@ export class AnalyticsService { max(created) AS last_seen, min(created) AS first_seen, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)}; ` @@ -4985,9 +4876,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -5034,7 +4926,7 @@ export class AnalyticsService { timeBucket, groupFrom, groupTo, - `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, + `FROM events WHERE type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, 'AND eid = {eid:FixedString(32)}', paramsData, safeTimezone, @@ -5066,11 +4958,12 @@ export class AnalyticsService { ? '' : "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)" - // Get total sessions from analytics table for the time range + // Get total sessions from pageview events for the time range const queryTotalSessions = ` SELECT count(DISTINCT psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -5081,7 +4974,7 @@ export class AnalyticsService { count(DISTINCT eid) as uniqueErrors, count(DISTINCT psid) as affectedSessions, count(DISTINCT profileId) as affectedUsers - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -5089,6 +4982,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5098,12 +4992,12 @@ export class AnalyticsService { const queryMostFrequentError = ` SELECT errors.eid as eid, - any(errors.name) as name, - any(errors.message) as message, + any(errors.error_name) as name, + any(errors.error_message) as message, count(*) as count, count(DISTINCT errors.profileId) as usersAffected, max(errors.created) as lastSeen - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -5111,6 +5005,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5135,7 +5030,7 @@ export class AnalyticsService { SELECT profileId, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -5143,6 +5038,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5254,8 +5150,9 @@ export class AnalyticsService { ): Promise<{ sessions: any[]; total: number }> { const queryCount = ` SELECT count(DISTINCT psid) as total - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -5270,9 +5167,13 @@ export class AnalyticsService { min(errors.created) as firstErrorAt, max(errors.created) as lastErrorAt, count(*) as errorCount - FROM errors - LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid + FROM events AS errors + LEFT JOIN events AS analytics + ON errors.psid = analytics.psid + AND errors.pid = analytics.pid + AND analytics.type = 'pageview' WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.eid = {eid:FixedString(32)} AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY errors.psid @@ -5319,7 +5220,7 @@ export class AnalyticsService { {}, ) - const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map( + const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map( params, (val, key) => `{${key}:FixedString(32)}`, ).join(', ')})` @@ -5482,12 +5383,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodFormatted:String} AND {now:String} ${kvQuery} ${filtersQuery} @@ -5500,12 +5402,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${kvQuery} ${filtersQuery} @@ -5629,42 +5532,26 @@ export class AnalyticsService { br, dv, created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics + if(type = 'pageview', 1, 0) AS isPageview, + if(type = 'custom_event', 1, 0) AS isEvent + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} AND profileId IN ( SELECT DISTINCT profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} ${filtersQuery} ) - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} )` } else { allProfileDataCTE = ` @@ -5677,28 +5564,11 @@ export class AnalyticsService { br, dv, created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV + if(type = 'pageview', 1, 0) AS isPageview, + if(type = 'custom_event', 1, 0) AS isEvent + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' @@ -5728,8 +5598,10 @@ export class AnalyticsService { SELECT profileId, count() AS errorsCount - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' + AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} @@ -5785,7 +5657,7 @@ export class AnalyticsService { AND profileId = {profileId:String} ` - // Query avg duration from analytics table (more accurate than sessions table) + // Query avg duration from pageview events (more accurate than sessions table) const queryAvgDuration = ` SELECT avg(session_duration) AS avgDuration @@ -5793,8 +5665,9 @@ export class AnalyticsService { SELECT psid, dateDiff('second', min(created), max(created)) AS session_duration - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND psid IS NOT NULL GROUP BY psid @@ -5802,19 +5675,19 @@ export class AnalyticsService { ) ` - // Query analytics for accurate pageview count const queryPageviews = ` SELECT count() AS pageviewsCount - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} ` - // Query customEV for accurate events count const queryEvents = ` SELECT count() AS eventsCount - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} ` @@ -5830,8 +5703,9 @@ export class AnalyticsService { any(brv) AS brv, any(dv) AS dv, any(lc) AS lc - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} ` @@ -5892,8 +5766,9 @@ export class AnalyticsService { SELECT pg AS page, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} GROUP BY pg ORDER BY count DESC @@ -5922,30 +5797,15 @@ export class AnalyticsService { .format('YYYY-MM-DD') const query = ` - WITH all_activity AS ( - SELECT - toDate(created) AS date, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - UNION ALL - SELECT - toDate(created) AS date, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - ) SELECT - date, - sum(isPageview) AS pageviews, - sum(isEvent) AS events - FROM all_activity + toDate(created) AS date, + countIf(type = 'pageview') AS pageviews, + countIf(type = 'custom_event') AS events + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND profileId = {profileId:String} + AND created >= {startDate:Date} GROUP BY date ORDER BY date ASC ` @@ -5980,8 +5840,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5992,8 +5853,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -6004,8 +5866,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -6079,56 +5942,6 @@ export class AnalyticsService { let allProfileEventsCTE: string if (customEVFilterApplied) { - allProfileEventsCTE = ` - all_profile_events AS ( - SELECT - all_events.psidCasted, - all_events.pid, - all_events.profileId, - all_events.cc, - all_events.os, - all_events.br, - all_events.created_tz - FROM ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) AS all_events - WHERE all_events.psidCasted IN ( - SELECT DISTINCT CAST(psid, 'String') - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - )` - } else { allProfileEventsCTE = ` all_profile_events AS ( SELECT @@ -6139,13 +5952,26 @@ export class AnalyticsService { os, br, toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND profileId = {profileId:String} + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + )` + } else { + allProfileEventsCTE = ` + all_profile_events AS ( SELECT CAST(psid, 'String') AS psidCasted, pid, @@ -6154,8 +5980,9 @@ export class AnalyticsService { os, br, toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6183,8 +6010,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6195,8 +6023,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6207,8 +6036,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} diff --git a/backend/apps/community/src/analytics/utils/transformers.ts b/backend/apps/community/src/analytics/utils/transformers.ts index b420cdac8..687fb0c3e 100644 --- a/backend/apps/community/src/analytics/utils/transformers.ts +++ b/backend/apps/community/src/analytics/utils/transformers.ts @@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc' dayjs.extend(utc) const processMetaKV = ( - rawMeta: Record, + rawMeta: Record | null | undefined, ): { 'meta.key': string[] | null; 'meta.value': string[] | null } => { if (!rawMeta || _isEmpty(rawMeta)) { return { @@ -23,10 +23,10 @@ const processMetaKV = ( } } -interface TrafficTransformerOptions { - psid: string - profileId: string +interface CommonOptions { pid: string + psid?: string | null + profileId?: string | null host?: string | null pg?: string | null dv?: string | null @@ -52,262 +52,42 @@ interface TrafficTransformerOptions { meta?: Record | null } -export const trafficTransformer = ({ - psid, - profileId, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: TrafficTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } +interface PageviewOptions extends CommonOptions { + type: 'pageview' } -interface CustomEventTransformerOptions { - psid: string - profileId: string - pid: string - host?: string | null +interface CustomEventOptions extends CommonOptions { + type: 'custom_event' ev: string - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - ref?: string | null - so?: string | null - me?: string | null - ca?: string | null - te?: string | null - co?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null - meta?: Record | null -} - -export const customEventTransformer = ({ - psid, - profileId, - pid, - host, - ev, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: CustomEventTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - ev, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface ErrorEventTransformerOptions { - psid: string - profileId: string +interface ErrorEventOptions extends CommonOptions { + type: 'error' eid: string - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null name?: string | null message?: string | null lineno?: number | null colno?: number | null filename?: string | null stackTrace?: string | null - meta?: Record | null -} - -export const errorEventTransformer = ({ - psid, - profileId, - eid, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - name, - message, - lineno, - colno, - filename, - stackTrace, - meta, -}: ErrorEventTransformerOptions) => { - return { - psid, - profileId, - eid, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - name: name || null, - message: message || null, - lineno: lineno || null, - colno: colno || null, - filename: filename || null, - stackTrace: stackTrace || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface PerformanceTransformerOptions { - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null +interface PerformanceOptions extends Omit< + CommonOptions, + | 'psid' + | 'profileId' + | 'os' + | 'osv' + | 'lc' + | 'ref' + | 'so' + | 'me' + | 'ca' + | 'te' + | 'co' + | 'meta' +> { + type: 'performance' dns: number tls: number conn: number @@ -318,53 +98,131 @@ interface PerformanceTransformerOptions { ttfb: number } -export const performanceTransformer = ({ - pid, - host, - pg, - dv, - br, - brv, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - dns, - tls, - conn, - response, - render, - domLoad, - pageLoad, - ttfb, -}: PerformanceTransformerOptions) => { +interface CaptchaOptions { + type: 'captcha' + pid: string + dv?: string | null + br?: string | null + os?: string | null + cc?: string | null + timestamp?: number +} + +type EventTransformerOptions = + | PageviewOptions + | CustomEventOptions + | ErrorEventOptions + | PerformanceOptions + | CaptchaOptions + +const buildCommon = (opts: CommonOptions) => ({ + pid: opts.pid, + psid: opts.psid ?? null, + profileId: opts.profileId ?? null, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + os: opts.os || null, + osv: opts.osv || null, + lc: opts.lc || null, + ref: opts.ref || null, + so: opts.so || null, + me: opts.me || null, + ca: opts.ca || null, + te: opts.te || null, + co: opts.co || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + ...processMetaKV(opts.meta), +}) + +// Single transformer for all event kinds; the discriminator is `type`. +// Renames at the storage boundary: ev -> event_name, name -> error_name, +// message -> error_message, filename -> error_filename. DTO and API field +// names stay as the legacy short forms. +export const eventTransformer = (opts: EventTransformerOptions) => { + const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss') + + if (opts.type === 'pageview') { + return { + type: 'pageview' as const, + ...buildCommon(opts), + created, + } + } + + if (opts.type === 'custom_event') { + return { + type: 'custom_event' as const, + ...buildCommon(opts), + event_name: opts.ev, + created, + } + } + + if (opts.type === 'error') { + return { + type: 'error' as const, + ...buildCommon(opts), + eid: opts.eid, + error_name: opts.name || null, + error_message: opts.message || null, + stackTrace: opts.stackTrace || null, + lineno: opts.lineno || null, + colno: opts.colno || null, + error_filename: opts.filename || null, + created, + } + } + + if (opts.type === 'performance') { + return { + type: 'performance' as const, + pid: opts.pid, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + dns: _round(opts.dns), + tls: _round(opts.tls), + conn: _round(opts.conn), + response: _round(opts.response), + render: _round(opts.render), + domLoad: _round(opts.domLoad), + pageLoad: _round(opts.pageLoad), + ttfb: _round(opts.ttfb), + created, + } + } + + // captcha return { - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - dns: _round(dns), - tls: _round(tls), - conn: _round(conn), - response: _round(response), - render: _round(render), - domLoad: _round(domLoad), - pageLoad: _round(pageLoad), - ttfb: _round(ttfb), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), + type: 'captcha' as const, + pid: opts.pid, + dv: opts.dv || null, + br: opts.br || null, + os: opts.os || null, + cc: opts.cc || null, + created: + typeof opts.timestamp === 'number' + ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss') + : created, } } diff --git a/backend/apps/community/src/captcha/captcha.service.ts b/backend/apps/community/src/captcha/captcha.service.ts index 921e6d7b7..99ac3516c 100644 --- a/backend/apps/community/src/captcha/captcha.service.ts +++ b/backend/apps/community/src/captcha/captcha.service.ts @@ -17,7 +17,7 @@ import { } from '../common/constants' import { getIPDetails } from '../common/utils' import { GeneratedChallenge } from './interfaces/generated-captcha' -import { captchaTransformer } from './utils/transformers' +import { eventTransformer } from '../analytics/utils/transformers' import { clickhouse } from '../common/integrations/clickhouse' dayjs.extend(utc) @@ -152,18 +152,19 @@ export class CaptchaService { const osName = ua.os.name const { country } = getIPDetails(ip) - const transformed = captchaTransformer( + const transformed = eventTransformer({ + type: 'captcha', pid, - deviceType, - browserName, - osName, - country, + dv: deviceType, + br: browserName, + os: osName, + cc: country, timestamp, - ) + }) try { await clickhouse.insert({ - table: 'captcha', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { diff --git a/backend/apps/community/src/captcha/utils/transformers.ts b/backend/apps/community/src/captcha/utils/transformers.ts deleted file mode 100644 index 3a0d21d43..000000000 --- a/backend/apps/community/src/captcha/utils/transformers.ts +++ /dev/null @@ -1,22 +0,0 @@ -import dayjs from 'dayjs' -import utc from 'dayjs/plugin/utc' - -dayjs.extend(utc) - -export const captchaTransformer = ( - pid: string, - dv: string | null, - br: string | null, - os: string | null, - cc: string | null, - timestamp: number, -) => { - return { - pid, - dv: dv || null, - br: br || null, - os: os || null, - cc: cc || null, - created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'), - } -} diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts index 0d0646d1b..06ee7e3a8 100644 --- a/backend/apps/community/src/data-import/data-import.processor.ts +++ b/backend/apps/community/src/data-import/data-import.processor.ts @@ -85,8 +85,7 @@ export class DataImportProcessor extends WorkerHost { let minDate: string | null = null let maxDate: string | null = null - const analyticsBatch: Record[] = [] - const customEVBatch: Record[] = [] + const eventsBatch: Record[] = [] try { for await (const row of mapper.createRowStream( @@ -102,28 +101,17 @@ export class DataImportProcessor extends WorkerHost { if (!maxDate || created > maxDate) maxDate = created } - if (row.table === 'analytics') { - analyticsBatch.push(row.data) - } else { - customEVBatch.push(row.data) - } - - if (analyticsBatch.length >= BATCH_SIZE) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - analyticsBatch.length = 0 - } + eventsBatch.push({ type: row.type, ...row.data }) - if (customEVBatch.length >= BATCH_SIZE) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length - customEVBatch.length = 0 + if (eventsBatch.length >= BATCH_SIZE) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length + eventsBatch.length = 0 } if (totalRows % 10000 === 0) { const progress = { - importedRows: - importedRows + analyticsBatch.length + customEVBatch.length, + importedRows: importedRows + eventsBatch.length, totalRows, } @@ -141,14 +129,9 @@ export class DataImportProcessor extends WorkerHost { } } - if (analyticsBatch.length > 0) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - } - - if (customEVBatch.length > 0) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length + if (eventsBatch.length > 0) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length } await this.dataImportService.markCompleted(importId, projectId, { @@ -187,12 +170,9 @@ export class DataImportProcessor extends WorkerHost { } } - private async flushBatch( - table: 'analytics' | 'customEV', - batch: Record[], - ): Promise { + private async flushBatch(batch: Record[]): Promise { await clickhouse.insert({ - table: `${CLICKHOUSE_DB}.${table}`, + table: `${CLICKHOUSE_DB}.events`, values: batch, format: 'JSONEachRow', }) diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.ts index ac9be0c50..e1453fbe0 100644 --- a/backend/apps/community/src/data-import/data-import.service.ts +++ b/backend/apps/community/src/data-import/data-import.service.ts @@ -281,11 +281,7 @@ export class DataImportService { importId: number, ): Promise { await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, - query_params: { pid: projectId, importID: importId }, - }) - await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`, query_params: { pid: projectId, importID: importId }, }) } @@ -330,7 +326,7 @@ export class DataImportService { ): Promise { const result = await clickhouse .query({ - query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, + query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, query_params: { pid: projectId, from, to }, }) .then((rs) => rs.json<{ cnt: string }>()) diff --git a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts index 55c578f7c..6f418aaeb 100644 --- a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts @@ -202,13 +202,13 @@ export class FathomMapper implements ImportMapper { if (fileType === 'pageviews') { for (let i = 0; i < count; i++) { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' - const evData = { ...baseData, ev: eventName } + const evData = { ...baseData, event_name: eventName } for (let i = 0; i < count; i++) { - yield { table: 'customEV', data: evData } + yield { type: 'custom_event', data: evData } } } } diff --git a/backend/apps/community/src/data-import/mappers/mapper.interface.ts b/backend/apps/community/src/data-import/mappers/mapper.interface.ts index b7bdc9bf0..73ea8538b 100644 --- a/backend/apps/community/src/data-import/mappers/mapper.interface.ts +++ b/backend/apps/community/src/data-import/mappers/mapper.interface.ts @@ -6,7 +6,7 @@ export class ImportError extends Error { } export interface AnalyticsImportRow { - table: 'analytics' | 'customEV' + type: 'pageview' | 'custom_event' data: Record } diff --git a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts index 99141bb58..f0c5926d3 100644 --- a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts @@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper { created, }) - yield { table: 'analytics', data } + yield { type: 'pageview', data } } } @@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper { page: null, created, }) - data.ev = truncate(ev.name, 256) || '' - yield { table: 'customEV', data } + data.event_name = truncate(ev.name, 256) || '' + yield { type: 'custom_event', data } } } } diff --git a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts index e95c04049..f62c9472c 100644 --- a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts @@ -172,11 +172,11 @@ export class SimpleAnalyticsMapper implements ImportMapper { const datapoint = normalizeNull(row.datapoint) if (!datapoint || datapoint === 'pageview') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { yield { - table: 'customEV', - data: { ...baseData, ev: truncate(datapoint, 256) || '' }, + type: 'custom_event', + data: { ...baseData, event_name: truncate(datapoint, 256) || '' }, } } } diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.ts index a1514ae46..c2bf13747 100644 --- a/backend/apps/community/src/data-import/mappers/umami.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts @@ -368,12 +368,12 @@ export class UmamiMapper implements ImportMapper { } if (eventType === '1') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' yield { - table: 'customEV', - data: { ...baseData, ev: eventName }, + type: 'custom_event', + data: { ...baseData, event_name: eventName }, } } } diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.ts index 0d9e12b71..600afe709 100644 --- a/backend/apps/community/src/goal/goal.controller.ts +++ b/backend/apps/community/src/goal/goal.controller.ts @@ -252,39 +252,33 @@ export class GoalController { await this.goalService.delete(id) } - private buildGoalMatchCondition( - goal: Goal, - _table: 'analytics' | 'customEV', - ): { condition: string; params: Record } { + private buildGoalMatchCondition(goal: Goal): { + condition: string + params: Record + } { const params: Record = {} if (goal.type === GoalType.CUSTOM_EVENT) { - // For custom events, match the event name if (goal.matchType === GoalMatchType.EXACT) { params.goalValue = goal.value || '' - return { condition: `ev = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `ev ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + return { condition: `event_name = {goalValue:String}`, params } } - } else { - // For pageview goals, match the page path - if (goal.matchType === GoalMatchType.EXACT) { - params.goalValue = goal.value || '' - return { condition: `pg = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + params.goalValue = goal.value || '' + return { + condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`, + params, } } + + if (goal.matchType === GoalMatchType.EXACT) { + params.goalValue = goal.value || '' + return { condition: `pg = {goalValue:String}`, params } + } + params.goalValue = goal.value || '' + return { + condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, + params, + } } private buildMetadataCondition(goal: Goal): { @@ -347,9 +341,10 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -358,9 +353,10 @@ export class GoalController { SELECT count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${goalType}' AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -386,9 +382,10 @@ export class GoalController { // Get total unique sessions for conversion rate const totalSessionsQuery = ` SELECT uniqExact(psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -566,7 +563,8 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const timeBucketFunc = Object.prototype.hasOwnProperty.call( timeBucketConversion, resolvedTimeBucket, @@ -576,7 +574,7 @@ export class GoalController { const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket) const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -588,9 +586,10 @@ export class GoalController { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${goalType}' AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.ts index 757444527..fa42f3f2f 100644 --- a/backend/apps/community/src/project/project.controller.ts +++ b/backend/apps/community/src/project/project.controller.ts @@ -576,11 +576,7 @@ export class ProjectController { try { await clickhouse.command({ - query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`, - query_params: { pid: id }, - }) - await clickhouse.command({ - query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`, + query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event')`, query_params: { pid: id }, }) return 'Project reset successfully' @@ -1021,11 +1017,7 @@ export class ProjectController { try { await deleteProjectSharesByProjectClickhouse(id) await clickhouse.command({ - query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`, - query_params: { pid: id }, - }) - await clickhouse.command({ - query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`, + query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event')`, query_params: { pid: id }, }) await deleteProjectRedis(id) diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.ts index 0d6147406..7055f823f 100644 --- a/backend/apps/community/src/project/project.service.ts +++ b/backend/apps/community/src/project/project.service.ts @@ -248,36 +248,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM analytics - WHERE pid IN (${pids}) - ) - OR EXISTS ( - SELECT 1 - FROM customEV - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM - ( - SELECT pid - FROM analytics - WHERE pid IN (${pids}) - UNION ALL - SELECT pid - FROM customEV - WHERE pid IN (${pids}) - ) AS t - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type IN ('pageview', 'custom_event') ` const { data } = await clickhouse @@ -310,23 +284,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM errors - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM errors - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'error' ` const { data } = await clickhouse @@ -359,23 +320,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM captcha - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM captcha - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'captcha' ` const { data } = await clickhouse @@ -394,10 +342,7 @@ export class ProjectService { to: string, ): Promise { const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', + "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error') AND created BETWEEN {from:String} AND {to:String}", 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', ] diff --git a/backend/apps/community/src/user/user.controller.ts b/backend/apps/community/src/user/user.controller.ts index 64dd758fd..cb3e4fe8b 100644 --- a/backend/apps/community/src/user/user.controller.ts +++ b/backend/apps/community/src/user/user.controller.ts @@ -181,10 +181,7 @@ export class UserController { if (!_isEmpty(projects)) { const pidArray = projects.map((el) => el.id) const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})', 'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})', ] await deleteProjectsByUserIdClickhouse(id) diff --git a/backend/meta/clickhouse/generate-dummy-data.js b/backend/meta/clickhouse/generate-dummy-data.js index 1b5f9b8fa..004d81d45 100644 --- a/backend/meta/clickhouse/generate-dummy-data.js +++ b/backend/meta/clickhouse/generate-dummy-data.js @@ -115,10 +115,10 @@ const chalk = { cyan: text => `\x1b[36m${text}\x1b[0m`, } -const insertData = async (pid, rowCount, processedRecords, table) => { +const insertData = async (pid, rowCount, processedRecords) => { try { await clickhouse.insert({ - table, + table: 'events', format: 'JSONEachRow', values: processedRecords, clickhouse_settings: { @@ -154,6 +154,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'pageview', psid: Math.random() < 0.05 ? faker.helpers.arrayElement(PSIDS) @@ -193,7 +194,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'analytics') + insertData(pid, rowCount, records) } const generateCaptchaData = async (pid, rowCount, from, to) => { @@ -201,6 +202,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'captcha', pid, dv: faker.helpers.arrayElement(DEVICES), br: faker.helpers.arrayElement(BROWSERS), @@ -223,7 +225,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'captcha') + insertData(pid, rowCount, records) } const generateCustomEventsData = async (pid, rowCount, from, to) => { @@ -231,8 +233,9 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'custom_event', pid, - ev: faker.helpers.arrayElement(CUSTOM_EVENTS), + event_name: faker.helpers.arrayElement(CUSTOM_EVENTS), pg: faker.helpers.arrayElement(PAGES), dv: faker.helpers.arrayElement(DEVICES), br: faker.helpers.arrayElement(BROWSERS), @@ -267,7 +270,7 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'customEV') + insertData(pid, rowCount, records) } const generatePerformanceData = async (pid, rowCount, from, to) => { @@ -275,6 +278,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'performance', pid, pg: faker.helpers.arrayElement(PAGES), dv: faker.helpers.arrayElement(DEVICES), @@ -308,7 +312,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'performance') + insertData(pid, rowCount, records) } const main = () => { diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.js new file mode 100644 index 000000000..c2a1d9771 --- /dev/null +++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js @@ -0,0 +1,94 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + `CREATE TABLE IF NOT EXISTS ${dbName}.events + ( + type LowCardinality(String), + pid FixedString(12), + psid Nullable(UInt64), + profileId Nullable(String) CODEC(ZSTD(3)), + host Nullable(String) CODEC(ZSTD(3)), + pg Nullable(String) CODEC(ZSTD(3)), + dv LowCardinality(Nullable(String)), + br LowCardinality(Nullable(String)), + brv Nullable(String) CODEC(ZSTD(3)), + os LowCardinality(Nullable(String)), + osv Nullable(String) CODEC(ZSTD(3)), + lc LowCardinality(Nullable(String)), + ref Nullable(String) CODEC(ZSTD(3)), + so Nullable(String) CODEC(ZSTD(3)), + me Nullable(String) CODEC(ZSTD(3)), + ca Nullable(String) CODEC(ZSTD(3)), + te Nullable(String) CODEC(ZSTD(3)), + co Nullable(String) CODEC(ZSTD(3)), + cc Nullable(FixedString(2)), + rg LowCardinality(Nullable(String)), + rgc LowCardinality(Nullable(String)), + ct Nullable(String) CODEC(ZSTD(3)), + isp LowCardinality(Nullable(String)), + og Nullable(String) CODEC(ZSTD(3)), + ut LowCardinality(Nullable(String)), + ctp LowCardinality(Nullable(String)), + \`meta.key\` Array(String) CODEC(ZSTD(3)), + \`meta.value\` Array(String) CODEC(ZSTD(3)), + importID Nullable(UInt8), + event_name Nullable(String) CODEC(ZSTD(3)), + eid Nullable(FixedString(32)), + error_name Nullable(String) CODEC(ZSTD(3)), + error_message Nullable(String) CODEC(ZSTD(3)), + stackTrace Nullable(String) CODEC(ZSTD(3)), + lineno Nullable(UInt32), + colno Nullable(UInt32), + error_filename Nullable(String) CODEC(ZSTD(3)), + dns Nullable(UInt32), + tls Nullable(UInt32), + conn Nullable(UInt32), + response Nullable(UInt32), + render Nullable(UInt32), + domLoad Nullable(UInt32), + pageLoad Nullable(UInt32), + ttfb Nullable(UInt32), + created DateTime('UTC') CODEC(Delta(4), LZ4) + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(created) + ORDER BY (pid, type, created);`, + + `INSERT INTO ${dbName}.events + (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created) + SELECT + 'pageview', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created + FROM ${dbName}.analytics;`, + + `INSERT INTO ${dbName}.events + (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, event_name, created) + SELECT + 'custom_event', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, ev, created + FROM ${dbName}.customEV;`, + + `INSERT INTO ${dbName}.events + (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, error_name, error_message, stackTrace, lineno, colno, error_filename, created) + SELECT + 'error', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, name, message, stackTrace, lineno, colno, filename, created + FROM ${dbName}.errors;`, + + `INSERT INTO ${dbName}.events + (type, pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created) + SELECT + 'performance', pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created + FROM ${dbName}.performance;`, + + `INSERT INTO ${dbName}.events + (type, pid, dv, br, os, cc, created) + SELECT + 'captcha', pid, dv, br, os, cc, created + FROM ${dbName}.captcha;`, + + `DROP TABLE IF EXISTS ${dbName}.analytics;`, + `DROP TABLE IF EXISTS ${dbName}.customEV;`, + `DROP TABLE IF EXISTS ${dbName}.errors;`, + `DROP TABLE IF EXISTS ${dbName}.performance;`, + `DROP TABLE IF EXISTS ${dbName}.captcha;`, +] + +queriesRunner(queries) diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js index 5018ca040..beb10a635 100644 --- a/backend/migrations/clickhouse/initialise_database.js +++ b/backend/migrations/clickhouse/initialise_database.js @@ -4,51 +4,14 @@ const { queriesRunner, dbName, databaselessQueriesRunner } = require('./setup') const CLICKHOUSE_DB_INIT_QUERIES = [`CREATE DATABASE IF NOT EXISTS ${dbName}`] const CLICKHOUSE_INIT_QUERIES = [ - // The traffic data table - `CREATE TABLE IF NOT EXISTS ${dbName}.analytics - ( - psid Nullable(UInt64), - profileId Nullable(String) CODEC(ZSTD(3)), - pid FixedString(12), - host Nullable(String) CODEC(ZSTD(3)), - pg Nullable(String) CODEC(ZSTD(3)), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - brv Nullable(String) CODEC(ZSTD(3)), - os LowCardinality(Nullable(String)), - osv Nullable(String) CODEC(ZSTD(3)), - lc LowCardinality(Nullable(String)), - ref Nullable(String) CODEC(ZSTD(3)), - so Nullable(String) CODEC(ZSTD(3)), - me Nullable(String) CODEC(ZSTD(3)), - ca Nullable(String) CODEC(ZSTD(3)), - te Nullable(String) CODEC(ZSTD(3)), - co Nullable(String) CODEC(ZSTD(3)), - cc Nullable(FixedString(2)), - rg LowCardinality(Nullable(String)), - rgc LowCardinality(Nullable(String)), - ct Nullable(String) CODEC(ZSTD(3)), - isp LowCardinality(Nullable(String)), - og Nullable(String) CODEC(ZSTD(3)), - ut LowCardinality(Nullable(String)), - ctp LowCardinality(Nullable(String)), - \`meta.key\` Array(String) CODEC(ZSTD(3)), - \`meta.value\` Array(String) CODEC(ZSTD(3)), - importID Nullable(UInt8), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - // Custom events table - `CREATE TABLE IF NOT EXISTS ${dbName}.customEV + `CREATE TABLE IF NOT EXISTS ${dbName}.events ( + type LowCardinality(String), + pid FixedString(12), psid Nullable(UInt64), profileId Nullable(String) CODEC(ZSTD(3)), - pid FixedString(12), host Nullable(String) CODEC(ZSTD(3)), - ev String CODEC(ZSTD(3)), pg Nullable(String) CODEC(ZSTD(3)), dv LowCardinality(Nullable(String)), br LowCardinality(Nullable(String)), @@ -73,29 +36,14 @@ const CLICKHOUSE_INIT_QUERIES = [ \`meta.key\` Array(String) CODEC(ZSTD(3)), \`meta.value\` Array(String) CODEC(ZSTD(3)), importID Nullable(UInt8), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - - // The performance data table - `CREATE TABLE IF NOT EXISTS ${dbName}.performance - ( - pid FixedString(12), - host Nullable(String) CODEC(ZSTD(3)), - pg Nullable(String) CODEC(ZSTD(3)), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - brv Nullable(String) CODEC(ZSTD(3)), - cc Nullable(FixedString(2)), - rg LowCardinality(Nullable(String)), - rgc LowCardinality(Nullable(String)), - ct Nullable(String) CODEC(ZSTD(3)), - isp LowCardinality(Nullable(String)), - og Nullable(String) CODEC(ZSTD(3)), - ut LowCardinality(Nullable(String)), - ctp LowCardinality(Nullable(String)), + event_name Nullable(String) CODEC(ZSTD(3)), + eid Nullable(FixedString(32)), + error_name Nullable(String) CODEC(ZSTD(3)), + error_message Nullable(String) CODEC(ZSTD(3)), + stackTrace Nullable(String) CODEC(ZSTD(3)), + lineno Nullable(UInt32), + colno Nullable(UInt32), + error_filename Nullable(String) CODEC(ZSTD(3)), dns Nullable(UInt32), tls Nullable(UInt32), conn Nullable(UInt32), @@ -108,44 +56,7 @@ const CLICKHOUSE_INIT_QUERIES = [ ) ENGINE = MergeTree() PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - - // Error events table - `CREATE TABLE IF NOT EXISTS ${dbName}.errors - ( - psid Nullable(UInt64), - profileId Nullable(String) CODEC(ZSTD(3)), - eid FixedString(32), - pid FixedString(12), - host Nullable(String) CODEC(ZSTD(3)), - pg Nullable(String) CODEC(ZSTD(3)), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - brv Nullable(String) CODEC(ZSTD(3)), - os LowCardinality(Nullable(String)), - osv Nullable(String) CODEC(ZSTD(3)), - lc LowCardinality(Nullable(String)), - cc LowCardinality(Nullable(FixedString(2))), - rg LowCardinality(Nullable(String)), - rgc LowCardinality(Nullable(String)), - ct Nullable(String) CODEC(ZSTD(3)), - isp LowCardinality(Nullable(String)), - og Nullable(String) CODEC(ZSTD(3)), - ut LowCardinality(Nullable(String)), - ctp LowCardinality(Nullable(String)), - name String CODEC(ZSTD(3)), - message Nullable(String) CODEC(ZSTD(3)), - stackTrace Nullable(String) CODEC(ZSTD(3)), - \`meta.key\` Array(String) CODEC(ZSTD(3)), - \`meta.value\` Array(String) CODEC(ZSTD(3)), - lineno Nullable(UInt32), - colno Nullable(UInt32), - filename Nullable(String) CODEC(ZSTD(3)), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, + ORDER BY (pid, type, created);`, // Error events status table `CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses ( @@ -201,20 +112,6 @@ const CLICKHOUSE_INIT_QUERIES = [ ORDER BY (pid, experimentId, created) TTL created + INTERVAL 1 YEAR;`, - // The CAPTCHA data table - `CREATE TABLE IF NOT EXISTS ${dbName}.captcha - ( - pid FixedString(12), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - os Nullable(String) CODEC(ZSTD(3)), - cc Nullable(FixedString(2)), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - // Bot block events table for advanced bot protection reporting `CREATE TABLE IF NOT EXISTS ${dbName}.bot_blocks ( From 3d4e68647b1886b3f6504dfc1af916e1a683a73a Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Fri, 1 May 2026 18:20:58 +0100 Subject: [PATCH 02/22] Minor code fixes --- .../cloud/src/analytics/analytics.service.ts | 4 +-- .../cloud/src/analytics/utils/transformers.ts | 4 +-- .../src/experiment/experiment.controller.ts | 4 +++ .../apps/cloud/src/goal/goal.controller.ts | 6 ++++ .../apps/cloud/src/project/project.service.ts | 2 +- .../src/task-manager/task-manager.service.ts | 2 +- .../src/analytics/analytics.service.ts | 28 +++++++++++-------- .../src/data-import/mappers/fathom.mapper.ts | 12 +++++--- .../community/src/goal/goal.controller.ts | 6 ++++ .../src/project/project.controller.ts | 6 +++- .../clickhouse/2026_05_01_unify_events.js | 18 ++++++++---- 11 files changed, 64 insertions(+), 28 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 70e85e154..7029ef1fb 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -6027,8 +6027,8 @@ export class AnalyticsService { timeBucket, groupFrom, groupTo, - `FROM events WHERE type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, - 'AND eid = {eid:FixedString(32)}', + `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, + 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}', paramsData, safeTimezone, ChartRenderMode.PERIODICAL, diff --git a/backend/apps/cloud/src/analytics/utils/transformers.ts b/backend/apps/cloud/src/analytics/utils/transformers.ts index 687fb0c3e..7ec1ed491 100644 --- a/backend/apps/cloud/src/analytics/utils/transformers.ts +++ b/backend/apps/cloud/src/analytics/utils/transformers.ts @@ -176,8 +176,8 @@ export const eventTransformer = (opts: EventTransformerOptions) => { error_name: opts.name || null, error_message: opts.message || null, stackTrace: opts.stackTrace || null, - lineno: opts.lineno || null, - colno: opts.colno || null, + lineno: opts.lineno ?? null, + colno: opts.colno ?? null, error_filename: opts.filename || null, created, } diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts index 14e7d58a2..b1679bfaa 100644 --- a/backend/apps/cloud/src/experiment/experiment.controller.ts +++ b/backend/apps/cloud/src/experiment/experiment.controller.ts @@ -948,6 +948,8 @@ export class ExperimentController { let matchCondition = '' if (experiment.goal.matchType === 'exact') { matchCondition = `c.${matchColumn} = {goalValue:String}` + } else if (goalValue.trim() === '') { + matchCondition = '1=0' } else { matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` } @@ -1191,6 +1193,8 @@ export class ExperimentController { let matchCondition = '' if (experiment.goal.matchType === 'exact') { matchCondition = `c.${matchColumn} = {goalValue:String}` + } else if (goalValue.trim() === '') { + matchCondition = '1=0' } else { matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` } diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.ts index 59c897c49..ac4cde6de 100644 --- a/backend/apps/cloud/src/goal/goal.controller.ts +++ b/backend/apps/cloud/src/goal/goal.controller.ts @@ -343,6 +343,9 @@ export class GoalController { params.goalValue = goal.value || '' return { condition: `event_name = {goalValue:String}`, params } } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } + } params.goalValue = goal.value || '' return { condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`, @@ -354,6 +357,9 @@ export class GoalController { params.goalValue = goal.value || '' return { condition: `pg = {goalValue:String}`, params } } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } + } params.goalValue = goal.value || '' return { condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts index 11fe2c715..9127ffbab 100644 --- a/backend/apps/cloud/src/project/project.service.ts +++ b/backend/apps/cloud/src/project/project.service.ts @@ -680,7 +680,7 @@ export class ProjectService { to: string, ): Promise { const queries = [ - "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error') AND created BETWEEN {from:String} AND {to:String}", + "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}", 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', ] diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 1d7b475b2..3f60f61d7 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -402,7 +402,7 @@ export class TaskManagerService { if (goal.matchType === GoalMatchType.CONTAINS) { params[paramKey] = `%${goalValue}%` - return { condition: `${column} LIKE {${paramKey}:String}`, params } + return { condition: `${column} ILIKE {${paramKey}:String}`, params } } // Regex goal diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 2a0ec2231..7607f4235 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -270,6 +270,14 @@ const EXCLUDE_NULL_FOR = [ 'ctp', ] +const ERROR_COLUMN_MAP: Record = { + name: 'error_name', + message: 'error_message', + filename: 'error_filename', +} + +const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type + const generateParamsQuery = ( col: string, subQuery: string, @@ -277,7 +285,8 @@ const generateParamsQuery = ( type: 'traffic' | 'performance' | 'errors' | 'captcha', measure?: PerfMeasure, ): string => { - let columns = [`${col} as name`] + const sqlCol = type === 'errors' ? mapErrorColumn(col) : col + let columns = [`${sqlCol} as name`] // For regions and cities we'll return an array of objects, that will also include the country code and region code // We need the conutry code to display the flag next to the region/city name @@ -301,15 +310,15 @@ const generateParamsQuery = ( const fn = MEASURES_MAP[processedMeasure] - if (col === 'pg' || col === 'host') { + if (sqlCol === 'pg' || sqlCol === 'host') { return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}` } - return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'errors') { - return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'captcha') { @@ -665,13 +674,7 @@ export class AnalyticsService { ) } - // Public ERROR_COLUMNS use legacy short names; map to new column names. - const errorColMap: Record = { - name: 'error_name', - message: 'error_message', - filename: 'error_filename', - } - const sqlCol = errorColMap[type] || type + const sqlCol = mapErrorColumn(type) const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}` @@ -1204,7 +1207,8 @@ export class AnalyticsService { } const { filter, isExclusive, isContains } = converted[column][f] - let sqlColumn = column + let sqlColumn = + dataType === DataType.ERRORS ? mapErrorColumn(column) : column let isArrayDataset = false const param = `qf_${col}_${f}` diff --git a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts index 6f418aaeb..1ae5397ca 100644 --- a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts @@ -1,7 +1,11 @@ import * as fs from 'fs' import { parse } from 'csv-parse' -import { ImportMapper, AnalyticsImportRow } from './mapper.interface' +import { + ImportMapper, + AnalyticsImportRow, + ImportError, +} from './mapper.interface' import { normalizeNull, truncate, @@ -41,7 +45,7 @@ function detectFileType(headers: string[]): { const missing = COMMON_REQUIRED.filter((c) => !normalized.includes(c)) if (missing.length > 0) { - throw new Error( + throw new ImportError( `CSV does not appear to be a Fathom Analytics export. Missing required columns: ${missing.join(', ')}.`, ) } @@ -54,7 +58,7 @@ function detectFileType(headers: string[]): { return { type: 'events', normalized } } - throw new Error( + throw new ImportError( 'CSV does not appear to be a Fathom Analytics export. Expected a pageviews export (with "pageviews" column) or an events export (with "event_name" column).', ) } @@ -214,7 +218,7 @@ export class FathomMapper implements ImportMapper { } if (!headerChecked) { - throw new Error( + throw new ImportError( 'CSV appears empty or is missing a header row. Please upload a CSV export from Fathom Analytics.', ) } diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.ts index 600afe709..4a022b2e8 100644 --- a/backend/apps/community/src/goal/goal.controller.ts +++ b/backend/apps/community/src/goal/goal.controller.ts @@ -263,6 +263,9 @@ export class GoalController { params.goalValue = goal.value || '' return { condition: `event_name = {goalValue:String}`, params } } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } + } params.goalValue = goal.value || '' return { condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`, @@ -274,6 +277,9 @@ export class GoalController { params.goalValue = goal.value || '' return { condition: `pg = {goalValue:String}`, params } } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } + } params.goalValue = goal.value || '' return { condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.ts index fa42f3f2f..1c98b7939 100644 --- a/backend/apps/community/src/project/project.controller.ts +++ b/backend/apps/community/src/project/project.controller.ts @@ -1017,7 +1017,11 @@ export class ProjectController { try { await deleteProjectSharesByProjectClickhouse(id) await clickhouse.command({ - query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event')`, + query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error', 'performance', 'captcha')`, + query_params: { pid: id }, + }) + await clickhouse.command({ + query: `ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}`, query_params: { pid: id }, }) await deleteProjectRedis(id) diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.js index c2a1d9771..590bb1673 100644 --- a/backend/migrations/clickhouse/2026_05_01_unify_events.js +++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js @@ -1,6 +1,9 @@ const { queriesRunner, dbName } = require('./setup') const queries = [ + `DROP TABLE IF EXISTS ${dbName}.events_tmp;`, + `DROP TABLE IF EXISTS ${dbName}.events_backup;`, + `CREATE TABLE IF NOT EXISTS ${dbName}.events ( type LowCardinality(String), @@ -54,41 +57,46 @@ const queries = [ PARTITION BY toYYYYMM(created) ORDER BY (pid, type, created);`, - `INSERT INTO ${dbName}.events + `CREATE TABLE ${dbName}.events_tmp AS ${dbName}.events;`, + + `INSERT INTO ${dbName}.events_tmp (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created) SELECT 'pageview', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created FROM ${dbName}.analytics;`, - `INSERT INTO ${dbName}.events + `INSERT INTO ${dbName}.events_tmp (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, event_name, created) SELECT 'custom_event', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, ev, created FROM ${dbName}.customEV;`, - `INSERT INTO ${dbName}.events + `INSERT INTO ${dbName}.events_tmp (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, error_name, error_message, stackTrace, lineno, colno, error_filename, created) SELECT 'error', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, name, message, stackTrace, lineno, colno, filename, created FROM ${dbName}.errors;`, - `INSERT INTO ${dbName}.events + `INSERT INTO ${dbName}.events_tmp (type, pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created) SELECT 'performance', pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created FROM ${dbName}.performance;`, - `INSERT INTO ${dbName}.events + `INSERT INTO ${dbName}.events_tmp (type, pid, dv, br, os, cc, created) SELECT 'captcha', pid, dv, br, os, cc, created FROM ${dbName}.captcha;`, + `RENAME TABLE ${dbName}.events TO ${dbName}.events_backup, ${dbName}.events_tmp TO ${dbName}.events;`, + `DROP TABLE IF EXISTS ${dbName}.analytics;`, `DROP TABLE IF EXISTS ${dbName}.customEV;`, `DROP TABLE IF EXISTS ${dbName}.errors;`, `DROP TABLE IF EXISTS ${dbName}.performance;`, `DROP TABLE IF EXISTS ${dbName}.captcha;`, + `DROP TABLE IF EXISTS ${dbName}.events_backup;`, ] queriesRunner(queries) From 97d65dabdfc4ffd87297e5b66723e200097e3050 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 02:22:02 +0100 Subject: [PATCH 03/22] Use templated queries --- .../cloud/src/analytics/analytics.service.ts | 82 +++++++++++++------ .../apps/cloud/src/project/project.service.ts | 8 +- .../src/task-manager/task-manager.service.ts | 6 +- .../src/analytics/analytics.service.ts | 9 +- 4 files changed, 71 insertions(+), 34 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 7029ef1fb..fa6177f1f 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -278,6 +278,14 @@ const EXCLUDE_NULL_FOR = [ 'ctp', ] +const ERROR_COLUMN_MAP: Record = { + name: 'error_name', + message: 'error_message', + filename: 'error_filename', +} + +const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type + const generateParamsQuery = ( col: string, subQuery: string, @@ -285,7 +293,8 @@ const generateParamsQuery = ( type: 'traffic' | 'performance' | 'captcha' | 'errors', measure?: PerfMeasure, ): string => { - let columns = [`${col} as name`] + const sqlCol = type === 'errors' ? mapErrorColumn(col) : col + let columns = [`${sqlCol} as name`] // For regions and cities we'll return an array of objects, that will also include the country code and region code // We need the conutry code to display the flag next to the region/city name @@ -309,15 +318,15 @@ const generateParamsQuery = ( const fn = MEASURES_MAP[processedMeasure] - if (col === 'pg' || col === 'host') { + if (sqlCol === 'pg' || sqlCol === 'host') { return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}` } - return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'errors') { - return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'captcha') { @@ -1265,6 +1274,8 @@ export class AnalyticsService { } else if (column === 'ev') { // Public filter contract `ev` maps to the renamed `event_name` column sqlColumn = 'event_name' + } else if (dataType === DataType.ERRORS) { + sqlColumn = mapErrorColumn(column) } const isNullFilter = @@ -2880,13 +2891,7 @@ export class AnalyticsService { ) } - // Public ERROR_COLUMNS use legacy short names; map to new column names. - const errorColMap: Record = { - name: 'error_name', - message: 'error_message', - filename: 'error_filename', - } - const sqlCol = errorColMap[type] || type + const sqlCol = mapErrorColumn(type) const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}` @@ -6259,26 +6264,53 @@ export class AnalyticsService { ` const querySessions = ` - SELECT DISTINCT + SELECT CAST(errors.psid, 'String') as psid, - any(errors.profileId) as profileId, + errors.profileId as profileId, any(analytics.cc) as cc, any(analytics.br) as br, any(analytics.os) as os, - min(errors.created) as firstErrorAt, - max(errors.created) as lastErrorAt, - count(*) as errorCount - FROM events AS errors - LEFT JOIN events AS analytics + errors.firstErrorAt as firstErrorAt, + errors.lastErrorAt as lastErrorAt, + errors.errorCount as errorCount + FROM ( + SELECT + pid, + psid, + any(profileId) AS profileId, + min(created) AS firstErrorAt, + max(created) AS lastErrorAt, + count(*) AS errorCount + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'error' + AND eid = {eid:FixedString(32)} + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY pid, psid + ) AS errors + LEFT JOIN ( + SELECT + pid, + psid, + any(cc) AS cc, + any(br) AS br, + any(os) AS os, + count(*) AS pageviews + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND psid IS NOT NULL + GROUP BY pid, psid + ) AS analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid - AND analytics.type = 'pageview' - WHERE errors.pid = {pid:FixedString(12)} - AND errors.type = 'error' - AND errors.eid = {eid:FixedString(32)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} - GROUP BY errors.psid - ORDER BY lastErrorAt DESC + GROUP BY + errors.psid, + errors.profileId, + errors.firstErrorAt, + errors.lastErrorAt, + errors.errorCount + ORDER BY errors.lastErrorAt DESC LIMIT {take:UInt32} OFFSET {skip:UInt32} ` diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts index 9127ffbab..f974e49c8 100644 --- a/backend/apps/cloud/src/project/project.service.ts +++ b/backend/apps/cloud/src/project/project.service.ts @@ -894,7 +894,11 @@ export class ProjectService { // Process PIDs in chunks for (let i = 0; i < pids.length; i += CHUNK_SIZE) { const pidChunk = pids.slice(i, i + CHUNK_SIZE) - const params = { pids: pidChunk } + const params = { + pids: pidChunk, + monthStart, + monthEnd, + } const query = ` SELECT @@ -904,7 +908,7 @@ export class ProjectService { countIf(type = 'error') AS errors FROM events WHERE pid IN ({pids:Array(FixedString(12))}) - AND created BETWEEN '${monthStart}' AND '${monthEnd}' + AND created BETWEEN {monthStart:String} AND {monthEnd:String} AND type IN ('pageview', 'custom_event', 'captcha', 'error') ` diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 3f60f61d7..5aedd4ee7 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -1996,9 +1996,9 @@ export class TaskManagerService { } // No need to check for performance activity because it's not tracked without tracking analytics - const queryAnalytics = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type = 'pageview' AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCaptcha = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type = 'captcha' AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCustomEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type = 'custom_event' AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryAnalytics = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryCaptcha = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('captcha', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryCustomEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('custom_event', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` // Process project IDs in chunks to avoid ClickHouse field value limit let totalAnalytics = 0 diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 7607f4235..8045092df 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -4400,7 +4400,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC @@ -4491,7 +4491,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'custom_event' + AND type IN ('custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC @@ -4930,8 +4930,8 @@ export class AnalyticsService { timeBucket, groupFrom, groupTo, - `FROM events WHERE type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, - 'AND eid = {eid:FixedString(32)}', + `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, + 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}', paramsData, safeTimezone, ChartRenderMode.PERIODICAL, @@ -4969,6 +4969,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} ` // Get error stats: total errors, unique errors, affected sessions, affected users From 4c046a380c399a13d38574c74be69e7138209a1c Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 15:17:05 +0100 Subject: [PATCH 04/22] Small refactor --- .../src/analytics/analytics.controller.ts | 16 +++-- .../src/task-manager/task-manager.service.ts | 72 +++---------------- .../src/analytics/analytics.service.ts | 6 +- 3 files changed, 26 insertions(+), 68 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index a64f0d2e6..f4cb87509 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -719,7 +719,9 @@ export class AnalyticsController { this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart') - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { pid, groupFrom, groupTo, ...filtersParams }, + } const result = await this.analyticsService.groupChartByTimeBucket( timeBucket, @@ -801,7 +803,9 @@ export class AnalyticsController { const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { pid, groupFrom, groupTo, ...filtersParams }, + } const result = await this.analyticsService.groupPerfByTimeBucket( newTimeBucket, @@ -865,7 +869,9 @@ export class AnalyticsController { 'GET /analytics/performance/chart', ) - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { pid, groupFrom, groupTo, ...filtersParams }, + } const chart = await this.analyticsService.getPerfChartData( timeBucket, @@ -2717,7 +2723,9 @@ export class AnalyticsController { diff, ) - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { pid, groupFrom, groupTo, ...filtersParams }, + } // customEvents comes as a JSON.stringified array from the frontend let customEventsList: string[] = [] diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 5aedd4ee7..6ea2e3363 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -387,11 +387,6 @@ export class TaskManagerService { ): { condition: string; params: Record } { const goalValue = (goal.value ?? '').toString() - // If goal value is blank, never match anything (avoid LIKE '%%') - if (goalValue.trim() === '') { - return { condition: '1=0', params: {} } - } - const params: Record = {} const column = goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg' @@ -401,6 +396,11 @@ export class TaskManagerService { } if (goal.matchType === GoalMatchType.CONTAINS) { + // Avoid wildcard goals matching every row via LIKE '%%'. + if (goalValue.trim() === '') { + return { condition: '1=0', params: {} } + } + params[paramKey] = `%${goalValue}%` return { condition: `${column} ILIKE {${paramKey}:String}`, params } } @@ -1996,60 +1996,10 @@ export class TaskManagerService { } // No need to check for performance activity because it's not tracked without tracking analytics - const queryAnalytics = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCaptcha = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('captcha', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCustomEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('custom_event', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` // Process project IDs in chunks to avoid ClickHouse field value limit - let totalAnalytics = 0 - let totalCaptcha = 0 - let totalCustomEvents = 0 - - for (let i = 0; i < pids.length; i += CHUNK_SIZE) { - const pidChunk = pids.slice(i, i + CHUNK_SIZE) - const queryParams = { - pids: pidChunk, - nineWeeksAgo, - now, - } - - const { data: analyticsResult } = await clickhouse - .query({ - query: queryAnalytics, - query_params: queryParams, - }) - .then((resultSet) => resultSet.json<{ 'count()': number }>()) - - totalAnalytics += analyticsResult[0]['count()'] - - // Early return if we found activity - if (totalAnalytics > 0) { - return - } - } - - for (let i = 0; i < pids.length; i += CHUNK_SIZE) { - const pidChunk = pids.slice(i, i + CHUNK_SIZE) - const queryParams = { - pids: pidChunk, - nineWeeksAgo, - now, - } - - const { data: captchaResult } = await clickhouse - .query({ - query: queryCaptcha, - query_params: queryParams, - }) - .then((resultSet) => resultSet.json<{ 'count()': number }>()) - - totalCaptcha += captchaResult[0]['count()'] - - // Early return if we found activity - if (totalCaptcha > 0) { - return - } - } + let totalEvents = 0 for (let i = 0; i < pids.length; i += CHUNK_SIZE) { const pidChunk = pids.slice(i, i + CHUNK_SIZE) @@ -2059,17 +2009,17 @@ export class TaskManagerService { now, } - const { data: customEventsResult } = await clickhouse + const { data: eventsResult } = await clickhouse .query({ - query: queryCustomEvents, + query: queryEvents, query_params: queryParams, }) .then((resultSet) => resultSet.json<{ 'count()': number }>()) - totalCustomEvents += customEventsResult[0]['count()'] + totalEvents += eventsResult[0]['count()'] // Early return if we found activity - if (totalCustomEvents > 0) { + if (totalEvents > 0) { return } } diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 8045092df..72bd1e44c 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -5166,9 +5166,9 @@ export class AnalyticsService { SELECT DISTINCT CAST(errors.psid, 'String') as psid, any(errors.profileId) as profileId, - any(analytics.cc) as cc, - any(analytics.br) as br, - any(analytics.os) as os, + COALESCE(any(analytics.cc), any(errors.cc)) as cc, + COALESCE(any(analytics.br), any(errors.br)) as br, + COALESCE(any(analytics.os), any(errors.os)) as os, min(errors.created) as firstErrorAt, max(errors.created) as lastErrorAt, count(*) as errorCount From d908a3695d4cc770092eb4c476c5229e02670891 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 15:49:22 +0100 Subject: [PATCH 05/22] Refactor --- .../src/analytics/analytics.controller.ts | 4 ++++ .../cloud/src/analytics/analytics.service.ts | 22 +++++++++++------- .../src/task-manager/task-manager.service.ts | 7 ++++-- .../src/analytics/analytics.controller.ts | 4 ++++ .../src/analytics/analytics.service.ts | 23 +++++++++++-------- 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index f4cb87509..f328189a9 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -2279,6 +2279,8 @@ export class AnalyticsController { DataType.ERRORS, true, ) + const [sessionFiltersQuery, sessionFiltersParams] = + this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo( @@ -2308,6 +2310,8 @@ export class AnalyticsController { groupToUTC, newTimeBucket, parsedOptions.showResolved || false, + sessionFiltersQuery, + sessionFiltersParams, ) } diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index fa6177f1f..8e2a1ec29 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -3590,10 +3590,11 @@ export class AnalyticsService { if (mode === ChartRenderMode.CUMULATIVE) { return ` SELECT - *, - sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews, - sum(uniques) OVER (ORDER BY ${groupBy}) as uniques + ${groupBy}, + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsers FROM (${baseQuery}) + ORDER BY ${groupBy} ` } @@ -5058,7 +5059,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND CAST(psid, 'String') IN ( @@ -5084,7 +5085,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} @@ -5421,7 +5422,6 @@ export class AnalyticsService { any(lc) AS lc FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' AND profileId = {profileId:String} ` @@ -5890,7 +5890,7 @@ export class AnalyticsService { ${ parsedOptions?.showResolved ? '' - : "WHERE status.status = 'active' OR status.status = 'regressed'" + : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL" } GROUP BY errors.eid, status.status ORDER BY last_seen DESC @@ -6059,6 +6059,8 @@ export class AnalyticsService { groupTo: string, timeBucket: string, showResolved: boolean, + sessionFiltersQuery = '', + sessionFiltersParams: Record = {}, ): Promise { const resolvedFilter = showResolved ? '' @@ -6071,6 +6073,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${sessionFiltersQuery} ` // Get error stats: total errors, unique errors, affected sessions, affected users @@ -6167,7 +6170,10 @@ export class AnalyticsService { clickhouse .query({ query: queryTotalSessions, - query_params: paramsData.params, + query_params: { + ...paramsData.params, + ...sessionFiltersParams, + }, }) .then((resultSet) => resultSet.json()) .then(({ data }) => data[0]), diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 6ea2e3363..2ec1a1568 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -1981,9 +1981,12 @@ export class TaskManagerService { }, select: ['id'], }) - const now = dayjs.utc().format('YYYY-MM-DD') + const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss') // a bit more than 2 months ago - const nineWeeksAgo = dayjs.utc().subtract(9, 'w').format('YYYY-MM-DD') + const nineWeeksAgo = dayjs + .utc() + .subtract(9, 'w') + .format('YYYY-MM-DD HH:mm:ss') await mapLimit(users, REPORTS_USERS_CONCURRENCY, async (user) => { const { id } = user diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts index 4816abb97..22898dbd0 100644 --- a/backend/apps/community/src/analytics/analytics.controller.ts +++ b/backend/apps/community/src/analytics/analytics.controller.ts @@ -2131,6 +2131,8 @@ export class AnalyticsController { DataType.ERRORS, true, ) + const [sessionFiltersQuery, sessionFiltersParams] = + this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo( @@ -2160,6 +2162,8 @@ export class AnalyticsController { groupToUTC, newTimeBucket, parsedOptions.showResolved || false, + sessionFiltersQuery, + sessionFiltersParams, ) } diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 72bd1e44c..d5841fc4b 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -3196,10 +3196,11 @@ export class AnalyticsService { if (mode === ChartRenderMode.CUMULATIVE) { return ` SELECT - *, - sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews, - sum(uniques) OVER (ORDER BY ${groupBy}) as uniques + ${groupBy}, + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsers FROM (${baseQuery}) + ORDER BY ${groupBy} ` } @@ -4595,7 +4596,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND CAST(psid, 'String') IN ( @@ -4621,7 +4622,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} @@ -4788,7 +4789,7 @@ export class AnalyticsService { ${ parsedOptions?.showResolved ? '' - : "WHERE status.status = 'active' OR status.status = 'regressed'" + : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL" } GROUP BY errors.eid, status.status ORDER BY last_seen DESC @@ -4957,6 +4958,8 @@ export class AnalyticsService { groupTo: string, timeBucket: string, showResolved: boolean, + sessionFiltersQuery = '', + sessionFiltersParams: Record = {}, ): Promise { const resolvedFilter = showResolved ? '' @@ -4969,7 +4972,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} + ${sessionFiltersQuery} ` // Get error stats: total errors, unique errors, affected sessions, affected users @@ -5066,7 +5069,10 @@ export class AnalyticsService { clickhouse .query({ query: queryTotalSessions, - query_params: paramsData.params, + query_params: { + ...paramsData.params, + ...sessionFiltersParams, + }, }) .then((resultSet) => resultSet.json()) .then(({ data }) => data[0]), @@ -5710,7 +5716,6 @@ export class AnalyticsService { any(lc) AS lc FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' AND profileId = {profileId:String} ` From 9547a1f396171d2c0202da19bfaa043dc2dbc31a Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 16:18:58 +0100 Subject: [PATCH 06/22] Refactor previous code that was referencing legacy table namings --- .../src/analytics/analytics.controller.ts | 56 +-- .../cloud/src/analytics/analytics.service.ts | 409 +++++++++-------- .../src/experiment/experiment.controller.ts | 2 +- .../src/analytics/analytics.controller.ts | 56 +-- .../src/analytics/analytics.service.ts | 411 +++++++++--------- 5 files changed, 469 insertions(+), 465 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index f328189a9..58d406fdf 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -261,7 +261,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) diff = res.diff @@ -289,13 +289,11 @@ export class AnalyticsController { diff, ) - let subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${ - isCaptcha ? 'captcha' : 'pageview' - }' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - - if (customEVFilterApplied && !isCaptcha) { - subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - } + const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery( + filtersQuery, + customEVFilterApplied, + isCaptcha, + ) const paramsData = { params: { @@ -417,12 +415,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -532,12 +533,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -922,7 +926,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) diff = res.diff @@ -2025,7 +2029,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -2110,7 +2114,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) timeBucket = res.timeBucket[0] @@ -2189,7 +2193,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2267,7 +2271,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2352,7 +2356,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2457,7 +2461,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -2536,7 +2540,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) timeBucket = res.timeBucket[0] diff = res.diff @@ -2624,7 +2628,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -2704,7 +2708,7 @@ export class AnalyticsController { if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'customEV', + 'custom_event', ) newTimeBucket = _includes(res.timeBucket, timeBucket) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 8e2a1ec29..1e6cd5517 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -347,6 +347,13 @@ export enum DataType { ERRORS = 'errors', } +type AnalyticsEventType = 'pageview' | 'custom_event' +type EventsAllTimeType = + | AnalyticsEventType + | 'performance' + | 'error' + | 'captcha' + const isValidOrigin = (origins: string[], origin: string) => { const escapeRegex = (str: string) => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -713,6 +720,22 @@ export class AnalyticsService { return CAPTCHA_COLUMNS } + getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType { + return customEVFilterApplied ? 'custom_event' : 'pageview' + } + + buildAnalyticsEventsSubQuery( + filtersQuery: string, + customEVFilterApplied: boolean, + isCaptcha = false, + ): string { + const eventType = isCaptcha + ? 'captcha' + : this.getAnalyticsEventType(customEVFilterApplied) + + return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + } + getGroupFromTo( from: string, to: string, @@ -980,22 +1003,15 @@ export class AnalyticsService { async calculateTimeBucketForAllTime( pid: string, - table: 'analytics' | 'customEV' | 'performance' | 'errors', + eventType: EventsAllTimeType, ): Promise<{ timeBucket: TimeBucketType[] diff: number }> { - const tableToType: Record = { - analytics: 'pageview', - customEV: 'custom_event', - performance: 'performance', - errors: 'error', - } - const { data: fromData } = await clickhouse .query({ query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type = {type:String}`, - query_params: { pid, type: tableToType[table] }, + query_params: { pid, type: eventType }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -2282,9 +2298,7 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - const allTimeType = customEVFilterApplied - ? 'custom_event' - : 'pageview' + const allTimeType = this.getAnalyticsEventType(customEVFilterApplied) const queryAll = ` WITH analytics_counts AS ( @@ -2358,7 +2372,7 @@ export class AnalyticsService { const { diff: allDiff, timeBucket: allowedBuckets } = await this.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.getAnalyticsEventType(customEVFilterApplied), ) const allTimeChartBucket = _includes( allowedBuckets, @@ -2399,7 +2413,7 @@ export class AnalyticsService { ) .format('YYYY-MM-DD HH:mm:ss') - const periodType = customEVFilterApplied ? 'custom_event' : 'pageview' + const periodType = this.getAnalyticsEventType(customEVFilterApplied) const queryCurrent = ` WITH analytics_counts AS ( @@ -2575,6 +2589,57 @@ export class AnalyticsService { return result } + private generateAnalyticsAggregationQueryForScope( + timeBucket: TimeBucketType, + filtersQuery: string, + mode: ChartRenderMode, + customEVFilterApplied: boolean, + ): string { + return customEVFilterApplied + ? this.generateCustomEventsAggregationQuery( + timeBucket, + filtersQuery, + mode, + ) + : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode) + } + + private extractAnalyticsChartDataForScope( + result: Array, + xShifted: string[], + customEVFilterApplied: boolean, + mode: ChartRenderMode, + ): IExtractChartData { + if (customEVFilterApplied) { + const uniques = + this.extractCustomEventsChartData(result, xShifted)?._unknown_event || + [] + const sdur = Array(_size(xShifted)).fill(0) + + return { + visits: uniques, + uniques, + sdur, + } + } + + const chartData = this.extractChartData(result, xShifted) + + // Propagate the previous cumulative value forward for empty buckets + if (mode === ChartRenderMode.CUMULATIVE) { + for (let i = 1; i < chartData.visits.length; ++i) { + if (chartData.visits[i] === 0) { + chartData.visits[i] = chartData.visits[i - 1] + } + if (chartData.uniques[i] === 0) { + chartData.uniques[i] = chartData.uniques[i - 1] + } + } + } + + return chartData + } + /** * Get simplified chart data for dashboard cards * Returns only x (dates) and visits (pageviews) for a lightweight chart @@ -2605,30 +2670,11 @@ export class AnalyticsService { }, } - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - ChartRenderMode.PERIODICAL, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const visits = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - return { x: xShifted, visits } - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, ChartRenderMode.PERIODICAL, + customEVFilterApplied, ) const { data } = await clickhouse @@ -2636,9 +2682,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) + .then((resultSet) => + resultSet.json(), + ) - const { visits } = this.extractChartData(data, xShifted) + const { visits } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + ChartRenderMode.PERIODICAL, + ) return { x: xShifted, visits } } @@ -3754,39 +3807,11 @@ export class AnalyticsService { ): Promise { const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone) - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - mode, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const uniques = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - const sdur = Array(_size(xShifted)).fill(0) - - return Promise.resolve({ - chart: { - x: xShifted, - visits: uniques, - uniques, - sdur, - }, - }) - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, mode, + customEVFilterApplied, ) const { data } = await clickhouse @@ -3794,21 +3819,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) - - const { visits, uniques, sdur } = this.extractChartData(data, xShifted) + .then((resultSet) => + resultSet.json(), + ) - // Propagate the previous cumulative value forward for empty buckets - if (mode === ChartRenderMode.CUMULATIVE) { - for (let i = 1; i < visits.length; ++i) { - if (visits[i] === 0) { - visits[i] = visits[i - 1] - } - if (uniques[i] === 0) { - uniques[i] = uniques[i - 1] - } - } - } + const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + mode, + ) return Promise.resolve({ chart: { @@ -4435,7 +4455,7 @@ export class AnalyticsService { ) if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'customEV') + const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event') diff = res.diff @@ -4527,7 +4547,7 @@ export class AnalyticsService { const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = this.getFiltersQuery(filters, DataType.ANALYTICS) - // We cannot make a query to customEV table using analytics table properties + // Page properties are only present on pageview-scoped analytics rows. if (customEVFilterApplied) { return { result: [], @@ -4536,7 +4556,7 @@ export class AnalyticsService { } if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'analytics') + const res = await this.calculateTimeBucketForAllTime(pid, 'pageview') diff = res.diff @@ -5042,55 +5062,10 @@ export class AnalyticsService { skip = 0, customEVFilterApplied = false, ): Promise { - let primaryEventsSubquery: string - - if (customEVFilterApplied) { - // When filtering by custom events, identify sessions (psids) that have - // matching custom events, then calculate session boundaries from ALL - // pageview/custom_event rows for those sessions. - primaryEventsSubquery = ` - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_for_grouping - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event', 'error') - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND CAST(psid, 'String') IN ( - SELECT DISTINCT CAST(psid, 'String') - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type = 'custom_event' - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - ` - } else { - primaryEventsSubquery = ` - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_for_grouping - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event', 'error') - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ` - } + const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH distinct_sessions_filtered AS ( @@ -5220,45 +5195,50 @@ export class AnalyticsService { return data } - async getProfilesList( - pid: string, + private buildSessionsListPrimaryEventsSubquery( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - profileType: 'all' | 'anonymous' | 'identified' = 'all', - customEVFilterApplied = false, - ): Promise { - let profileTypeFilter = '' - if (profileType === 'anonymous') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` - } else if (profileType === 'identified') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` - } + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + ` + : filtersQuery - let allProfileDataCTE: string + return ` + SELECT + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${scopedSessionFilter} + ` + } - if (customEVFilterApplied) { - allProfileDataCTE = ` - all_profile_data AS ( - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - if(type = 'pageview', 1, 0) AS isPageview, - if(type = 'custom_event', 1, 0) AS isEvent - FROM events - WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} + private buildProfilesListDataCTE( + filtersQuery: string, + profileTypeFilter: string, + customEVFilterApplied: boolean, + ): string { + const scopedProfileFilter = customEVFilterApplied + ? ` AND profileId IN ( SELECT DISTINCT profileId FROM events @@ -5270,9 +5250,10 @@ export class AnalyticsService { ${profileTypeFilter} ${filtersQuery} ) - )` - } else { - allProfileDataCTE = ` + ` + : filtersQuery + + return ` all_profile_data AS ( SELECT profileId, @@ -5291,10 +5272,33 @@ export class AnalyticsService { AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} - ${filtersQuery} + ${scopedProfileFilter} )` + } + + async getProfilesList( + pid: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + profileType: 'all' | 'anonymous' | 'identified' = 'all', + customEVFilterApplied = false, + ): Promise { + let profileTypeFilter = '' + if (profileType === 'anonymous') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` + } else if (profileType === 'identified') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` } + const allProfileDataCTE = this.buildProfilesListDataCTE( + filtersQuery, + profileTypeFilter, + customEVFilterApplied, + ) + const query = ` WITH ${allProfileDataCTE}, profile_aggregated AS ( @@ -5670,35 +5674,12 @@ export class AnalyticsService { } } - async getProfileSessionsList( - pid: string, - profileId: string, + private buildProfileSessionsEventsCTE( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - customEVFilterApplied = false, - ): Promise { - let allProfileEventsCTE: string - - if (customEVFilterApplied) { - allProfileEventsCTE = ` - all_profile_events AS ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM events - WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` AND CAST(psid, 'String') IN ( SELECT DISTINCT CAST(psid, 'String') FROM events @@ -5709,9 +5690,10 @@ export class AnalyticsService { AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) - )` - } else { - allProfileEventsCTE = ` + ` + : filtersQuery + + return ` all_profile_events AS ( SELECT CAST(psid, 'String') AS psidCasted, @@ -5727,9 +5709,24 @@ export class AnalyticsService { AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} + ${scopedSessionFilter} )` - } + } + + async getProfileSessionsList( + pid: string, + profileId: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + customEVFilterApplied = false, + ): Promise { + const allProfileEventsCTE = this.buildProfileSessionsEventsCTE( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH ${allProfileEventsCTE}, diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts index b1679bfaa..0a388ab1b 100644 --- a/backend/apps/cloud/src/experiment/experiment.controller.ts +++ b/backend/apps/cloud/src/experiment/experiment.controller.ts @@ -885,7 +885,7 @@ export class ExperimentController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( experiment.project.id, - 'analytics', + 'pageview', ) diff = res.diff diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts index 22898dbd0..1d5b37746 100644 --- a/backend/apps/community/src/analytics/analytics.controller.ts +++ b/backend/apps/community/src/analytics/analytics.controller.ts @@ -245,7 +245,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - isCaptcha ? 'captcha' : 'analytics', + isCaptcha ? 'captcha' : 'pageview', ) diff = res.diff @@ -273,13 +273,11 @@ export class AnalyticsController { diff, ) - let subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${ - isCaptcha ? 'captcha' : 'pageview' - }' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - - if (customEVFilterApplied && !isCaptcha) { - subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - } + const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery( + filtersQuery, + customEVFilterApplied, + isCaptcha, + ) const paramsData = { params: { @@ -408,12 +406,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -521,12 +522,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -831,7 +835,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) diff = res.diff @@ -1550,7 +1554,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -1654,7 +1658,7 @@ export class AnalyticsController { if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'customEV', + 'custom_event', ) newTimeBucket = _includes(res.timeBucket, timeBucket) @@ -1966,7 +1970,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) timeBucket = res.timeBucket[0] @@ -2043,7 +2047,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2119,7 +2123,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2202,7 +2206,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2275,7 +2279,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -2352,7 +2356,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) timeBucket = res.timeBucket[0] diff = res.diff @@ -2438,7 +2442,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index d5841fc4b..47395aa06 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -339,6 +339,13 @@ export enum DataType { CAPTCHA = 'captcha', } +type AnalyticsEventType = 'pageview' | 'custom_event' +type EventsAllTimeType = + | AnalyticsEventType + | 'performance' + | 'error' + | 'captcha' + const isValidOrigin = (origins: string[], origin: string) => { const escapeRegex = (str: string) => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -734,6 +741,22 @@ export class AnalyticsService { return CAPTCHA_COLUMNS } + getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType { + return customEVFilterApplied ? 'custom_event' : 'pageview' + } + + buildAnalyticsEventsSubQuery( + filtersQuery: string, + customEVFilterApplied: boolean, + isCaptcha = false, + ): string { + const eventType = isCaptcha + ? 'captcha' + : this.getAnalyticsEventType(customEVFilterApplied) + + return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + } + getGroupFromTo( from: string, to: string, @@ -998,23 +1021,15 @@ export class AnalyticsService { async calculateTimeBucketForAllTime( pid: string, - table: 'analytics' | 'customEV' | 'performance' | 'errors' | 'captcha', + eventType: EventsAllTimeType, ): Promise<{ timeBucket: TimeBucketType[] diff: number }> { - const tableToType: Record = { - analytics: 'pageview', - customEV: 'custom_event', - performance: 'performance', - errors: 'error', - captcha: 'captcha', - } - const { data: fromData } = await clickhouse .query({ query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type = {type:String}`, - query_params: { pid, type: tableToType[table] }, + query_params: { pid, type: eventType }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -1956,9 +1971,7 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - const allTimeType = customEVFilterApplied - ? 'custom_event' - : 'pageview' + const allTimeType = this.getAnalyticsEventType(customEVFilterApplied) const queryAll = ` WITH analytics_counts AS ( @@ -2032,7 +2045,7 @@ export class AnalyticsService { const { diff: allDiff, timeBucket: allowedBuckets } = await this.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.getAnalyticsEventType(customEVFilterApplied), ) const allTimeChartBucket = _includes( allowedBuckets, @@ -2073,7 +2086,7 @@ export class AnalyticsService { ) .format('YYYY-MM-DD HH:mm:ss') - const periodType = customEVFilterApplied ? 'custom_event' : 'pageview' + const periodType = this.getAnalyticsEventType(customEVFilterApplied) const queryCurrent = ` WITH analytics_counts AS ( @@ -2249,6 +2262,57 @@ export class AnalyticsService { return result } + private generateAnalyticsAggregationQueryForScope( + timeBucket: TimeBucketType, + filtersQuery: string, + mode: ChartRenderMode, + customEVFilterApplied: boolean, + ): string { + return customEVFilterApplied + ? this.generateCustomEventsAggregationQuery( + timeBucket, + filtersQuery, + mode, + ) + : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode) + } + + private extractAnalyticsChartDataForScope( + result: Array, + xShifted: string[], + customEVFilterApplied: boolean, + mode: ChartRenderMode, + ): IExtractChartData { + if (customEVFilterApplied) { + const uniques = + this.extractCustomEventsChartData(result, xShifted)?._unknown_event || + [] + const sdur = Array(_size(xShifted)).fill(0) + + return { + visits: uniques, + uniques, + sdur, + } + } + + const chartData = this.extractChartData(result, xShifted) + + // Propagate the previous cumulative value forward for empty buckets + if (mode === ChartRenderMode.CUMULATIVE) { + for (let i = 1; i < chartData.visits.length; ++i) { + if (chartData.visits[i] === 0) { + chartData.visits[i] = chartData.visits[i - 1] + } + if (chartData.uniques[i] === 0) { + chartData.uniques[i] = chartData.uniques[i - 1] + } + } + } + + return chartData + } + /** * Get simplified chart data for dashboard cards * Returns only x (dates) and visits (pageviews) for a lightweight chart @@ -2279,30 +2343,11 @@ export class AnalyticsService { }, } - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - ChartRenderMode.PERIODICAL, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const visits = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - return { x: xShifted, visits } - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, ChartRenderMode.PERIODICAL, + customEVFilterApplied, ) const { data } = await clickhouse @@ -2310,9 +2355,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) + .then((resultSet) => + resultSet.json(), + ) - const { visits } = this.extractChartData(data, xShifted) + const { visits } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + ChartRenderMode.PERIODICAL, + ) return { x: xShifted, visits } } @@ -3360,39 +3412,11 @@ export class AnalyticsService { ): Promise { const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone) - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - mode, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const uniques = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - const sdur = Array(_size(xShifted)).fill(0) - - return Promise.resolve({ - chart: { - x: xShifted, - visits: uniques, - uniques, - sdur, - }, - }) - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, mode, + customEVFilterApplied, ) const { data } = await clickhouse @@ -3400,21 +3424,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) - - const { visits, uniques, sdur } = this.extractChartData(data, xShifted) + .then((resultSet) => + resultSet.json(), + ) - // Propagate the previous cumulative value forward for empty buckets - if (mode === ChartRenderMode.CUMULATIVE) { - for (let i = 1; i < visits.length; ++i) { - if (visits[i] === 0) { - visits[i] = visits[i - 1] - } - if (uniques[i] === 0) { - uniques[i] = uniques[i - 1] - } - } - } + const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + mode, + ) return Promise.resolve({ chart: { @@ -4049,7 +4068,7 @@ export class AnalyticsService { ) if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'customEV') + const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event') diff = res.diff @@ -4141,7 +4160,7 @@ export class AnalyticsService { const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = this.getFiltersQuery(filters, DataType.ANALYTICS) - // We cannot make a query to customEV table using analytics table properties + // Page properties are only present on pageview-scoped analytics rows. if (customEVFilterApplied) { return { result: [], @@ -4150,7 +4169,7 @@ export class AnalyticsService { } if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'analytics') + const res = await this.calculateTimeBucketForAllTime(pid, 'pageview') diff = res.diff @@ -4582,52 +4601,10 @@ export class AnalyticsService { skip = 0, customEVFilterApplied = false, ): Promise { - let primaryEventsSubquery: string - - if (customEVFilterApplied) { - primaryEventsSubquery = ` - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_for_grouping - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event', 'error') - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND CAST(psid, 'String') IN ( - SELECT DISTINCT CAST(psid, 'String') - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type = 'custom_event' - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - ` - } else { - primaryEventsSubquery = ` - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_for_grouping - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event', 'error') - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ` - } + const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH distinct_sessions_filtered AS ( @@ -4734,6 +4711,43 @@ export class AnalyticsService { return data } + private buildSessionsListPrimaryEventsSubquery( + filtersQuery: string, + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + ` + : filtersQuery + + return ` + SELECT + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${scopedSessionFilter} + ` + } + async getErrorsList( options: string, filtersQuery: string, @@ -5513,45 +5527,13 @@ export class AnalyticsService { } } - async getProfilesList( - pid: string, + private buildProfilesListDataCTE( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - profileType: 'all' | 'anonymous' | 'identified' = 'all', - customEVFilterApplied = false, - ): Promise { - let profileTypeFilter = '' - if (profileType === 'anonymous') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` - } else if (profileType === 'identified') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` - } - - let allProfileDataCTE: string - - if (customEVFilterApplied) { - allProfileDataCTE = ` - all_profile_data AS ( - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - if(type = 'pageview', 1, 0) AS isPageview, - if(type = 'custom_event', 1, 0) AS isEvent - FROM events - WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} + profileTypeFilter: string, + customEVFilterApplied: boolean, + ): string { + const scopedProfileFilter = customEVFilterApplied + ? ` AND profileId IN ( SELECT DISTINCT profileId FROM events @@ -5563,9 +5545,10 @@ export class AnalyticsService { ${profileTypeFilter} ${filtersQuery} ) - )` - } else { - allProfileDataCTE = ` + ` + : filtersQuery + + return ` all_profile_data AS ( SELECT profileId, @@ -5584,10 +5567,33 @@ export class AnalyticsService { AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} - ${filtersQuery} + ${scopedProfileFilter} )` + } + + async getProfilesList( + pid: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + profileType: 'all' | 'anonymous' | 'identified' = 'all', + customEVFilterApplied = false, + ): Promise { + let profileTypeFilter = '' + if (profileType === 'anonymous') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` + } else if (profileType === 'identified') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` } + const allProfileDataCTE = this.buildProfilesListDataCTE( + filtersQuery, + profileTypeFilter, + customEVFilterApplied, + ) + const query = ` WITH ${allProfileDataCTE}, profile_aggregated AS ( @@ -5939,35 +5945,12 @@ export class AnalyticsService { } } - async getProfileSessionsList( - pid: string, - profileId: string, + private buildProfileSessionsEventsCTE( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - customEVFilterApplied = false, - ): Promise { - let allProfileEventsCTE: string - - if (customEVFilterApplied) { - allProfileEventsCTE = ` - all_profile_events AS ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM events - WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` AND CAST(psid, 'String') IN ( SELECT DISTINCT CAST(psid, 'String') FROM events @@ -5978,9 +5961,10 @@ export class AnalyticsService { AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) - )` - } else { - allProfileEventsCTE = ` + ` + : filtersQuery + + return ` all_profile_events AS ( SELECT CAST(psid, 'String') AS psidCasted, @@ -5996,9 +5980,24 @@ export class AnalyticsService { AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} + ${scopedSessionFilter} )` - } + } + + async getProfileSessionsList( + pid: string, + profileId: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + customEVFilterApplied = false, + ): Promise { + const allProfileEventsCTE = this.buildProfileSessionsEventsCTE( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH ${allProfileEventsCTE}, From 4503afa28d19114ac91a3b9588dcc7e94a2f7fe7 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 18:23:56 +0100 Subject: [PATCH 07/22] fixes --- .../src/analytics/analytics.controller.ts | 17 ++- .../cloud/src/analytics/analytics.service.ts | 47 ++++-- .../src/experiment/experiment.controller.ts | 138 +++++++++--------- 3 files changed, 118 insertions(+), 84 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index 58d406fdf..146540b24 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -258,10 +258,19 @@ export class AnalyticsController { let diff + const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = + this.analyticsService.getFiltersQuery( + filters, + isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, + ) + if (period === 'all') { + const eventType = isCaptcha + ? 'captcha' + : this.analyticsService.getAnalyticsEventType(customEVFilterApplied) const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'pageview', + eventType, ) diff = res.diff @@ -272,12 +281,6 @@ export class AnalyticsController { allowedTumebucketForPeriodAll = res.timeBucket } - const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = - this.analyticsService.getFiltersQuery( - filters, - isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, - ) - const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFrom, groupTo, groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo( diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 1e6cd5517..624c4dd97 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -2320,6 +2320,14 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) as duration FROM sessions WHERE pid = {pid:FixedString(12)} + AND psid IN ( + SELECT DISTINCT psid + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' + AND psid IS NOT NULL + ${filtersQuery} + ) GROUP BY psid ) ) @@ -2610,21 +2618,19 @@ export class AnalyticsService { customEVFilterApplied: boolean, mode: ChartRenderMode, ): IExtractChartData { + const chartData = this.extractChartData(result, xShifted) + if (customEVFilterApplied) { - const uniques = + const visits = this.extractCustomEventsChartData(result, xShifted)?._unknown_event || [] - const sdur = Array(_size(xShifted)).fill(0) - return { - visits: uniques, - uniques, - sdur, - } + chartData.visits = Array.from( + { length: _size(xShifted) }, + (_, index) => visits[index] || 0, + ) } - const chartData = this.extractChartData(result, xShifted) - // Propagate the previous cumulative value forward for empty buckets if (mode === ChartRenderMode.CUMULATIVE) { for (let i = 1; i < chartData.visits.length; ++i) { @@ -3475,16 +3481,31 @@ export class AnalyticsService { const baseQuery = ` SELECT ${selector}, + avgOrNull(sessions_data.duration) as sdur, + count(DISTINCT psid) as uniques, count() as count FROM ( - SELECT *, - ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created + SELECT + pid, + psid, + ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery + LEFT JOIN ( + SELECT + pid, + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) as duration + FROM sessions + WHERE pid = {pid:FixedString(12)} + GROUP BY pid, psid + ) as sessions_data + ON subquery.pid = sessions_data.pid + AND subquery.psid = sessions_data.psid GROUP BY ${groupBy} ORDER BY ${groupBy} ` @@ -3493,7 +3514,8 @@ export class AnalyticsService { return ` SELECT *, - sum(count) OVER (ORDER BY ${groupBy}) as count + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(uniques) OVER (ORDER BY ${groupBy}) as uniques FROM (${baseQuery}) ` } @@ -6303,6 +6325,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid, psid ) AS analytics ON errors.psid = analytics.psid diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts index 0a388ab1b..869176359 100644 --- a/backend/apps/cloud/src/experiment/experiment.controller.ts +++ b/backend/apps/cloud/src/experiment/experiment.controller.ts @@ -60,6 +60,7 @@ import { } from './dto/experiment.dto' import { ExperimentService } from './experiment.service' import { GoalService } from '../goal/goal.service' +import { Goal, GoalMatchType, GoalType } from '../goal/entity/goal.entity' import { FeatureFlagService } from '../feature-flag/feature-flag.service' import { FeatureFlag, @@ -73,6 +74,15 @@ import { Pagination } from '../common/pagination' const EXPERIMENTS_MAXIMUM = 20 // Maximum experiments per project const FEATURE_FLAG_KEY_REGEX = /^[a-zA-Z0-9_-]+$/ +type GoalEventConditions = { + eventType: 'pageview' | 'custom_event' + matchColumn: 'pg' | 'event_name' + matchCondition: string + metaCondition: string + metaParams: Record + goalValue: string +} + @ApiTags('Experiment') @Controller(['experiment', 'v1/experiment']) export class ExperimentController { @@ -881,11 +891,14 @@ export class ExperimentController { timeBucketParam || getLowestPossibleTimeBucket(period, from, to) let allowedTimeBucketForPeriodAll: TimeBucketType[] | undefined let diff: number | undefined + const goalConditions = experiment.goal + ? this.buildGoalEventConditions(experiment.goal) + : null if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( experiment.project.id, - 'pageview', + goalConditions?.eventType || 'pageview', ) diff = res.diff @@ -938,39 +951,14 @@ export class ExperimentController { } let conversionsData: { variantKey: string; conversions: number }[] = [] - if (experiment.goal) { - const goalType = experiment.goal.type - const eventType = - goalType === 'custom_event' ? 'custom_event' : 'pageview' - const matchColumn = goalType === 'custom_event' ? 'event_name' : 'pg' - const goalValue = experiment.goal.value || '' - - let matchCondition = '' - if (experiment.goal.matchType === 'exact') { - matchCondition = `c.${matchColumn} = {goalValue:String}` - } else if (goalValue.trim() === '') { - matchCondition = '1=0' - } else { - matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` - } - - let metaCondition = '' - const metaParams: Record = {} - const metadataFilters = experiment.goal.metadataFilters - if (metadataFilters && metadataFilters.length > 0) { - const conditions: string[] = [] - metadataFilters.forEach((filter, index) => { - const keyParam = `metaKey${index}` - const valueParam = `metaValue${index}` - metaParams[keyParam] = filter.key - metaParams[valueParam] = filter.value - conditions.push( - `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`, - ) - }) - - metaCondition = `AND (${conditions.join(' AND ')})` - } + if (goalConditions) { + const { + eventType, + matchCondition, + metaCondition, + metaParams, + goalValue, + } = goalConditions const conversionsQuery = ` SELECT @@ -1108,6 +1096,50 @@ export class ExperimentController { } } + private buildGoalEventConditions(goal: Goal): GoalEventConditions { + const eventType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' + const matchColumn = + goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg' + const goalValue = goal.value || '' + const metaParams: Record = {} + + let matchCondition = '' + if (goal.matchType === GoalMatchType.EXACT) { + matchCondition = `c.${matchColumn} = {goalValue:String}` + } else if (goalValue.trim() === '') { + matchCondition = '1=0' + } else { + matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` + } + + let metaCondition = '' + if (goal.metadataFilters && goal.metadataFilters.length > 0) { + const conditions: string[] = [] + + goal.metadataFilters.forEach((filter, index) => { + const keyParam = `metaKey${index}` + const valueParam = `metaValue${index}` + metaParams[keyParam] = filter.key + metaParams[valueParam] = filter.value + conditions.push( + `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`, + ) + }) + + metaCondition = `AND (${conditions.join(' AND ')})` + } + + return { + eventType, + matchColumn, + matchCondition, + metaCondition, + metaParams, + goalValue, + } + } + /** * Generate time-series chart data for experiment win probabilities */ @@ -1180,42 +1212,18 @@ export class ExperimentController { let conversionsData: any[] = [] if (experiment.goal) { - const goalType = experiment.goal.type - const eventType = - goalType === 'custom_event' ? 'custom_event' : 'pageview' - const matchColumn = goalType === 'custom_event' ? 'event_name' : 'pg' - const goalValue = experiment.goal.value || '' + const { + eventType, + matchCondition, + metaCondition, + metaParams, + goalValue, + } = this.buildGoalEventConditions(experiment.goal) const conversionsDateColumnsSelect = this.getTimeBucketDateColumnsSelect( timeBucket, 'firstConversion', ) - let matchCondition = '' - if (experiment.goal.matchType === 'exact') { - matchCondition = `c.${matchColumn} = {goalValue:String}` - } else if (goalValue.trim() === '') { - matchCondition = '1=0' - } else { - matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` - } - - let metaCondition = '' - const metaParams: Record = {} - const metadataFilters = experiment.goal.metadataFilters - if (metadataFilters && metadataFilters.length > 0) { - const conditions: string[] = [] - metadataFilters.forEach((filter, index) => { - const keyParam = `metaKey${index}` - const valueParam = `metaValue${index}` - metaParams[keyParam] = filter.key - metaParams[valueParam] = filter.value - conditions.push( - `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`, - ) - }) - metaCondition = `AND (${conditions.join(' AND ')})` - } - const conversionsQuery = ` SELECT ${conversionsDateColumnsSelect}, From 3f40c8e93f1f7f2aa3e9b3921ec757aec1e68b9d Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 20:22:26 +0100 Subject: [PATCH 08/22] clean-up whitespaces --- .../cloud/src/analytics/analytics.service.ts | 62 +++++++++---------- .../src/analytics/analytics.service.ts | 38 ++++++------ 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 624c4dd97..3417ed5fd 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -2060,11 +2060,11 @@ export class AnalyticsService { groupTo: string, ): Promise<{ cc: string; count: number } | null> { const query = ` - SELECT + SELECT cc, count() as count FROM events - WHERE + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2096,12 +2096,12 @@ export class AnalyticsService { const query = ` WITH counts AS ( - SELECT + SELECT pid, cc, count() as cnt FROM events - WHERE + WHERE pid IN {pids:Array(FixedString(12))} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2146,11 +2146,11 @@ export class AnalyticsService { groupTo: string, ): Promise<{ count: number; uniqueErrors: number }> { const query = ` - SELECT + SELECT count() as count, count(DISTINCT eid) as uniqueErrors FROM events - WHERE + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2178,12 +2178,12 @@ export class AnalyticsService { } const query = ` - SELECT + SELECT pid, count() as count, count(DISTINCT eid) as uniqueErrors FROM events - WHERE + WHERE pid IN {pids:Array(FixedString(12))} AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2227,11 +2227,11 @@ export class AnalyticsService { } const query = ` - SELECT + SELECT pid, uniqExact(psid) as totalSessions FROM events - WHERE + WHERE pid IN {pids:Array(FixedString(12))} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2975,9 +2975,9 @@ export class AnalyticsService { const safeVersionCol = column === 'br' ? 'brv' : 'osv' const query = ` - SELECT ${column}, ${safeVersionCol} + SELECT ${column}, ${safeVersionCol} FROM events - WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL + WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL GROUP BY ${column}, ${safeVersionCol} ` @@ -3037,7 +3037,7 @@ export class AnalyticsService { const query = ` WITH ${withClauses.join(',\n')} - SELECT + SELECT column_name, name, count, @@ -3058,7 +3058,7 @@ export class AnalyticsService { } return ` - SELECT + SELECT '${col}' as column_name, name, count, @@ -3734,7 +3734,7 @@ export class AnalyticsService { ): Promise<{ name: string; count: number }[]> { const query = ` WITH session_first_pages AS ( - SELECT + SELECT psid, argMin(pg, created) as entry_page ${subQuery} @@ -5093,7 +5093,7 @@ export class AnalyticsService { WITH distinct_sessions_filtered AS ( SELECT psidCasted, - pid, + pid, any(cc) AS cc, any(os) AS os, any(br) AS br, @@ -5103,32 +5103,32 @@ export class AnalyticsService { GROUP BY psidCasted, pid ), pageview_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count + pid, + count() as count FROM events - WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), event_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count + pid, + count() as count FROM events - WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), error_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count + pid, + count() as count FROM events - WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -5153,9 +5153,9 @@ export class AnalyticsService { GROUP BY psidCasted, pid ), session_duration_agg AS ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, + SELECT + CAST(psid, 'String') AS psidCasted, + pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, argMax(profileId, lastSeen) as profileId FROM sessions diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 47395aa06..7d2407008 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -2652,7 +2652,7 @@ export class AnalyticsService { const query = ` WITH ${withClauses.join(',\n')} - SELECT + SELECT column_name, name, count, @@ -2673,7 +2673,7 @@ export class AnalyticsService { } return ` - SELECT + SELECT '${col}' as column_name, name, count, @@ -3317,7 +3317,7 @@ export class AnalyticsService { ): Promise<{ name: string; count: number }[]> { const query = ` WITH session_first_pages AS ( - SELECT + SELECT psid, argMin(pg, created) as entry_page ${subQuery} @@ -4610,7 +4610,7 @@ export class AnalyticsService { WITH distinct_sessions_filtered AS ( SELECT psidCasted, - pid, + pid, any(cc) AS cc, any(os) AS os, any(br) AS br, @@ -4620,39 +4620,39 @@ export class AnalyticsService { GROUP BY psidCasted, pid ), pageview_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count + pid, + count() as count FROM events - WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), event_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count + pid, + count() as count FROM events - WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), error_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count + pid, + count() as count FROM events - WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), session_duration_agg AS ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, + SELECT + CAST(psid, 'String') AS psidCasted, + pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, argMax(profileId, lastSeen) as profileId FROM sessions From d31db177f58ee3a062c54ca63cd41149051dbcb8 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sat, 2 May 2026 21:45:32 +0100 Subject: [PATCH 09/22] Refer new events table --- admin/src/db/clickhouse.ts | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/admin/src/db/clickhouse.ts b/admin/src/db/clickhouse.ts index 2e4f960b3..ac48e9657 100644 --- a/admin/src/db/clickhouse.ts +++ b/admin/src/db/clickhouse.ts @@ -34,6 +34,17 @@ export interface ClickHouseStats { tables: TableStats[]; } +const EVENT_TABLE = "events"; +const PROJECT_ACTIVITY_EVENT_TYPES = [ + "pageview", + "custom_event", + "error", + "captcha", +]; +const PROJECT_ACTIVITY_TYPE_LIST = PROJECT_ACTIVITY_EVENT_TYPES.map( + (type) => `'${type}'` +).join(", "); + function formatBytes(bytes: number): string { if (bytes === 0) return "0 B"; const k = 1024; @@ -47,20 +58,16 @@ export async function getClickHouseStats(): Promise { const database = process.env.CLICKHOUSE_DATABASE || "analytics"; const tables = [ - "analytics", - "customEV", - "performance", - "errors", - "captcha", + EVENT_TABLE, "sessions", "revenue", "feature_flag_evaluations", "experiment_exposures", "error_statuses", + "bot_blocks", ]; const tableStats: TableStats[] = []; - let totalEvents = 0; for (const table of tables) { try { @@ -93,8 +100,6 @@ export async function getClickHouseStats(): Promise { bytes, bytesFormatted: formatBytes(bytes), }); - - totalEvents += rows; } catch { // Table might not exist, skip it tableStats.push({ @@ -106,6 +111,9 @@ export async function getClickHouseStats(): Promise { } } + const totalEvents = + tableStats.find((table) => table.table === EVENT_TABLE)?.rows || 0; + return { totalEvents, tables: tableStats, @@ -118,7 +126,12 @@ export async function getProjectEventCount(pid: string): Promise { try { const result = await client.query({ - query: `SELECT count() as count FROM ${database}.analytics WHERE pid = {pid:String}`, + query: ` + SELECT count() as count + FROM ${database}.${EVENT_TABLE} + WHERE pid = {pid:FixedString(12)} + AND type IN (${PROJECT_ACTIVITY_TYPE_LIST}) + `, query_params: { pid }, format: "JSONEachRow", }); @@ -158,8 +171,9 @@ export async function getTopProjectsByEvents( SELECT pid, count() as eventCount - FROM ${database}.analytics + FROM ${database}.${EVENT_TABLE} WHERE created >= now() - INTERVAL ${days} DAY + AND type IN (${PROJECT_ACTIVITY_TYPE_LIST}) GROUP BY pid ORDER BY eventCount DESC LIMIT ${limit} @@ -183,8 +197,9 @@ export async function getProjectsWithRecentEvents( const result = await client.query({ query: ` SELECT DISTINCT pid - FROM ${database}.analytics + FROM ${database}.${EVENT_TABLE} WHERE created >= now() - INTERVAL ${days} DAY + AND type IN (${PROJECT_ACTIVITY_TYPE_LIST}) `, format: "JSONEachRow", }); From b7c801c9b7bac8b90d95b7149fb505e5f337e6ee Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sun, 3 May 2026 03:36:37 +0100 Subject: [PATCH 10/22] fix: Missing custom events / sale events from pageflow --- .../cloud/src/analytics/analytics.service.ts | 42 +++++++++++++------ .../src/analytics/analytics.service.ts | 38 ++++++++++++----- .../clickhouse/2026_05_01_unify_events.js | 10 ++--- web/app/api/api.server.ts | 5 ++- .../tabs/Sessions/SessionDetailView.tsx | 4 +- .../Project/tabs/Sessions/SessionsView.tsx | 8 +++- 6 files changed, 74 insertions(+), 33 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 3417ed5fd..fb6f2a169 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -4818,31 +4818,47 @@ export class AnalyticsService { const queryPages = ` WITH events_with_meta AS ( SELECT - multiIf(type = 'pageview', 'pageview', type = 'custom_event', 'event', 'error') AS type, + multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type, multiIf( - type = 'pageview', toString(pg), - type = 'custom_event', toString(event_name), + event_type = 'pageview', toString(pg), + event_type = 'custom_event', toString(event_name), toString(error_name) ) AS value, toTimeZone(created, {timezone:String}) AS created, pid, toString(psid) AS psid, if( - type = 'error', + event_type = 'error', [ tuple('message', COALESCE(error_message, '')), tuple('lineno', toString(COALESCE(lineno, 0))), tuple('colno', toString(COALESCE(colno, 0))), tuple('filename', COALESCE(error_filename, '')) ], - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) + arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value)) ) AS metadata - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event', 'error') - AND psid IS NOT NULL - AND toString(psid) = {psid:String} + FROM ( + SELECT + type AS event_type, + pg, + event_name, + error_name, + error_message, + lineno, + colno, + error_filename, + created, + pid, + psid, + meta.key AS meta_key, + meta.value AS meta_value + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} + ) UNION ALL @@ -4897,7 +4913,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC @@ -4994,7 +5010,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'custom_event' + AND type IN ('custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 7d2407008..c1b82f71c 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -4378,31 +4378,47 @@ export class AnalyticsService { const queryPages = ` WITH events_with_meta AS ( SELECT - multiIf(type = 'pageview', 'pageview', type = 'custom_event', 'event', 'error') AS type, + multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type, multiIf( - type = 'pageview', toString(pg), - type = 'custom_event', toString(event_name), + event_type = 'pageview', toString(pg), + event_type = 'custom_event', toString(event_name), toString(error_name) ) AS value, toTimeZone(created, {timezone:String}) AS created, pid, toString(psid) AS psid, if( - type = 'error', + event_type = 'error', [ tuple('message', COALESCE(error_message, '')), tuple('lineno', toString(COALESCE(lineno, 0))), tuple('colno', toString(COALESCE(colno, 0))), tuple('filename', COALESCE(error_filename, '')) ], - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) + arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value)) ) AS metadata - FROM events - WHERE - pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event', 'error') - AND psid IS NOT NULL - AND toString(psid) = {psid:String} + FROM ( + SELECT + type AS event_type, + pg, + event_name, + error_name, + error_message, + lineno, + colno, + error_filename, + created, + pid, + psid, + meta.key AS meta_key, + meta.value AS meta_value + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} + ) ) SELECT diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.js index 590bb1673..ef8689b0d 100644 --- a/backend/migrations/clickhouse/2026_05_01_unify_events.js +++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js @@ -91,11 +91,11 @@ const queries = [ `RENAME TABLE ${dbName}.events TO ${dbName}.events_backup, ${dbName}.events_tmp TO ${dbName}.events;`, - `DROP TABLE IF EXISTS ${dbName}.analytics;`, - `DROP TABLE IF EXISTS ${dbName}.customEV;`, - `DROP TABLE IF EXISTS ${dbName}.errors;`, - `DROP TABLE IF EXISTS ${dbName}.performance;`, - `DROP TABLE IF EXISTS ${dbName}.captcha;`, + // `DROP TABLE IF EXISTS ${dbName}.analytics;`, + // `DROP TABLE IF EXISTS ${dbName}.customEV;`, + // `DROP TABLE IF EXISTS ${dbName}.errors;`, + // `DROP TABLE IF EXISTS ${dbName}.performance;`, + // `DROP TABLE IF EXISTS ${dbName}.captcha;`, `DROP TABLE IF EXISTS ${dbName}.events_backup;`, ] diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.ts index 3eb66d8d5..052d09680 100644 --- a/web/app/api/api.server.ts +++ b/web/app/api/api.server.ts @@ -1036,10 +1036,12 @@ export async function getFunnelSessionsServer( } interface PageflowItem { - type: 'pageview' | 'event' | 'error' + type: 'pageview' | 'event' | 'error' | 'sale' | 'refund' value: string created: string metadata?: { key: string; value: string }[] + amount?: number + currency?: string } export interface SessionDetailsResponse { @@ -1067,6 +1069,7 @@ export interface SessionDetailsResponse { refunds?: number created: string sdur?: number + isLive?: boolean profileId: string | null isIdentified: 1 | 0 isFirstSession: 1 | 0 diff --git a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx index 0dd99425a..7a3573cc7 100644 --- a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx +++ b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx @@ -42,10 +42,12 @@ import { Pageflow } from './Pageflow' import { SessionChart } from './SessionChart' interface PageflowItem { - type: 'pageview' | 'event' | 'error' + type: 'pageview' | 'event' | 'error' | 'sale' | 'refund' value: string created: string metadata?: { key: string; value: string }[] + amount?: number + currency?: string } interface SessionDetailViewProps { diff --git a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx index c9631ed51..e43ad3018 100644 --- a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx +++ b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx @@ -85,10 +85,12 @@ class SessionsErrorBoundary extends Component< const SESSIONS_TAKE = 30 interface PageflowItem { - type: 'pageview' | 'event' | 'error' + type: 'pageview' | 'event' | 'error' | 'sale' | 'refund' value: string created: string metadata?: { key: string; value: string }[] + amount?: number + currency?: string } interface ActiveSession { @@ -201,7 +203,9 @@ const SessionsViewInner = ({ dv: apiDetails.dv, profileId: apiDetails.profileId, sdur: apiDetails.sdur, - isLive: matchingSession?.isLive === 1, + isLive: matchingSession + ? matchingSession.isLive === 1 + : apiDetails.isLive, } return { details, From 0861da5f067a9df2405743f46b9f0795234c82fb Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Sun, 3 May 2026 17:43:46 +0100 Subject: [PATCH 11/22] Query optimisations --- admin/src/db/clickhouse.ts | 1 + .../src/analytics/analytics.controller.ts | 50 ++++++++++++------- .../cloud/src/analytics/analytics.service.ts | 3 +- .../src/task-manager/task-manager.service.ts | 33 +++++++++--- .../src/analytics/analytics.service.ts | 26 +++++----- 5 files changed, 73 insertions(+), 40 deletions(-) diff --git a/admin/src/db/clickhouse.ts b/admin/src/db/clickhouse.ts index ac48e9657..3b76d08eb 100644 --- a/admin/src/db/clickhouse.ts +++ b/admin/src/db/clickhouse.ts @@ -40,6 +40,7 @@ const PROJECT_ACTIVITY_EVENT_TYPES = [ "custom_event", "error", "captcha", + "performance", ]; const PROJECT_ACTIVITY_TYPE_LIST = PROJECT_ACTIVITY_EVENT_TYPES.map( (type) => `'${type}'` diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index 146540b24..97392c786 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -709,13 +709,14 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - timeBucket, - period, - safeTimezone, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + timeBucket, + period, + safeTimezone, + ) await this.analyticsService.checkProjectAccess( pid, uid, @@ -727,7 +728,12 @@ export class AnalyticsController { this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart') const paramsData = { - params: { pid, groupFrom, groupTo, ...filtersParams }, + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, } const result = await this.analyticsService.groupChartByTimeBucket( @@ -882,8 +888,8 @@ export class AnalyticsController { const chart = await this.analyticsService.getPerfChartData( timeBucket, - from, - to, + groupFrom, + groupTo, filtersQuery, paramsData, safeTimezone, @@ -2725,17 +2731,23 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - newTimeBucket, - period, - safeTimezone, - diff, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + newTimeBucket, + period, + safeTimezone, + diff, + ) const paramsData = { - params: { pid, groupFrom, groupTo, ...filtersParams }, + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, } // customEvents comes as a JSON.stringified array from the frontend diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index fb6f2a169..9840fb787 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -6101,12 +6101,11 @@ export class AnalyticsService { ? '' : "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)" - // Get total sessions from pageview events for the time range + // Get total sessions from all matching events for the time range const queryTotalSessions = ` SELECT count(DISTINCT psid) as totalSessions FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${sessionFiltersQuery} ` diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 2ec1a1568..e4b96f794 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -389,10 +389,32 @@ export class TaskManagerService { const params: Record = {} const column = goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg' + const appendMetadataFilters = (condition: string) => { + if (!goal.metadataFilters || goal.metadataFilters.length === 0) { + return { condition, params } + } + + const metaConditions: string[] = [] + + goal.metadataFilters.forEach((filter, index) => { + const keyParam = `${paramKey}_metaKey${index}` + const valueParam = `${paramKey}_metaValue${index}` + params[keyParam] = filter.key + params[valueParam] = filter.value + metaConditions.push( + `has(meta.key, {${keyParam}:String}) AND meta.value[indexOf(meta.key, {${keyParam}:String})] = {${valueParam}:String}`, + ) + }) + + return { + condition: `(${condition}) AND (${metaConditions.join(' AND ')})`, + params, + } + } if (goal.matchType === GoalMatchType.EXACT) { params[paramKey] = goalValue - return { condition: `${column} = {${paramKey}:String}`, params } + return appendMetadataFilters(`${column} = {${paramKey}:String}`) } if (goal.matchType === GoalMatchType.CONTAINS) { @@ -402,12 +424,12 @@ export class TaskManagerService { } params[paramKey] = `%${goalValue}%` - return { condition: `${column} ILIKE {${paramKey}:String}`, params } + return appendMetadataFilters(`${column} ILIKE {${paramKey}:String}`) } // Regex goal params[paramKey] = goalValue - return { condition: `match(${column}, {${paramKey}:String})`, params } + return appendMetadataFilters(`match(${column}, {${paramKey}:String})`) } /** @@ -579,7 +601,7 @@ export class TaskManagerService { SELECT count() AS totalEvents FROM events WHERE pid IN ({pids:Array(FixedString(12))}) - AND type IN ('pageview', 'custom_event', 'error', 'captcha') + AND type IN ('pageview', 'custom_event', 'error', 'captcha', 'performance') ` const { data } = await clickhouse @@ -1998,8 +2020,7 @@ export class TaskManagerService { return } - // No need to check for performance activity because it's not tracked without tracking analytics - const queryEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error', 'performance') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` // Process project IDs in chunks to avoid ClickHouse field value limit let totalEvents = 0 diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index c1b82f71c..be8305095 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -4995,12 +4995,11 @@ export class AnalyticsService { ? '' : "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)" - // Get total sessions from pageview events for the time range + // Get total sessions from all matching events for the time range const queryTotalSessions = ` SELECT count(DISTINCT psid) as totalSessions FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${sessionFiltersQuery} ` @@ -5578,7 +5577,7 @@ export class AnalyticsService { if(type = 'custom_event', 1, 0) AS isEvent FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' @@ -5727,18 +5726,19 @@ export class AnalyticsService { // Query for device/location details const queryDetails = ` SELECT - any(cc) AS cc, - any(rg) AS rg, - any(ct) AS ct, - any(os) AS os, - any(osv) AS osv, - any(br) AS br, - any(brv) AS brv, - any(dv) AS dv, - any(lc) AS lc + argMax(cc, created) AS cc, + argMax(rg, created) AS rg, + argMax(ct, created) AS ct, + argMax(os, created) AS os, + argMax(osv, created) AS osv, + argMax(br, created) AS br, + argMax(brv, created) AS brv, + argMax(dv, created) AS dv, + argMax(lc, created) AS lc FROM events WHERE pid = {pid:FixedString(12)} AND profileId = {profileId:String} + AND type IN ('pageview', 'custom_event', 'error') ` const params = { pid, profileId } @@ -5992,7 +5992,7 @@ export class AnalyticsService { toTimeZone(created, {timezone:String}) AS created_tz FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} From baa7eb4611be1e6100e3835b255f31c05bbf43c6 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 01:24:10 +0100 Subject: [PATCH 12/22] fix: Session duration calculation improvements --- .../src/analytics/analytics.controller.ts | 60 +++++++++------ .../cloud/src/analytics/analytics.service.ts | 64 ++++++++++------ .../src/task-manager/task-manager.service.ts | 6 +- .../src/analytics/analytics.service.ts | 73 +++++++++++++------ 4 files changed, 131 insertions(+), 72 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index 97392c786..4f3063996 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -265,12 +265,12 @@ export class AnalyticsController { ) if (period === 'all') { - const eventType = isCaptcha + const eventTypes = isCaptcha ? 'captcha' - : this.analyticsService.getAnalyticsEventType(customEVFilterApplied) + : (['pageview', 'custom_event', 'error'] as const) const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - eventType, + eventTypes, ) diff = res.diff @@ -805,19 +805,25 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - newTimeBucket, - period, - safeTimezone, - diff, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + newTimeBucket, + period, + safeTimezone, + diff, + ) const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` const paramsData = { - params: { pid, groupFrom, groupTo, ...filtersParams }, + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, } const result = await this.analyticsService.groupPerfByTimeBucket( @@ -862,13 +868,14 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - timeBucket, - period, - safeTimezone, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + timeBucket, + period, + safeTimezone, + ) await this.analyticsService.checkProjectAccess( pid, uid, @@ -883,7 +890,12 @@ export class AnalyticsController { ) const paramsData = { - params: { pid, groupFrom, groupTo, ...filtersParams }, + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, } const chart = await this.analyticsService.getPerfChartData( @@ -2038,7 +2050,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - this.analyticsService.getAnalyticsEventType(customEVFilterApplied), + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] @@ -2470,7 +2482,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - this.analyticsService.getAnalyticsEventType(customEVFilterApplied), + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] @@ -2549,7 +2561,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'pageview', + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] diff = res.diff @@ -2637,7 +2649,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - this.analyticsService.getAnalyticsEventType(customEVFilterApplied), + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 9840fb787..bf7010df9 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -1003,15 +1003,17 @@ export class AnalyticsService { async calculateTimeBucketForAllTime( pid: string, - eventType: EventsAllTimeType, + eventTypes: EventsAllTimeType | readonly EventsAllTimeType[], ): Promise<{ timeBucket: TimeBucketType[] diff: number }> { + const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes] + const { data: fromData } = await clickhouse .query({ - query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type = {type:String}`, - query_params: { pid, type: eventType }, + query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`, + query_params: { pid, types }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -2718,7 +2720,7 @@ export class AnalyticsService { if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) { const safeTimezone = this.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.getGroupFromTo( + const { groupFromUTC, groupToUTC } = this.getGroupFromTo( from, to, ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null, @@ -2726,8 +2728,8 @@ export class AnalyticsService { safeTimezone, ) - _from = groupFrom - _to = groupTo + _from = groupFromUTC + _to = groupToUTC } const result = {} @@ -2791,11 +2793,12 @@ export class AnalyticsService { if (_from && _to) { // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well - const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1 + const diff = dayjs.utc(_to).diff(dayjs.utc(_from), 'days') || 1 now = _to periodFormatted = _from - periodSubtracted = dayjs(_from) + periodSubtracted = dayjs + .utc(_from) .subtract(diff, 'days') .format('YYYY-MM-DD HH:mm:ss') } else { @@ -5417,22 +5420,41 @@ export class AnalyticsService { AND profileId = {profileId:String} ` - // Query avg duration from pageview events (more accurate than sessions table) + // Prefer pageview timings, but fall back to sessions for event/error-only profiles. const queryAvgDuration = ` - SELECT - avg(session_duration) AS avgDuration - FROM ( + WITH pageview_avg AS ( SELECT - psid, - dateDiff('second', min(created), max(created)) AS session_duration - FROM events - WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' - AND profileId = {profileId:String} - AND psid IS NOT NULL - GROUP BY psid - HAVING session_duration > 0 + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(created), max(created)) AS session_duration + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND profileId = {profileId:String} + AND psid IS NOT NULL + GROUP BY psid + HAVING session_duration > 0 + ) + ), + sessions_avg AS ( + SELECT + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration + FROM sessions FINAL + WHERE pid = {pid:FixedString(12)} + AND profileId = {profileId:String} + GROUP BY psid + HAVING session_duration > 0 + ) ) + SELECT + coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration + FROM pageview_avg, sessions_avg ` const queryPageviews = ` diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index e4b96f794..6204ba238 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -2020,7 +2020,7 @@ export class TaskManagerService { return } - const queryEvents = `SELECT count() FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error', 'performance') AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryEvents = `SELECT 1 FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error', 'performance') AND created BETWEEN {nineWeeksAgo:String} AND {now:String} LIMIT 1` // Process project IDs in chunks to avoid ClickHouse field value limit let totalEvents = 0 @@ -2038,9 +2038,9 @@ export class TaskManagerService { query: queryEvents, query_params: queryParams, }) - .then((resultSet) => resultSet.json<{ 'count()': number }>()) + .then((resultSet) => resultSet.json>()) - totalEvents += eventsResult[0]['count()'] + totalEvents += eventsResult.length // Early return if we found activity if (totalEvents > 0) { diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index be8305095..fc419e769 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -1021,15 +1021,17 @@ export class AnalyticsService { async calculateTimeBucketForAllTime( pid: string, - eventType: EventsAllTimeType, + eventTypes: EventsAllTimeType | readonly EventsAllTimeType[], ): Promise<{ timeBucket: TimeBucketType[] diff: number }> { + const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes] + const { data: fromData } = await clickhouse .query({ - query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type = {type:String}`, - query_params: { pid, type: eventType }, + query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`, + query_params: { pid, types }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -1993,6 +1995,14 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) as duration FROM sessions WHERE pid = {pid:FixedString(12)} + AND psid IN ( + SELECT DISTINCT psid + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' + AND psid IS NOT NULL + ${filtersQuery} + ) GROUP BY psid ) ) @@ -4252,7 +4262,7 @@ export class AnalyticsService { SELECT uniqExact(psid) as count FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND created >= {since:DateTime} AND psid IS NOT NULL ` @@ -5201,17 +5211,13 @@ export class AnalyticsService { SELECT DISTINCT CAST(errors.psid, 'String') as psid, any(errors.profileId) as profileId, - COALESCE(any(analytics.cc), any(errors.cc)) as cc, - COALESCE(any(analytics.br), any(errors.br)) as br, - COALESCE(any(analytics.os), any(errors.os)) as os, + any(errors.cc) as cc, + any(errors.br) as br, + any(errors.os) as os, min(errors.created) as firstErrorAt, max(errors.created) as lastErrorAt, count(*) as errorCount FROM events AS errors - LEFT JOIN events AS analytics - ON errors.psid = analytics.psid - AND errors.pid = analytics.pid - AND analytics.type = 'pageview' WHERE errors.pid = {pid:FixedString(12)} AND errors.type = 'error' AND errors.eid = {eid:FixedString(32)} @@ -5689,22 +5695,41 @@ export class AnalyticsService { AND profileId = {profileId:String} ` - // Query avg duration from pageview events (more accurate than sessions table) + // Prefer pageview timings, but fall back to sessions for event/error-only profiles. const queryAvgDuration = ` - SELECT - avg(session_duration) AS avgDuration - FROM ( + WITH pageview_avg AS ( SELECT - psid, - dateDiff('second', min(created), max(created)) AS session_duration - FROM events - WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' - AND profileId = {profileId:String} - AND psid IS NOT NULL - GROUP BY psid - HAVING session_duration > 0 + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(created), max(created)) AS session_duration + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND profileId = {profileId:String} + AND psid IS NOT NULL + GROUP BY psid + HAVING session_duration > 0 + ) + ), + sessions_avg AS ( + SELECT + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration + FROM sessions FINAL + WHERE pid = {pid:FixedString(12)} + AND profileId = {profileId:String} + GROUP BY psid + HAVING session_duration > 0 + ) ) + SELECT + coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration + FROM pageview_avg, sessions_avg ` const queryPageviews = ` From bba7f04674aac626d0ba67dde33ce398716d30a5 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 01:53:24 +0100 Subject: [PATCH 13/22] fix profile error scoping and custom event chart metrics --- .../cloud/src/analytics/analytics.service.ts | 24 +--- .../src/task-manager/task-manager.service.ts | 4 + .../src/analytics/analytics.service.ts | 123 ++++++++++++------ .../src/analytics/interfaces/index.ts | 3 +- 4 files changed, 93 insertions(+), 61 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index bf7010df9..81d03a933 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -5305,10 +5305,11 @@ export class AnalyticsService { dv, created, if(type = 'pageview', 1, 0) AS isPageview, - if(type = 'custom_event', 1, 0) AS isEvent + if(type = 'custom_event', 1, 0) AS isEvent, + if(type = 'error', 1, 0) AS isError FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' @@ -5348,6 +5349,7 @@ export class AnalyticsService { sum(isPageview) AS pageviewsCount, countDistinct(psid) AS sessionsCount, sum(isEvent) AS eventsCount, + sum(isError) AS errorsCount, min(created) AS firstSeen, max(created) AS lastSeen, any(cc) AS cc_agg, @@ -5356,26 +5358,13 @@ export class AnalyticsService { any(dv) AS dv_agg FROM all_profile_data GROUP BY profileId - ), - profile_errors AS ( - SELECT - profileId, - count() AS errorsCount - FROM events - WHERE pid = {pid:FixedString(12)} - AND type = 'error' - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - GROUP BY profileId ) SELECT pa.profileId AS profileId, pa.sessionsCount AS sessionsCount, pa.pageviewsCount AS pageviewsCount, pa.eventsCount AS eventsCount, - COALESCE(perr.errorsCount, 0) AS errorsCount, + pa.errorsCount AS errorsCount, pa.firstSeen AS firstSeen, pa.lastSeen AS lastSeen, pa.cc_agg AS cc, @@ -5383,7 +5372,6 @@ export class AnalyticsService { pa.br_agg AS br, pa.dv_agg AS dv FROM profile_aggregated AS pa - LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileId ORDER BY pa.lastSeen DESC LIMIT {take:UInt32} OFFSET {skip:UInt32} @@ -5765,7 +5753,7 @@ export class AnalyticsService { toTimeZone(created, {timezone:String}) AS created_tz FROM events WHERE pid = {pid:FixedString(12)} - AND type IN ('pageview', 'custom_event') + AND type IN ('pageview', 'custom_event', 'error') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 6204ba238..e4b4cb4a7 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -428,6 +428,10 @@ export class TaskManagerService { } // Regex goal + if (!goalValue || goalValue === '') { + return appendMetadataFilters('false') + } + params[paramKey] = goalValue return appendMetadataFilters(`match(${column}, {${paramKey}:String})`) } diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index fc419e769..48707d320 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -2294,10 +2294,15 @@ export class AnalyticsService { mode: ChartRenderMode, ): IExtractChartData { if (customEVFilterApplied) { + const customEventsData = this.extractCustomEventsChartData( + result, + xShifted, + ) const uniques = - this.extractCustomEventsChartData(result, xShifted)?._unknown_event || - [] - const sdur = Array(_size(xShifted)).fill(0) + customEventsData._uniques || + customEventsData._unknown_event || + Array(_size(xShifted)).fill(0) + const sdur = customEventsData._sdur || Array(_size(xShifted)).fill(0) return { visits: uniques, @@ -2881,6 +2886,10 @@ export class AnalyticsService { extractCustomEventsChartData(queryResult, x: string[]): any { const result = {} + const uniques = Array(x.length).fill(0) + const sdur = Array(x.length).fill(0) + let hasUniques = false + let hasSdur = false for (let row = 0; row < _size(queryResult); ++row) { const { ev = '_unknown_event' } = queryResult[row] @@ -2895,9 +2904,27 @@ export class AnalyticsService { } result[ev][index] = queryResult[row].count + + if (typeof queryResult[row].uniques !== 'undefined') { + uniques[index] = queryResult[row].uniques || 0 + hasUniques = true + } + + if (typeof queryResult[row].sdur !== 'undefined') { + sdur[index] = queryResult[row].sdur || 0 + hasSdur = true + } } } + if (hasUniques) { + result['_uniques'] = uniques + } + + if (hasSdur) { + result['_sdur'] = sdur + } + return result } @@ -3098,16 +3125,31 @@ export class AnalyticsService { const baseQuery = ` SELECT ${selector}, - count() as count + count() as count, + countDistinct(subquery.psid) as uniques, + avgOrNull(sessions_data.duration) as sdur FROM ( - SELECT *, - ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created + SELECT + pid, + psid, + ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery + LEFT JOIN ( + SELECT + pid, + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) as duration + FROM sessions + WHERE pid = {pid:FixedString(12)} + GROUP BY pid, psid + ) as sessions_data + ON subquery.pid = sessions_data.pid + AND subquery.psid = sessions_data.psid GROUP BY ${groupBy} ORDER BY ${groupBy} ` @@ -3115,8 +3157,10 @@ export class AnalyticsService { if (mode === ChartRenderMode.CUMULATIVE) { return ` SELECT - *, - sum(count) OVER (ORDER BY ${groupBy}) as count + ${groupBy}, + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(uniques) OVER (ORDER BY ${groupBy}) as uniques, + sdur FROM (${baseQuery}) ` } @@ -4366,16 +4410,21 @@ export class AnalyticsService { } return _map(pages, (page: IPageflow) => { - if (!page.metadata) { - return page + const { created_utc: _createdUtc, ...displayPage } = page + + if (!displayPage.metadata) { + return displayPage } return { - ...page, - metadata: _map(page.metadata, ([key, value]: [string, string]) => ({ - key, - value, - })), + ...displayPage, + metadata: _map( + displayPage.metadata, + ([key, value]: [string, string]) => ({ + key, + value, + }), + ), } }) } @@ -4394,7 +4443,8 @@ export class AnalyticsService { event_type = 'custom_event', toString(event_name), toString(error_name) ) AS value, - toTimeZone(created, {timezone:String}) AS created, + created AS created_utc, + toTimeZone(created, {timezone:String}) AS created_local, pid, toString(psid) AS psid, if( @@ -4434,10 +4484,11 @@ export class AnalyticsService { SELECT type, value, - created, + created_local AS created, + created_utc, metadata FROM events_with_meta - ORDER BY created ASC + ORDER BY created_utc ASC ` const querySessionDetails = ` @@ -4514,7 +4565,7 @@ export class AnalyticsService { } if (!duration && lastSeen && Array.isArray(pages) && pages.length >= 1) { - const first = dayjs(pages[0].created) + const first = dayjs(pages[0].created_utc) const diffSeconds = dayjs(lastSeen).diff(first, 'second') if (diffSeconds > 0) { duration = diffSeconds @@ -4522,8 +4573,8 @@ export class AnalyticsService { } if (!duration && Array.isArray(pages) && pages.length >= 2) { - const first = dayjs(pages[0].created) - const last = dayjs(pages[pages.length - 1].created) + const first = dayjs(pages[0].created_utc) + const last = dayjs(pages[pages.length - 1].created_utc) const diffSeconds = last.diff(first, 'second') if (diffSeconds > 0) { duration = diffSeconds @@ -4560,12 +4611,12 @@ export class AnalyticsService { if (!_isEmpty(pages)) { const from = dayjs - .utc(pages[0].created) + .utc(pages[0].created_utc) .tz(safeTimezone) .startOf('minute') .format('YYYY-MM-DD HH:mm:ss') const to = dayjs - .utc(pages[_size(pages) - 1].created) + .utc(pages[_size(pages) - 1].created_utc) .tz(safeTimezone) .endOf('minute') .format('YYYY-MM-DD HH:mm:ss') @@ -4583,8 +4634,8 @@ export class AnalyticsService { { params: { ...paramsData.params, - groupFrom: pages[0].created, - groupTo: pages[_size(pages) - 1].created, + groupFrom: from, + groupTo: to, }, }, safeTimezone, @@ -4597,7 +4648,7 @@ export class AnalyticsService { let isLive = false if (!_isEmpty(pages)) { - const lastActivityTime = dayjs(pages[pages.length - 1].created) + const lastActivityTime = dayjs(pages[pages.length - 1].created_utc) const liveThresholdTime = dayjs().subtract( LIVE_SESSION_THRESHOLD_SECONDS, @@ -5580,7 +5631,8 @@ export class AnalyticsService { dv, created, if(type = 'pageview', 1, 0) AS isPageview, - if(type = 'custom_event', 1, 0) AS isEvent + if(type = 'custom_event', 1, 0) AS isEvent, + if(type = 'error', 1, 0) AS isError FROM events WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') @@ -5623,6 +5675,7 @@ export class AnalyticsService { sum(isPageview) AS pageviewsCount, countDistinct(psid) AS sessionsCount, sum(isEvent) AS eventsCount, + sum(isError) AS errorsCount, min(created) AS firstSeen, max(created) AS lastSeen, any(cc) AS cc_agg, @@ -5631,26 +5684,13 @@ export class AnalyticsService { any(dv) AS dv_agg FROM all_profile_data GROUP BY profileId - ), - profile_errors AS ( - SELECT - profileId, - count() AS errorsCount - FROM events - WHERE pid = {pid:FixedString(12)} - AND type = 'error' - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - GROUP BY profileId ) SELECT pa.profileId AS profileId, pa.sessionsCount AS sessionsCount, pa.pageviewsCount AS pageviewsCount, pa.eventsCount AS eventsCount, - COALESCE(perr.errorsCount, 0) AS errorsCount, + pa.errorsCount AS errorsCount, pa.firstSeen AS firstSeen, pa.lastSeen AS lastSeen, pa.cc_agg AS cc, @@ -5658,7 +5698,6 @@ export class AnalyticsService { pa.br_agg AS br, pa.dv_agg AS dv FROM profile_aggregated AS pa - LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileId ORDER BY pa.lastSeen DESC LIMIT {take:UInt32} OFFSET {skip:UInt32} diff --git a/backend/apps/community/src/analytics/interfaces/index.ts b/backend/apps/community/src/analytics/interfaces/index.ts index 84da70732..2dace48be 100644 --- a/backend/apps/community/src/analytics/interfaces/index.ts +++ b/backend/apps/community/src/analytics/interfaces/index.ts @@ -183,9 +183,10 @@ export interface IOverallPerformance { } export interface IPageflow { - type: 'pageview' | 'event' + type: 'pageview' | 'event' | 'error' value: string created: string + created_utc?: string metadata?: [string, string][] } From c2af5194c99006f7f57aef32eb1b743d95388966 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 02:03:35 +0100 Subject: [PATCH 14/22] Restrict the error-rate denominator to visitor session events --- backend/apps/cloud/src/analytics/analytics.service.ts | 2 ++ backend/apps/community/src/analytics/analytics.service.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 81d03a933..431d3c68d 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -6116,6 +6116,8 @@ export class AnalyticsService { SELECT count(DISTINCT psid) as totalSessions FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${sessionFiltersQuery} ` diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 48707d320..7b252e922 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -5061,6 +5061,8 @@ export class AnalyticsService { SELECT count(DISTINCT psid) as totalSessions FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${sessionFiltersQuery} ` From 5001b3a8eb1b913aa73e62f75026c8c9e3efee1b Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 02:05:51 +0100 Subject: [PATCH 15/22] =?UTF-8?q?Don=E2=80=99t=20rely=20on=20pageviews=20a?= =?UTF-8?q?lone=20for=20affected-session=20metadata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/apps/cloud/src/analytics/analytics.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 431d3c68d..c00f2a1eb 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -6347,10 +6347,10 @@ export class AnalyticsService { any(cc) AS cc, any(br) AS br, any(os) AS os, - count(*) AS pageviews + countIf(type = 'pageview') AS pageviews FROM events WHERE pid = {pid:FixedString(12)} - AND type = 'pageview' + AND type IN ('pageview', 'error', 'custom_event') AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid, psid From ac5b71c7b40ef6093e6585d5a76d601abd7461e5 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 02:06:16 +0100 Subject: [PATCH 16/22] Keep profile details off non-visitor event types --- backend/apps/cloud/src/analytics/analytics.service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index c00f2a1eb..f3fcdb67f 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -5475,6 +5475,7 @@ export class AnalyticsService { FROM events WHERE pid = {pid:FixedString(12)} AND profileId = {profileId:String} + AND type IN ('pageview', 'custom_event', 'error') ` // Query for total revenue from profile (only sales, not refunds) From 929a7b38a823c7027d9ac1a9e4585f615ede5788 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 15:12:54 +0100 Subject: [PATCH 17/22] correct error metadata and event counts --- .../cloud/src/analytics/analytics.service.ts | 19 +++++++++++++------ .../src/analytics/analytics.service.ts | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index f3fcdb67f..445091801 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -4832,12 +4832,12 @@ export class AnalyticsService { toString(psid) AS psid, if( event_type = 'error', - [ + arrayFilter(x -> x.2 != '' AND x.2 != '0', [ tuple('message', COALESCE(error_message, '')), - tuple('lineno', toString(COALESCE(lineno, 0))), - tuple('colno', toString(COALESCE(colno, 0))), + tuple('lineno', ifNull(toString(lineno), '')), + tuple('colno', ifNull(toString(colno), '')), tuple('filename', COALESCE(error_filename, '')) - ], + ]), arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value)) ) AS metadata FROM ( @@ -4919,7 +4919,13 @@ export class AnalyticsService { AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} - ORDER BY created ASC + ORDER BY + CASE + WHEN type = 'pageview' THEN 0 + WHEN type = 'custom_event' THEN 1 + ELSE 2 + END, + created ASC LIMIT 1; ` @@ -6016,7 +6022,8 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type = 'error' - AND eid = {eid:FixedString(32)}; + AND eid = {eid:FixedString(32)} + AND created BETWEEN {groupFrom:String} AND {groupTo:String}; ` const queryMetadata = ` diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index 7b252e922..fbc62e798 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -2302,10 +2302,11 @@ export class AnalyticsService { customEventsData._uniques || customEventsData._unknown_event || Array(_size(xShifted)).fill(0) + const visits = customEventsData._count || Array(_size(xShifted)).fill(0) const sdur = customEventsData._sdur || Array(_size(xShifted)).fill(0) return { - visits: uniques, + visits, uniques, sdur, } @@ -2892,7 +2893,9 @@ export class AnalyticsService { let hasSdur = false for (let row = 0; row < _size(queryResult); ++row) { - const { ev = '_unknown_event' } = queryResult[row] + const rowEventName = queryResult[row].ev + const ev = + typeof rowEventName === 'undefined' ? '_unknown_event' : rowEventName const dateString = this.generateDateString(queryResult[row]) @@ -2905,6 +2908,14 @@ export class AnalyticsService { result[ev][index] = queryResult[row].count + if (typeof rowEventName === 'undefined') { + if (!result['_count']) { + result['_count'] = Array(x.length).fill(0) + } + + result['_count'][index] = queryResult[row].count || 0 + } + if (typeof queryResult[row].uniques !== 'undefined') { uniques[index] = queryResult[row].uniques || 0 hasUniques = true From ad698683bcce7f88a46e9c7b8aff8bc4bf780dd8 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 17:52:54 +0100 Subject: [PATCH 18/22] fix analytics session and error edge cases --- .../cloud/src/analytics/analytics.service.ts | 10 +++++--- .../src/analytics/analytics.service.ts | 23 ++++++++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 445091801..f6610e2e7 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -1247,8 +1247,8 @@ export class AnalyticsService { ? 'argMin(pg, created)' : 'argMax(pg, created)' const subQueryForPages = isContains - ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` - : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page = {${param}:String}` + ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` + : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page = {${param}:String}` // For exclusive filter (isNot) we exclude sessions with matching entry/exit page query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})` @@ -2235,7 +2235,7 @@ export class AnalyticsService { FROM events WHERE pid IN {pids:Array(FixedString(12))} - AND type = 'pageview' + AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid ` @@ -6405,6 +6405,10 @@ export class AnalyticsService { } async validateEIDs(eids: string[], pid: string) { + if (_isEmpty(eids)) { + throw new BadRequestException('At least one error ID is required') + } + const params = _reduce( eids, (acc, curr, index) => ({ diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index fbc62e798..4215d6bce 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -1878,7 +1878,7 @@ export class AnalyticsService { COALESCE(errc.count, 0) AS errors, dsf.sessionStart, dsf.lastActivity, - if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, + if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, sda.avg_duration AS sdur, sda.profileId AS profileId, if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified, @@ -4767,7 +4767,7 @@ export class AnalyticsService { COALESCE(errc.count, 0) AS errors, dsf.sessionStart, dsf.lastActivity, - if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, + if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, sda.avg_duration AS sdur, sda.profileId AS profileId, if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified, @@ -5094,6 +5094,7 @@ export class AnalyticsService { ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} AND type = 'error' + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5268,6 +5269,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid = {eid:FixedString(32)} + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -5285,6 +5287,7 @@ export class AnalyticsService { WHERE errors.pid = {pid:FixedString(12)} AND errors.type = 'error' AND errors.eid = {eid:FixedString(32)} + AND errors.psid IS NOT NULL AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY errors.psid ORDER BY lastErrorAt DESC @@ -5800,6 +5803,14 @@ export class AnalyticsService { AND profileId = {profileId:String} ` + const queryErrors = ` + SELECT count() AS errorsCount + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'error' + AND profileId = {profileId:String} + ` + // Query for device/location details const queryDetails = ` SELECT @@ -5825,6 +5836,7 @@ export class AnalyticsService { avgDurationResult, pageviewsResult, eventsResult, + errorsResult, detailsResult, ] = await Promise.all([ clickhouse @@ -5839,6 +5851,9 @@ export class AnalyticsService { clickhouse .query({ query: queryEvents, query_params: params }) .then((resultSet) => resultSet.json()), + clickhouse + .query({ query: queryErrors, query_params: params }) + .then((resultSet) => resultSet.json()), clickhouse .query({ query: queryDetails, query_params: params }) .then((resultSet) => resultSet.json()), @@ -5851,6 +5866,7 @@ export class AnalyticsService { const avgDuration = (avgDurationResult.data[0] || {}) as Record const pageviews = (pageviewsResult.data[0] || {}) as Record const events = (eventsResult.data[0] || {}) as Record + const errors = (errorsResult.data[0] || {}) as Record const details = (detailsResult.data[0] || {}) as Record return { @@ -5859,6 +5875,7 @@ export class AnalyticsService { sessionsCount: sessionCount.sessionsCount || 0, pageviewsCount: pageviews.pageviewsCount || 0, eventsCount: events.eventsCount || 0, + errorsCount: errors.errorsCount || 0, firstSeen: sessionCount.firstSeen, lastSeen: sessionCount.lastSeen, avgDuration: avgDuration.avgDuration || 0, @@ -6166,7 +6183,7 @@ export class AnalyticsService { COALESCE(errc.count, 0) AS errors, ps.sessionStart, ps.lastActivity, - if(dateDiff('second', ps.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, + if(dateDiff('second', toTimeZone(ps.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, sda.avg_duration AS sdur FROM profile_sessions ps LEFT JOIN pageview_counts pc ON ps.psidCasted = pc.psidCasted AND ps.pid = pc.pid From d8501ec5e66364e06b78957215b049047ba9594e Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 20:21:16 +0100 Subject: [PATCH 19/22] fix: include custom events in filters and valid sessions --- .../apps/cloud/src/analytics/analytics.service.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index f6610e2e7..ba4e2dbcb 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -2236,6 +2236,8 @@ export class AnalyticsService { WHERE pid IN {pids:Array(FixedString(12))} AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND psid != '' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid ` @@ -2932,7 +2934,7 @@ export class AnalyticsService { let query = type === 'ev' ? `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name` - : `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND ${type} IS NOT NULL GROUP BY ${type}` + : `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event') AND ${type} IS NOT NULL GROUP BY ${type}` const { data } = await clickhouse .query({ @@ -2974,13 +2976,16 @@ export class AnalyticsService { type: 'traffic' | 'errors', column: 'br' | 'os', ): Promise> { - const safeType = type === 'errors' ? 'error' : 'pageview' + const safeType = + type === 'errors' + ? "type = 'error'" + : "type IN ('pageview', 'custom_event')" const safeVersionCol = column === 'br' ? 'brv' : 'osv' const query = ` SELECT ${column}, ${safeVersionCol} FROM events - WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL + WHERE pid={pid:FixedString(12)} AND ${safeType} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL GROUP BY ${column}, ${safeVersionCol} ` @@ -6126,6 +6131,7 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL + AND psid != '' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${sessionFiltersQuery} ` From 975a47ab2beaf96ea4c745e2120c1bca341cef4a Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 21:42:15 +0100 Subject: [PATCH 20/22] Use safe SQL parameters --- backend/apps/cloud/src/ai/ai.service.ts | 6 ++++-- backend/apps/cloud/src/goal/goal.controller.ts | 6 ++++-- backend/apps/community/src/goal/goal.controller.ts | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/backend/apps/cloud/src/ai/ai.service.ts b/backend/apps/cloud/src/ai/ai.service.ts index 35aa6114e..5414f6c3c 100644 --- a/backend/apps/cloud/src/ai/ai.service.ts +++ b/backend/apps/cloud/src/ai/ai.service.ts @@ -1599,7 +1599,7 @@ Filter modifiers: uniqExact(psid) as uniqueSessions FROM events WHERE pid = {pid:FixedString(12)} - AND type = '${goalType}' + AND type = {goalType:String} AND ${matchCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -1609,6 +1609,7 @@ Filter modifiers: query, query_params: { pid, + goalType, goalValue: goal.value || '', groupFrom: groupFromUTC, groupTo: groupToUTC, @@ -1645,7 +1646,7 @@ Filter modifiers: SELECT count(*) as conversions FROM events WHERE pid = {pid:FixedString(12)} - AND type = '${goalType}' + AND type = {goalType:String} AND ${matchCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -1655,6 +1656,7 @@ Filter modifiers: query, query_params: { pid, + goalType, goalValue: goal.value || '', groupFrom: groupFromUTC, groupTo: groupToUTC, diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.ts index ac4cde6de..ed2807bee 100644 --- a/backend/apps/cloud/src/goal/goal.controller.ts +++ b/backend/apps/cloud/src/goal/goal.controller.ts @@ -444,7 +444,7 @@ export class GoalController { FROM events WHERE pid = {pid:FixedString(12)} - AND type = '${goalType}' + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -452,6 +452,7 @@ export class GoalController { const queryParams = { pid: goal.project.id, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, ...matchParams, @@ -678,7 +679,7 @@ export class GoalController { FROM events WHERE pid = {pid:FixedString(12)} - AND type = '${goalType}' + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -689,6 +690,7 @@ export class GoalController { const queryParams = { pid: goal.project.id, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, timezone: safeTimezone, diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.ts index 4a022b2e8..37f85f30a 100644 --- a/backend/apps/community/src/goal/goal.controller.ts +++ b/backend/apps/community/src/goal/goal.controller.ts @@ -362,7 +362,7 @@ export class GoalController { FROM events WHERE pid = {pid:FixedString(12)} - AND type = '${goalType}' + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -370,6 +370,7 @@ export class GoalController { const queryParams = { pid: goal.projectId, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, ...matchParams, @@ -595,7 +596,7 @@ export class GoalController { FROM events WHERE pid = {pid:FixedString(12)} - AND type = '${goalType}' + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -606,6 +607,7 @@ export class GoalController { const queryParams = { pid: goal.projectId, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, timezone: safeTimezone, From 81de0775db4ee7d97407fe05fd4e04865f62eaf3 Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 21:46:06 +0100 Subject: [PATCH 21/22] Add missing fixes --- .../community/src/analytics/utils/transformers.ts | 4 ++-- .../src/data-import/data-import.processor.ts | 8 +++++++- .../data-import/mappers/simple-analytics.mapper.ts | 12 ++++++++---- .../src/data-import/mappers/umami.mapper.ts | 12 ++++++++---- .../apps/community/src/project/project.service.ts | 2 +- 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/backend/apps/community/src/analytics/utils/transformers.ts b/backend/apps/community/src/analytics/utils/transformers.ts index 687fb0c3e..7ec1ed491 100644 --- a/backend/apps/community/src/analytics/utils/transformers.ts +++ b/backend/apps/community/src/analytics/utils/transformers.ts @@ -176,8 +176,8 @@ export const eventTransformer = (opts: EventTransformerOptions) => { error_name: opts.name || null, error_message: opts.message || null, stackTrace: opts.stackTrace || null, - lineno: opts.lineno || null, - colno: opts.colno || null, + lineno: opts.lineno ?? null, + colno: opts.colno ?? null, error_filename: opts.filename || null, created, } diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts index 06ee7e3a8..5f3ebfe97 100644 --- a/backend/apps/community/src/data-import/data-import.processor.ts +++ b/backend/apps/community/src/data-import/data-import.processor.ts @@ -8,6 +8,7 @@ import { Job } from 'bullmq' import { DataImportService } from './data-import.service' import { DataImportStatus } from './entity/data-import.entity' import { getMapper } from './mappers' +import { ImportError } from './mappers/mapper.interface' import { clickhouse } from '../common/integrations/clickhouse' const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics' @@ -159,10 +160,15 @@ export class DataImportProcessor extends WorkerHost { ) } + const userMessage = + error instanceof ImportError + ? error.message + : 'An unexpected error occurred while processing the import. Please try again or contact support.' + await this.dataImportService.markFailed( importId, projectId, - error.message, + userMessage, ) } } finally { diff --git a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts index f62c9472c..b95e37380 100644 --- a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts @@ -1,7 +1,11 @@ import * as fs from 'fs' import { parse } from 'csv-parse' -import { ImportMapper, AnalyticsImportRow } from './mapper.interface' +import { + ImportMapper, + ImportError, + AnalyticsImportRow, +} from './mapper.interface' import { normalizeNull, truncate, @@ -20,7 +24,7 @@ function validateHeaders(headers: string[]): string[] { ) if (missingColumns.length > 0) { - throw new Error( + throw new ImportError( `CSV does not appear to be a Simple Analytics export. Missing required columns: ${missingColumns.join(', ')}.`, ) } @@ -133,7 +137,7 @@ export class SimpleAnalyticsMapper implements ImportMapper { hostname, ) - const cc = normalizeNull(row.country_code) + const cc = normalizeNull(row.country_code)?.toUpperCase() || null const validCC = cc && /^[A-Z]{2}$/.test(cc) ? cc : null const locale = buildLocale( @@ -182,7 +186,7 @@ export class SimpleAnalyticsMapper implements ImportMapper { } if (!headerChecked) { - throw new Error( + throw new ImportError( 'CSV appears empty or is missing a header row. Please upload the raw CSV export from Simple Analytics.', ) } diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.ts index c2bf13747..3d8c9d9b8 100644 --- a/backend/apps/community/src/data-import/mappers/umami.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts @@ -4,7 +4,11 @@ import * as os from 'os' import { Unzip, UnzipInflate } from 'fflate' import { parse } from 'csv-parse' -import { ImportMapper, AnalyticsImportRow } from './mapper.interface' +import { + ImportMapper, + ImportError, + AnalyticsImportRow, +} from './mapper.interface' import { normalizeNull, truncate, @@ -188,7 +192,7 @@ export class UmamiMapper implements ImportMapper { file.originalSize > MAX_UMAMI_CSV_BYTES ) { fail( - new Error( + new ImportError( `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, ), ) @@ -213,7 +217,7 @@ export class UmamiMapper implements ImportMapper { extractedBytes += chunk.length if (extractedBytes > MAX_UMAMI_CSV_BYTES) { fail( - new Error( + new ImportError( `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, ), ) @@ -263,7 +267,7 @@ export class UmamiMapper implements ImportMapper { if (!foundEntry) { fail( - new Error( + new ImportError( `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`, ), ) diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.ts index 7055f823f..160c46ac3 100644 --- a/backend/apps/community/src/project/project.service.ts +++ b/backend/apps/community/src/project/project.service.ts @@ -342,7 +342,7 @@ export class ProjectService { to: string, ): Promise { const queries = [ - "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error') AND created BETWEEN {from:String} AND {to:String}", + "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}", 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', ] From 219e419e267516f6eb795fd11df4426cabdfdaae Mon Sep 17 00:00:00 2001 From: Blue Mouse Date: Mon, 4 May 2026 22:14:50 +0100 Subject: [PATCH 22/22] Split removal of legacy tables from the main migration --- .../clickhouse/2026_05_01_unify_events.js | 5 ----- .../2026_05_04_drop_legacy_tables.js | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.js index ef8689b0d..164c012af 100644 --- a/backend/migrations/clickhouse/2026_05_01_unify_events.js +++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js @@ -91,11 +91,6 @@ const queries = [ `RENAME TABLE ${dbName}.events TO ${dbName}.events_backup, ${dbName}.events_tmp TO ${dbName}.events;`, - // `DROP TABLE IF EXISTS ${dbName}.analytics;`, - // `DROP TABLE IF EXISTS ${dbName}.customEV;`, - // `DROP TABLE IF EXISTS ${dbName}.errors;`, - // `DROP TABLE IF EXISTS ${dbName}.performance;`, - // `DROP TABLE IF EXISTS ${dbName}.captcha;`, `DROP TABLE IF EXISTS ${dbName}.events_backup;`, ] diff --git a/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js new file mode 100644 index 000000000..4964e865c --- /dev/null +++ b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js @@ -0,0 +1,18 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + `SELECT throwIf( + count() = 0, + 'The ${dbName}.events table does not exist. Run 2026_05_01_unify_events.js before dropping legacy tables.' + ) + FROM system.tables + WHERE database = '${dbName}' AND name = 'events';`, + + `DROP TABLE IF EXISTS ${dbName}.analytics;`, + `DROP TABLE IF EXISTS ${dbName}.customEV;`, + `DROP TABLE IF EXISTS ${dbName}.errors;`, + `DROP TABLE IF EXISTS ${dbName}.performance;`, + `DROP TABLE IF EXISTS ${dbName}.captcha;`, +] + +queriesRunner(queries)