diff --git a/admin/src/db/clickhouse.ts b/admin/src/db/clickhouse.ts index 2e4f960b3..3b76d08eb 100644 --- a/admin/src/db/clickhouse.ts +++ b/admin/src/db/clickhouse.ts @@ -34,6 +34,18 @@ export interface ClickHouseStats { tables: TableStats[]; } +const EVENT_TABLE = "events"; +const PROJECT_ACTIVITY_EVENT_TYPES = [ + "pageview", + "custom_event", + "error", + "captcha", + "performance", +]; +const PROJECT_ACTIVITY_TYPE_LIST = PROJECT_ACTIVITY_EVENT_TYPES.map( + (type) => `'${type}'` +).join(", "); + function formatBytes(bytes: number): string { if (bytes === 0) return "0 B"; const k = 1024; @@ -47,20 +59,16 @@ export async function getClickHouseStats(): Promise { const database = process.env.CLICKHOUSE_DATABASE || "analytics"; const tables = [ - "analytics", - "customEV", - "performance", - "errors", - "captcha", + EVENT_TABLE, "sessions", "revenue", "feature_flag_evaluations", "experiment_exposures", "error_statuses", + "bot_blocks", ]; const tableStats: TableStats[] = []; - let totalEvents = 0; for (const table of tables) { try { @@ -93,8 +101,6 @@ export async function getClickHouseStats(): Promise { bytes, bytesFormatted: formatBytes(bytes), }); - - totalEvents += rows; } catch { // Table might not exist, skip it tableStats.push({ @@ -106,6 +112,9 @@ export async function getClickHouseStats(): Promise { } } + const totalEvents = + tableStats.find((table) => table.table === EVENT_TABLE)?.rows || 0; + return { totalEvents, tables: tableStats, @@ -118,7 +127,12 @@ export async function getProjectEventCount(pid: string): Promise { try { const result = await client.query({ - query: `SELECT count() as count FROM ${database}.analytics WHERE pid = {pid:String}`, + query: ` + SELECT count() as count + FROM ${database}.${EVENT_TABLE} + WHERE pid = {pid:FixedString(12)} + AND type IN (${PROJECT_ACTIVITY_TYPE_LIST}) + `, query_params: { pid }, format: "JSONEachRow", }); @@ -158,8 +172,9 @@ export async function getTopProjectsByEvents( SELECT pid, count() as eventCount - FROM ${database}.analytics + FROM ${database}.${EVENT_TABLE} WHERE created >= now() - INTERVAL ${days} DAY + AND type IN (${PROJECT_ACTIVITY_TYPE_LIST}) GROUP BY pid ORDER BY eventCount DESC LIMIT ${limit} @@ -183,8 +198,9 @@ export async function getProjectsWithRecentEvents( const result = await client.query({ query: ` SELECT DISTINCT pid - FROM ${database}.analytics + FROM ${database}.${EVENT_TABLE} WHERE created >= now() - INTERVAL ${days} DAY + AND type IN (${PROJECT_ACTIVITY_TYPE_LIST}) `, format: "JSONEachRow", }); diff --git a/backend/apps/cloud/src/ai/ai.service.ts b/backend/apps/cloud/src/ai/ai.service.ts index 8671e63fd..5414f6c3c 100644 --- a/backend/apps/cloud/src/ai/ai.service.ts +++ b/backend/apps/cloud/src/ai/ai.service.ts @@ -1218,8 +1218,9 @@ Filter modifiers: SELECT count(*) as pageviews, count(DISTINCT psid) as sessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1242,8 +1243,9 @@ Filter modifiers: ${this.getTimeBucketSelect(timeBucket, timezone)} as date, count(*) as pageviews, count(DISTINCT psid) as sessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -1266,8 +1268,9 @@ Filter modifiers: // Get top pages const pagesQuery = ` SELECT pg as name, count(*) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY pg @@ -1285,8 +1288,9 @@ Filter modifiers: // Get top countries const countriesQuery = ` SELECT cc as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND cc IS NOT NULL AND cc != '' ${filterConditions.where} @@ -1305,8 +1309,9 @@ Filter modifiers: // Get top referrers const referrersQuery = ` SELECT ref as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND ref IS NOT NULL AND ref != '' ${filterConditions.where} @@ -1325,8 +1330,9 @@ Filter modifiers: // Get browsers const browsersQuery = ` SELECT br as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND br IS NOT NULL AND br != '' ${filterConditions.where} @@ -1345,8 +1351,9 @@ Filter modifiers: // Get devices const devicesQuery = ` SELECT dv as name, uniqExact(psid) as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND dv IS NOT NULL AND dv != '' ${filterConditions.where} @@ -1416,8 +1423,9 @@ Filter modifiers: round(${measureFn}(render) / 1000, 2) as render, round(${measureFn}(domLoad) / 1000, 2) as domLoad, round(${measureFn}(ttfb) / 1000, 2) as ttfb - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1440,8 +1448,9 @@ Filter modifiers: ${this.getTimeBucketSelect(timeBucket, timezone)} as date, round(${measureFn}(pageLoad) / 1000, 2) as pageLoad, round(${measureFn}(ttfb) / 1000, 2) as ttfb - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -1497,8 +1506,9 @@ Filter modifiers: SELECT count(*) as totalErrors, uniqExact(eid) as uniqueErrors - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1512,15 +1522,16 @@ Filter modifiers: const topErrorsQuery = ` SELECT - name, - message, + error_name AS name, + error_message AS message, count(*) as count, max(created) as lastSeen - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} - GROUP BY name, message + GROUP BY error_name, error_message ORDER BY count DESC LIMIT 10 ` @@ -1571,22 +1582,24 @@ Filter modifiers: } // Query specific goal stats - const table = goal.type === 'custom_event' ? 'customEV' : 'analytics' + const goalType = + goal.type === 'custom_event' ? 'custom_event' : 'pageview' const matchCondition = goal.matchType === 'exact' ? goal.type === 'custom_event' - ? `ev = {goalValue:String}` + ? `event_name = {goalValue:String}` : `pg = {goalValue:String}` : goal.type === 'custom_event' - ? `ev ILIKE concat('%', {goalValue:String}, '%')` + ? `event_name ILIKE concat('%', {goalValue:String}, '%')` : `pg ILIKE concat('%', {goalValue:String}, '%')` const query = ` SELECT count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = {goalType:String} AND ${matchCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -1596,6 +1609,7 @@ Filter modifiers: query, query_params: { pid, + goalType, goalValue: goal.value || '', groupFrom: groupFromUTC, groupTo: groupToUTC, @@ -1617,20 +1631,22 @@ Filter modifiers: const results = await Promise.all( goals.map(async (goal) => { - const table = goal.type === 'custom_event' ? 'customEV' : 'analytics' + const goalType = + goal.type === 'custom_event' ? 'custom_event' : 'pageview' const matchCondition = goal.matchType === 'exact' ? goal.type === 'custom_event' - ? `ev = {goalValue:String}` + ? `event_name = {goalValue:String}` : `pg = {goalValue:String}` : goal.type === 'custom_event' - ? `ev ILIKE concat('%', {goalValue:String}, '%')` + ? `event_name ILIKE concat('%', {goalValue:String}, '%')` : `pg ILIKE concat('%', {goalValue:String}, '%')` const query = ` SELECT count(*) as conversions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = {goalType:String} AND ${matchCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -1640,6 +1656,7 @@ Filter modifiers: query, query_params: { pid, + goalType, goalValue: goal.value || '', groupFrom: groupFromUTC, groupTo: groupToUTC, @@ -1747,8 +1764,9 @@ Filter modifiers: const overallQuery = ` SELECT count(*) as total - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` @@ -1757,8 +1775,9 @@ Filter modifiers: SELECT ${this.getTimeBucketSelect(timeBucket, timezone)} as date, count(*) as challenges - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -1767,8 +1786,9 @@ Filter modifiers: const breakdownQuery = (column: string) => ` SELECT ${column} as name, count(*) as count - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND ${column} IS NOT NULL AND ${column} != '' ${filterConditions.where} @@ -1844,21 +1864,23 @@ Filter modifiers: const overallQuery = ` SELECT count(*) as totalEvents, - uniqExact(ev) as uniqueEvents, + uniqExact(event_name) as uniqueEvents, uniqExact(psid) as sessions - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} ` const topEventsQuery = ` - SELECT ev as name, count(*) as count, uniqExact(psid) as sessions - FROM customEV + SELECT event_name AS name, count(*) as count, uniqExact(psid) as sessions + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} - GROUP BY ev + GROUP BY event_name ORDER BY count DESC LIMIT 25 ` @@ -1868,8 +1890,9 @@ Filter modifiers: ${this.getTimeBucketSelect(timeBucket, timezone)} as date, count(*) as events, uniqExact(psid) as sessions - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filterConditions.where} GROUP BY date @@ -2137,10 +2160,10 @@ Filter modifiers: let conversions: { variantKey: string; conversions: number }[] = [] if (experiment.goal) { - const table = - experiment.goal.type === 'custom_event' ? 'customEV' : 'analytics' + const eventType = + experiment.goal.type === 'custom_event' ? 'custom_event' : 'pageview' const matchColumn = - experiment.goal.type === 'custom_event' ? 'ev' : 'pg' + experiment.goal.type === 'custom_event' ? 'event_name' : 'pg' const matchCondition = experiment.goal.matchType === 'exact' ? `c.${matchColumn} = {goalValue:String}` @@ -2149,7 +2172,7 @@ Filter modifiers: const conversionsQuery = ` SELECT e.variantKey, uniqExact(e.profileId) as conversions FROM experiment_exposures e - INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) + INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}' WHERE e.pid = {pid:FixedString(12)} AND e.experimentId = {experimentId:String} AND e.created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -2291,8 +2314,9 @@ Filter modifiers: min(created) as startedAt, max(created) as endedAt, count(*) as pageviews - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IS NOT NULL ${extraWhere.join(' ')} @@ -2356,8 +2380,9 @@ Filter modifiers: uniqExactIf(profileId, profileId LIKE 'usr_%') as identifiedProfiles, uniqExact(psid) as sessions, count(*) as pageviews - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL ` @@ -2368,8 +2393,9 @@ Filter modifiers: uniqExact(psid) as sessions, count(*) as pageviews, max(created) as lastSeen - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL GROUP BY profileId @@ -2379,8 +2405,9 @@ Filter modifiers: const topPagesQuery = ` SELECT pg as name, uniqExact(profileId) as profiles - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND pg IS NOT NULL AND pg != '' diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts index 8fa6767c0..4f3063996 100644 --- a/backend/apps/cloud/src/analytics/analytics.controller.ts +++ b/backend/apps/cloud/src/analytics/analytics.controller.ts @@ -81,12 +81,7 @@ import { GetErrorOverviewOptions, } from './dto/get-error-overview.dto' import { PatchStatusDto } from './dto/patch-status.dto' -import { - customEventTransformer, - errorEventTransformer, - performanceTransformer, - trafficTransformer, -} from './utils/transformers' +import { eventTransformer } from './utils/transformers' import { enrichTrafficSource } from './utils/clickIdSources' import { MAX_METRICS_IN_VIEW, @@ -263,10 +258,19 @@ export class AnalyticsController { let diff + const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = + this.analyticsService.getFiltersQuery( + filters, + isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, + ) + if (period === 'all') { + const eventTypes = isCaptcha + ? 'captcha' + : (['pageview', 'custom_event', 'error'] as const) const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + eventTypes, ) diff = res.diff @@ -277,12 +281,6 @@ export class AnalyticsController { allowedTumebucketForPeriodAll = res.timeBucket } - const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = - this.analyticsService.getFiltersQuery( - filters, - isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS, - ) - const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFrom, groupTo, groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo( @@ -294,13 +292,11 @@ export class AnalyticsController { diff, ) - let subQuery = `FROM ${ - isCaptcha ? 'captcha' : 'analytics' - } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - - if (customEVFilterApplied && !isCaptcha) { - subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - } + const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery( + filtersQuery, + customEVFilterApplied, + isCaptcha, + ) const paramsData = { params: { @@ -422,12 +418,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -537,12 +536,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -707,13 +709,14 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - timeBucket, - period, - safeTimezone, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + timeBucket, + period, + safeTimezone, + ) await this.analyticsService.checkProjectAccess( pid, uid, @@ -724,7 +727,14 @@ export class AnalyticsController { this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart') - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, + } const result = await this.analyticsService.groupChartByTimeBucket( timeBucket, @@ -795,18 +805,26 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - newTimeBucket, - period, - safeTimezone, - diff, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + newTimeBucket, + period, + safeTimezone, + diff, + ) - const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, + } const result = await this.analyticsService.groupPerfByTimeBucket( newTimeBucket, @@ -850,13 +868,14 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - timeBucket, - period, - safeTimezone, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + timeBucket, + period, + safeTimezone, + ) await this.analyticsService.checkProjectAccess( pid, uid, @@ -870,12 +889,19 @@ export class AnalyticsController { 'GET /analytics/performance/chart', ) - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, + } const chart = await this.analyticsService.getPerfChartData( timeBucket, - from, - to, + groupFrom, + groupTo, filtersQuery, paramsData, safeTimezone, @@ -921,7 +947,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) diff = res.diff @@ -1332,32 +1358,12 @@ export class AnalyticsController { any(os) AS os, any(cc) AS cc, toString(psid) AS psid - FROM - ( - SELECT - psid, - dv, - br, - os, - cc - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT - psid, - dv, - br, - os, - cc - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL GROUP BY psid ` @@ -1464,7 +1470,8 @@ export class AnalyticsController { const { name, message, lineno, colno, filename, stackTrace, meta } = errorDTO - const transformed = errorEventTransformer({ + const transformed = eventTransformer({ + type: 'error', psid, profileId, eid: this.analyticsService.getErrorID(errorDTO), @@ -1496,7 +1503,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'errors', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1618,7 +1625,8 @@ export class AnalyticsController { enrichTrafficSource(eventsDTO) - const transformed = customEventTransformer({ + const transformed = eventTransformer({ + type: 'custom_event', psid, profileId, pid: eventsDTO.pid, @@ -1650,7 +1658,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'customEV', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1789,7 +1797,8 @@ export class AnalyticsController { enrichTrafficSource(logDTO) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1832,7 +1841,8 @@ export class AnalyticsController { ttfb, } = logDTO.perf - perfTransformed = performanceTransformer({ + perfTransformed = eventTransformer({ + type: 'performance', pid: logDTO.pid, host: this.analyticsService.getHostFromOrigin(headers.origin), pg: logDTO.pg, @@ -1860,7 +1870,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1868,7 +1878,7 @@ export class AnalyticsController { if (!_isEmpty(perfTransformed)) { await clickhouse.insert({ - table: 'performance', + table: 'events', format: 'JSONEachRow', values: [perfTransformed], clickhouse_settings: { async_insert: 1 }, @@ -1955,7 +1965,8 @@ export class AnalyticsController { const { deviceType, browserName, browserVersion, osName, osVersion } = await this.analyticsService.getRequestInformation(headers) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1986,7 +1997,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -2039,7 +2050,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] @@ -2124,7 +2135,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) timeBucket = res.timeBucket[0] @@ -2203,7 +2214,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2281,7 +2292,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2293,6 +2304,8 @@ export class AnalyticsController { DataType.ERRORS, true, ) + const [sessionFiltersQuery, sessionFiltersParams] = + this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo( @@ -2322,6 +2335,8 @@ export class AnalyticsController { groupToUTC, newTimeBucket, parsedOptions.showResolved || false, + sessionFiltersQuery, + sessionFiltersParams, ) } @@ -2362,7 +2377,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2467,7 +2482,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] @@ -2546,7 +2561,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] diff = res.diff @@ -2634,7 +2649,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + ['pageview', 'custom_event', 'error'], ) timeBucket = res.timeBucket[0] @@ -2714,7 +2729,7 @@ export class AnalyticsController { if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'customEV', + 'custom_event', ) newTimeBucket = _includes(res.timeBucket, timeBucket) @@ -2728,16 +2743,24 @@ export class AnalyticsController { this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( - from, - to, - newTimeBucket, - period, - safeTimezone, - diff, - ) + const { groupFrom, groupTo, groupFromUTC, groupToUTC } = + this.analyticsService.getGroupFromTo( + from, + to, + newTimeBucket, + period, + safeTimezone, + diff, + ) - const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } + const paramsData = { + params: { + pid, + groupFrom: groupFromUTC, + groupTo: groupToUTC, + ...filtersParams, + }, + } // customEvents comes as a JSON.stringified array from the frontend let customEventsList: string[] = [] diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts index 80c4a8b6a..ba4e2dbcb 100644 --- a/backend/apps/cloud/src/analytics/analytics.service.ts +++ b/backend/apps/cloud/src/analytics/analytics.service.ts @@ -278,6 +278,14 @@ const EXCLUDE_NULL_FOR = [ 'ctp', ] +const ERROR_COLUMN_MAP: Record = { + name: 'error_name', + message: 'error_message', + filename: 'error_filename', +} + +const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type + const generateParamsQuery = ( col: string, subQuery: string, @@ -285,7 +293,8 @@ const generateParamsQuery = ( type: 'traffic' | 'performance' | 'captcha' | 'errors', measure?: PerfMeasure, ): string => { - let columns = [`${col} as name`] + const sqlCol = type === 'errors' ? mapErrorColumn(col) : col + let columns = [`${sqlCol} as name`] // For regions and cities we'll return an array of objects, that will also include the country code and region code // We need the conutry code to display the flag next to the region/city name @@ -309,15 +318,15 @@ const generateParamsQuery = ( const fn = MEASURES_MAP[processedMeasure] - if (col === 'pg' || col === 'host') { + if (sqlCol === 'pg' || sqlCol === 'host') { return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}` } - return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'errors') { - return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'captcha') { @@ -338,6 +347,13 @@ export enum DataType { ERRORS = 'errors', } +type AnalyticsEventType = 'pageview' | 'custom_event' +type EventsAllTimeType = + | AnalyticsEventType + | 'performance' + | 'error' + | 'captcha' + const isValidOrigin = (origins: string[], origin: string) => { const escapeRegex = (str: string) => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -704,6 +720,22 @@ export class AnalyticsService { return CAPTCHA_COLUMNS } + getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType { + return customEVFilterApplied ? 'custom_event' : 'pageview' + } + + buildAnalyticsEventsSubQuery( + filtersQuery: string, + customEVFilterApplied: boolean, + isCaptcha = false, + ): string { + const eventType = isCaptcha + ? 'captcha' + : this.getAnalyticsEventType(customEVFilterApplied) + + return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + } + getGroupFromTo( from: string, to: string, @@ -857,9 +889,10 @@ export class AnalyticsService { pg, created, lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND pg IS NOT NULL ${filtersQuery} @@ -970,15 +1003,17 @@ export class AnalyticsService { async calculateTimeBucketForAllTime( pid: string, - table: 'analytics' | 'customEV' | 'performance' | 'errors', + eventTypes: EventsAllTimeType | readonly EventsAllTimeType[], ): Promise<{ timeBucket: TimeBucketType[] diff: number }> { + const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes] + const { data: fromData } = await clickhouse .query({ - query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`, - query_params: { pid }, + query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`, + query_params: { pid, types }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -1212,8 +1247,8 @@ export class AnalyticsService { ? 'argMin(pg, created)' : 'argMax(pg, created)' const subQueryForPages = isContains - ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` - : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}` + ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` + : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page = {${param}:String}` // For exclusive filter (isNot) we exclude sessions with matching entry/exit page query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})` @@ -1254,6 +1289,11 @@ export class AnalyticsService { ) { sqlColumn = 'meta.value' isArrayDataset = true + } else if (column === 'ev') { + // Public filter contract `ev` maps to the renamed `event_name` column + sqlColumn = 'event_name' + } else if (dataType === DataType.ERRORS) { + sqlColumn = mapErrorColumn(column) } const isNullFilter = @@ -1631,21 +1671,11 @@ export class AnalyticsService { FROM ( SELECT psid, - pg AS value, - created - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - - UNION ALL - - SELECT - psid, - ev AS value, + if(type = 'pageview', pg, event_name) AS value, created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -1693,14 +1723,14 @@ export class AnalyticsService { psid, CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1715,17 +1745,11 @@ export class AnalyticsService { psid, argMin(cc, created) AS cc, argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source - FROM ( - SELECT psid, cc, ref, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, cc, ref, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 + AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid ) SELECT step, type, val, cnt FROM ( @@ -1803,14 +1827,14 @@ export class AnalyticsService { psid, windowFunnel(86400)(created, ${pagesStr}) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1826,19 +1850,12 @@ export class AnalyticsService { any(br) AS br, min(toTimeZone(created, {timezone:String})) AS sessionStart, max(toTimeZone(created, {timezone:String})) AS lastActivity - FROM ( - SELECT psid, pid, cc, os, br, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND psid IN (SELECT psid FROM funnel_qualified) - UNION ALL - SELECT psid, pid, cc, os, br, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) - ) GROUP BY psidCasted, pid ), pageview_counts AS ( @@ -1846,8 +1863,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1857,8 +1874,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1868,8 +1885,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1947,8 +1964,9 @@ export class AnalyticsService { const query = ` SELECT count() as c - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -2044,12 +2062,13 @@ export class AnalyticsService { groupTo: string, ): Promise<{ cc: string; count: number } | null> { const query = ` - SELECT + SELECT cc, count() as count - FROM analytics - WHERE + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND cc IS NOT NULL AND cc != '' @@ -2079,13 +2098,14 @@ export class AnalyticsService { const query = ` WITH counts AS ( - SELECT + SELECT pid, cc, count() as cnt - FROM analytics - WHERE + FROM events + WHERE pid IN {pids:Array(FixedString(12))} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND cc IS NOT NULL AND cc != '' @@ -2128,12 +2148,13 @@ export class AnalyticsService { groupTo: string, ): Promise<{ count: number; uniqueErrors: number }> { const query = ` - SELECT + SELECT count() as count, count(DISTINCT eid) as uniqueErrors - FROM errors - WHERE + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -2159,13 +2180,14 @@ export class AnalyticsService { } const query = ` - SELECT + SELECT pid, count() as count, count(DISTINCT eid) as uniqueErrors - FROM errors - WHERE + FROM events + WHERE pid IN {pids:Array(FixedString(12))} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid ` @@ -2207,12 +2229,15 @@ export class AnalyticsService { } const query = ` - SELECT + SELECT pid, uniqExact(psid) as totalSessions - FROM analytics - WHERE + FROM events + WHERE pid IN {pids:Array(FixedString(12))} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND psid != '' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY pid ` @@ -2277,15 +2302,18 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - let queryAll = ` + const allTimeType = this.getAnalyticsEventType(customEVFilterApplied) + + const queryAll = ` WITH analytics_counts AS ( SELECT count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' ${filtersQuery} ), duration_avg AS ( @@ -2296,6 +2324,14 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) as duration FROM sessions WHERE pid = {pid:FixedString(12)} + AND psid IN ( + SELECT DISTINCT psid + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' + AND psid IS NOT NULL + ${filtersQuery} + ) GROUP BY psid ) ) @@ -2305,36 +2341,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryAll = ` - WITH analytics_counts AS ( - SELECT - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const { data } = await clickhouse .query({ query: queryAll, @@ -2378,7 +2384,7 @@ export class AnalyticsService { const { diff: allDiff, timeBucket: allowedBuckets } = await this.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.getAnalyticsEventType(customEVFilterApplied), ) const allTimeChartBucket = _includes( allowedBuckets, @@ -2419,16 +2425,19 @@ export class AnalyticsService { ) .format('YYYY-MM-DD HH:mm:ss') - let queryCurrent = ` + const periodType = this.getAnalyticsEventType(customEVFilterApplied) + + const queryCurrent = ` WITH analytics_counts AS ( SELECT 1 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} ), @@ -2442,8 +2451,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} @@ -2457,16 +2467,17 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - let queryPrevious = ` + const queryPrevious = ` WITH analytics_counts AS ( SELECT 2 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} ), @@ -2480,8 +2491,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} @@ -2495,83 +2507,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryCurrent = ` - WITH analytics_counts AS ( - SELECT - 1 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - queryPrevious = ` - WITH analytics_counts AS ( - SELECT - 2 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const query = `${queryCurrent} UNION ALL ${queryPrevious}` let { data } = await clickhouse @@ -2666,6 +2601,55 @@ export class AnalyticsService { return result } + private generateAnalyticsAggregationQueryForScope( + timeBucket: TimeBucketType, + filtersQuery: string, + mode: ChartRenderMode, + customEVFilterApplied: boolean, + ): string { + return customEVFilterApplied + ? this.generateCustomEventsAggregationQuery( + timeBucket, + filtersQuery, + mode, + ) + : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode) + } + + private extractAnalyticsChartDataForScope( + result: Array, + xShifted: string[], + customEVFilterApplied: boolean, + mode: ChartRenderMode, + ): IExtractChartData { + const chartData = this.extractChartData(result, xShifted) + + if (customEVFilterApplied) { + const visits = + this.extractCustomEventsChartData(result, xShifted)?._unknown_event || + [] + + chartData.visits = Array.from( + { length: _size(xShifted) }, + (_, index) => visits[index] || 0, + ) + } + + // Propagate the previous cumulative value forward for empty buckets + if (mode === ChartRenderMode.CUMULATIVE) { + for (let i = 1; i < chartData.visits.length; ++i) { + if (chartData.visits[i] === 0) { + chartData.visits[i] = chartData.visits[i - 1] + } + if (chartData.uniques[i] === 0) { + chartData.uniques[i] = chartData.uniques[i - 1] + } + } + } + + return chartData + } + /** * Get simplified chart data for dashboard cards * Returns only x (dates) and visits (pageviews) for a lightweight chart @@ -2696,30 +2680,11 @@ export class AnalyticsService { }, } - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - ChartRenderMode.PERIODICAL, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const visits = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - return { x: xShifted, visits } - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, ChartRenderMode.PERIODICAL, + customEVFilterApplied, ) const { data } = await clickhouse @@ -2727,9 +2692,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) + .then((resultSet) => + resultSet.json(), + ) - const { visits } = this.extractChartData(data, xShifted) + const { visits } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + ChartRenderMode.PERIODICAL, + ) return { x: xShifted, visits } } @@ -2750,7 +2722,7 @@ export class AnalyticsService { if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) { const safeTimezone = this.getSafeTimezone(timezone) - const { groupFrom, groupTo } = this.getGroupFromTo( + const { groupFromUTC, groupToUTC } = this.getGroupFromTo( from, to, ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null, @@ -2758,8 +2730,8 @@ export class AnalyticsService { safeTimezone, ) - _from = groupFrom - _to = groupTo + _from = groupFromUTC + _to = groupToUTC } const result = {} @@ -2779,7 +2751,7 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}` + const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}` const { data } = await clickhouse .query({ @@ -2823,11 +2795,12 @@ export class AnalyticsService { if (_from && _to) { // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well - const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1 + const diff = dayjs.utc(_to).diff(dayjs.utc(_from), 'days') || 1 now = _to periodFormatted = _from - periodSubtracted = dayjs(_from) + periodSubtracted = dayjs + .utc(_from) .subtract(diff, 'days') .format('YYYY-MM-DD HH:mm:ss') } else { @@ -2842,8 +2815,8 @@ export class AnalyticsService { .format('YYYY-MM-DD HH:mm:ss') } - const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` - const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` + const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` + const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` const query = `${queryCurrent} UNION ALL ${queryPrevious}` @@ -2940,8 +2913,8 @@ export class AnalyticsService { const query = ` WITH session_pages AS ( SELECT psid, ${selector} as page - FROM analytics - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid ) SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page @@ -2957,11 +2930,11 @@ export class AnalyticsService { return _map(data, 'page') } - let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` - - if (type === 'ev') { - query = `SELECT ${type} FROM customEV WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` - } + // Public filter contract `ev` reads from the renamed event_name column. + let query = + type === 'ev' + ? `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name` + : `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event') AND ${type} IS NOT NULL GROUP BY ${type}` const { data } = await clickhouse .query({ @@ -2982,7 +2955,9 @@ export class AnalyticsService { ) } - const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` + const sqlCol = mapErrorColumn(type) + + const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}` const { data } = await clickhouse .query({ @@ -3001,13 +2976,16 @@ export class AnalyticsService { type: 'traffic' | 'errors', column: 'br' | 'os', ): Promise> { - const safeTable = type === 'errors' ? 'errors' : 'analytics' + const safeType = + type === 'errors' + ? "type = 'error'" + : "type IN ('pageview', 'custom_event')" const safeVersionCol = column === 'br' ? 'brv' : 'osv' const query = ` - SELECT ${column}, ${safeVersionCol} - FROM ${safeTable} - WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL + SELECT ${column}, ${safeVersionCol} + FROM events + WHERE pid={pid:FixedString(12)} AND ${safeType} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL GROUP BY ${column}, ${safeVersionCol} ` @@ -3067,7 +3045,7 @@ export class AnalyticsService { const query = ` WITH ${withClauses.join(',\n')} - SELECT + SELECT column_name, name, count, @@ -3088,7 +3066,7 @@ export class AnalyticsService { } return ` - SELECT + SELECT '${col}' as column_name, name, count, @@ -3385,8 +3363,8 @@ export class AnalyticsService { pid, psid, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3434,8 +3412,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3462,8 +3440,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3488,8 +3466,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'error' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3511,15 +3489,31 @@ export class AnalyticsService { const baseQuery = ` SELECT ${selector}, + avgOrNull(sessions_data.duration) as sdur, + count(DISTINCT psid) as uniques, count() as count FROM ( - SELECT *, - ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + SELECT + pid, + psid, + ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery + LEFT JOIN ( + SELECT + pid, + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) as duration + FROM sessions + WHERE pid = {pid:FixedString(12)} + GROUP BY pid, psid + ) as sessions_data + ON subquery.pid = sessions_data.pid + AND subquery.psid = sessions_data.psid GROUP BY ${groupBy} ORDER BY ${groupBy} ` @@ -3528,7 +3522,8 @@ export class AnalyticsService { return ` SELECT *, - sum(count) OVER (ORDER BY ${groupBy}) as count + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(uniques) OVER (ORDER BY ${groupBy}) as uniques FROM (${baseQuery}) ` } @@ -3559,9 +3554,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3594,9 +3590,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3622,9 +3619,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3661,9 +3659,10 @@ export class AnalyticsService { FROM ( SELECT pg, dv, br, os, lc, cc, rg, ct, profileId, ${timeBucketFunc}(created) as tz_created - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) as subquery @@ -3674,10 +3673,11 @@ export class AnalyticsService { if (mode === ChartRenderMode.CUMULATIVE) { return ` SELECT - *, - sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews, - sum(uniques) OVER (ORDER BY ${groupBy}) as uniques + ${groupBy}, + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsers FROM (${baseQuery}) + ORDER BY ${groupBy} ` } @@ -3742,7 +3742,7 @@ export class AnalyticsService { ): Promise<{ name: string; count: number }[]> { const query = ` WITH session_first_pages AS ( - SELECT + SELECT psid, argMin(pg, created) as entry_page ${subQuery} @@ -3837,39 +3837,11 @@ export class AnalyticsService { ): Promise { const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone) - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - mode, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const uniques = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - const sdur = Array(_size(xShifted)).fill(0) - - return Promise.resolve({ - chart: { - x: xShifted, - visits: uniques, - uniques, - sdur, - }, - }) - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, mode, + customEVFilterApplied, ) const { data } = await clickhouse @@ -3877,21 +3849,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) - - const { visits, uniques, sdur } = this.extractChartData(data, xShifted) + .then((resultSet) => + resultSet.json(), + ) - // Propagate the previous cumulative value forward for empty buckets - if (mode === ChartRenderMode.CUMULATIVE) { - for (let i = 1; i < visits.length; ++i) { - if (visits[i] === 0) { - visits[i] = visits[i - 1] - } - if (uniques[i] === 0) { - uniques[i] = uniques[i - 1] - } - } - } + const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + mode, + ) return Promise.resolve({ chart: { @@ -4431,7 +4398,7 @@ export class AnalyticsService { filtersQuery: string, params: any, ): Promise { - const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` + const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name` const result = {} const { data } = await clickhouse @@ -4461,10 +4428,10 @@ export class AnalyticsService { FROM ( SELECT meta.key AS key - FROM - analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -4518,7 +4485,7 @@ export class AnalyticsService { ) if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'customEV') + const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event') diff = res.diff @@ -4546,12 +4513,12 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND ev = {event:String} + AND event_name = {event:String} ${filtersQuery} ) ARRAY JOIN meta.key, meta.value @@ -4610,7 +4577,7 @@ export class AnalyticsService { const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = this.getFiltersQuery(filters, DataType.ANALYTICS) - // We cannot make a query to customEV table using analytics table properties + // Page properties are only present on pageview-scoped analytics rows. if (customEVFilterApplied) { return { result: [], @@ -4619,7 +4586,7 @@ export class AnalyticsService { } if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'analytics') + const res = await this.calculateTimeBucketForAllTime(pid, 'pageview') diff = res.diff @@ -4647,10 +4614,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND indexOf(meta.key, {property:String}) > 0 ${filtersQuery} @@ -4700,17 +4667,11 @@ export class AnalyticsService { const query = ` SELECT uniqExact(psid) as count - FROM ( - SELECT psid FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT psid FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL ` try { @@ -4748,25 +4709,26 @@ export class AnalyticsService { const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})` const customEventsFilter = customEvents && customEvents.length > 0 - ? 'AND ev IN {customEvents:Array(String)}' + ? 'AND event_name IN {customEvents:Array(String)}' : '' const query = ` SELECT ${selector}, - ev, + event_name AS ev, count() as count FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ${customEventsFilter} ) as subquery - GROUP BY ${groupBy}, ev + GROUP BY ${groupBy}, event_name ORDER BY ${groupBy} ` @@ -4864,52 +4826,47 @@ export class AnalyticsService { const queryPages = ` WITH events_with_meta AS ( SELECT - 'pageview' AS type, - toString(pg) AS value, - toTimeZone(analytics.created, {timezone:String}) AS created, - pid, - toString(analytics.psid) AS psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND toString(analytics.psid) = {psid:String} - - UNION ALL - - SELECT - 'event' AS type, - toString(ev) AS value, - toTimeZone(customEV.created, {timezone:String}) AS created, - pid, - toString(customEV.psid) AS psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND toString(customEV.psid) = {psid:String} - - UNION ALL - - SELECT - 'error' AS type, - toString(errors.name) AS value, - toTimeZone(errors.created, {timezone:String}) AS created, + multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type, + multiIf( + event_type = 'pageview', toString(pg), + event_type = 'custom_event', toString(event_name), + toString(error_name) + ) AS value, + toTimeZone(created, {timezone:String}) AS created, pid, - toString(errors.psid) AS psid, - [ - tuple('message', COALESCE(errors.message, '')), - tuple('lineno', toString(COALESCE(errors.lineno, 0))), - tuple('colno', toString(COALESCE(errors.colno, 0))), - tuple('filename', COALESCE(errors.filename, '')) - ] AS metadata - FROM errors - WHERE - pid = {pid:FixedString(12)} - AND errors.psid IS NOT NULL - AND toString(errors.psid) = {psid:String} + toString(psid) AS psid, + if( + event_type = 'error', + arrayFilter(x -> x.2 != '' AND x.2 != '0', [ + tuple('message', COALESCE(error_message, '')), + tuple('lineno', ifNull(toString(lineno), '')), + tuple('colno', ifNull(toString(colno), '')), + tuple('filename', COALESCE(error_filename, '')) + ]), + arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value)) + ) AS metadata + FROM ( + SELECT + type AS event_type, + pg, + event_name, + error_name, + error_message, + lineno, + colno, + error_filename, + created, + pid, + psid, + meta.key AS meta_key, + meta.value AS meta_value + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} + ) UNION ALL @@ -4961,12 +4918,19 @@ export class AnalyticsService { const querySessionDetails = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} - ORDER BY created ASC + ORDER BY + CASE + WHEN type = 'pageview' THEN 0 + WHEN type = 'custom_event' THEN 1 + ELSE 2 + END, + created ASC LIMIT 1; ` @@ -5057,9 +5021,10 @@ export class AnalyticsService { const querySessionDetailsFromCustomEV = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('custom_event', 'error') AND psid IS NOT NULL AND toString(psid) = {psid:String} ORDER BY created ASC @@ -5149,94 +5114,16 @@ export class AnalyticsService { skip = 0, customEVFilterApplied = false, ): Promise { - let primaryEventsSubquery: string - - if (customEVFilterApplied) { - // When filtering by custom events, we need to: - // 1. Identify sessions (psids) that have matching custom events - // 2. Calculate session boundaries from ALL analytics events for those sessions - primaryEventsSubquery = ` - SELECT - all_events.psidCasted, - all_events.pid, - all_events.cc, - all_events.os, - all_events.br, - all_events.created_for_grouping - FROM ( - SELECT - CAST(analytics.psid, 'String') AS psidCasted, - analytics.pid, - analytics.cc, - analytics.os, - analytics.br, - toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping - FROM analytics - WHERE - analytics.pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ) AS all_events - WHERE all_events.psidCasted IN ( - SELECT DISTINCT CAST(customEV.psid, 'String') - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - ` - } else { - primaryEventsSubquery = ` - SELECT - CAST(analytics.psid, 'String') AS psidCasted, - analytics.pid, - analytics.cc, - analytics.os, - analytics.br, - toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping - FROM analytics - WHERE - analytics.pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ` - } + const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH distinct_sessions_filtered AS ( SELECT psidCasted, - pid, + pid, any(cc) AS cc, any(os) AS os, any(br) AS br, @@ -5246,32 +5133,32 @@ export class AnalyticsService { GROUP BY psidCasted, pid ), pageview_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + pid, + count() as count + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), event_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + pid, + count() as count + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), error_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + pid, + count() as count + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), @@ -5296,9 +5183,9 @@ export class AnalyticsService { GROUP BY psidCasted, pid ), session_duration_agg AS ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, + SELECT + CAST(psid, 'String') AS psidCasted, + pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, argMax(profileId, lastSeen) as profileId FROM sessions @@ -5360,75 +5247,65 @@ export class AnalyticsService { return data } - async getProfilesList( - pid: string, + private buildSessionsListPrimaryEventsSubquery( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - profileType: 'all' | 'anonymous' | 'identified' = 'all', - customEVFilterApplied = false, - ): Promise { - let profileTypeFilter = '' - if (profileType === 'anonymous') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` - } else if (profileType === 'identified') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` - } + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + ` + : filtersQuery - let allProfileDataCTE: string + return ` + SELECT + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${scopedSessionFilter} + ` + } - if (customEVFilterApplied) { - allProfileDataCTE = ` - all_profile_data AS ( - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} + private buildProfilesListDataCTE( + filtersQuery: string, + profileTypeFilter: string, + customEVFilterApplied: boolean, + ): string { + const scopedProfileFilter = customEVFilterApplied + ? ` AND profileId IN ( SELECT DISTINCT profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} ${filtersQuery} ) - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} - )` - } else { - allProfileDataCTE = ` + ` + : filtersQuery + + return ` all_profile_data AS ( SELECT profileId, @@ -5438,36 +5315,43 @@ export class AnalyticsService { br, dv, created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics + if(type = 'pageview', 1, 0) AS isPageview, + if(type = 'custom_event', 1, 0) AS isEvent, + if(type = 'error', 1, 0) AS isError + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} - ${filtersQuery} - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} + ${scopedProfileFilter} )` + } + + async getProfilesList( + pid: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + profileType: 'all' | 'anonymous' | 'identified' = 'all', + customEVFilterApplied = false, + ): Promise { + let profileTypeFilter = '' + if (profileType === 'anonymous') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` + } else if (profileType === 'identified') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` } + const allProfileDataCTE = this.buildProfilesListDataCTE( + filtersQuery, + profileTypeFilter, + customEVFilterApplied, + ) + const query = ` WITH ${allProfileDataCTE}, profile_aggregated AS ( @@ -5476,6 +5360,7 @@ export class AnalyticsService { sum(isPageview) AS pageviewsCount, countDistinct(psid) AS sessionsCount, sum(isEvent) AS eventsCount, + sum(isError) AS errorsCount, min(created) AS firstSeen, max(created) AS lastSeen, any(cc) AS cc_agg, @@ -5484,25 +5369,13 @@ export class AnalyticsService { any(dv) AS dv_agg FROM all_profile_data GROUP BY profileId - ), - profile_errors AS ( - SELECT - profileId, - count() AS errorsCount - FROM errors - WHERE pid = {pid:FixedString(12)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - GROUP BY profileId ) SELECT pa.profileId AS profileId, pa.sessionsCount AS sessionsCount, pa.pageviewsCount AS pageviewsCount, pa.eventsCount AS eventsCount, - COALESCE(perr.errorsCount, 0) AS errorsCount, + pa.errorsCount AS errorsCount, pa.firstSeen AS firstSeen, pa.lastSeen AS lastSeen, pa.cc_agg AS cc, @@ -5510,7 +5383,6 @@ export class AnalyticsService { pa.br_agg AS br, pa.dv_agg AS dv FROM profile_aggregated AS pa - LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileId ORDER BY pa.lastSeen DESC LIMIT {take:UInt32} OFFSET {skip:UInt32} @@ -5547,40 +5419,59 @@ export class AnalyticsService { AND profileId = {profileId:String} ` - // Query avg duration from analytics table (more accurate than sessions table) + // Prefer pageview timings, but fall back to sessions for event/error-only profiles. const queryAvgDuration = ` - SELECT - avg(session_duration) AS avgDuration - FROM ( + WITH pageview_avg AS ( SELECT - psid, - dateDiff('second', min(created), max(created)) AS session_duration - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - GROUP BY psid - HAVING session_duration > 0 + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(created), max(created)) AS session_duration + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND profileId = {profileId:String} + AND psid IS NOT NULL + GROUP BY psid + HAVING session_duration > 0 + ) + ), + sessions_avg AS ( + SELECT + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration + FROM sessions FINAL + WHERE pid = {pid:FixedString(12)} + AND profileId = {profileId:String} + GROUP BY psid + HAVING session_duration > 0 + ) ) + SELECT + coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration + FROM pageview_avg, sessions_avg ` - // Query analytics for accurate pageview count const queryPageviews = ` SELECT count() AS pageviewsCount - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} ` - // Query customEV for accurate events count const queryEvents = ` SELECT count() AS eventsCount - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} ` - // Query for device/location details const queryDetails = ` SELECT any(cc) AS cc, @@ -5592,9 +5483,10 @@ export class AnalyticsService { any(brv) AS brv, any(dv) AS dv, any(lc) AS lc - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} AND profileId = {profileId:String} + AND type IN ('pageview', 'custom_event', 'error') ` // Query for total revenue from profile (only sales, not refunds) @@ -5679,8 +5571,9 @@ export class AnalyticsService { SELECT pg AS page, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} GROUP BY pg ORDER BY count DESC @@ -5709,30 +5602,15 @@ export class AnalyticsService { .format('YYYY-MM-DD') const query = ` - WITH all_activity AS ( - SELECT - toDate(created) AS date, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - UNION ALL - SELECT - toDate(created) AS date, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - ) SELECT - date, - sum(isPageview) AS pageviews, - sum(isEvent) AS events - FROM all_activity + toDate(created) AS date, + countIf(type = 'pageview') AS pageviews, + countIf(type = 'custom_event') AS events + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND profileId = {profileId:String} + AND created >= {startDate:Date} GROUP BY date ORDER BY date ASC ` @@ -5767,8 +5645,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5779,8 +5658,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5791,8 +5671,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5853,70 +5734,26 @@ export class AnalyticsService { } } - async getProfileSessionsList( - pid: string, - profileId: string, + private buildProfileSessionsEventsCTE( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - customEVFilterApplied = false, - ): Promise { - let allProfileEventsCTE: string + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND profileId = {profileId:String} + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + ` + : filtersQuery - if (customEVFilterApplied) { - allProfileEventsCTE = ` - all_profile_events AS ( - SELECT - all_events.psidCasted, - all_events.pid, - all_events.profileId, - all_events.cc, - all_events.os, - all_events.br, - all_events.created_tz - FROM ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) AS all_events - WHERE all_events.psidCasted IN ( - SELECT DISTINCT CAST(psid, 'String') - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - )` - } else { - allProfileEventsCTE = ` + return ` all_profile_events AS ( SELECT CAST(psid, 'String') AS psidCasted, @@ -5926,29 +5763,30 @@ export class AnalyticsService { os, br, toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} + ${scopedSessionFilter} )` - } + } + + async getProfileSessionsList( + pid: string, + profileId: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + customEVFilterApplied = false, + ): Promise { + const allProfileEventsCTE = this.buildProfileSessionsEventsCTE( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH ${allProfileEventsCTE}, @@ -5970,8 +5808,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -5982,8 +5821,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -5994,8 +5834,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6081,10 +5922,18 @@ export class AnalyticsService { count(DISTINCT psid) as sessions, status.status FROM ( - SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created - FROM errors + SELECT + eid, + error_name AS name, + error_message AS message, + error_filename AS filename, + psid, + profileId, + toTimeZone(created, {timezone:String}) AS created + FROM events WHERE pid = {pid:FixedString(12)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} + AND type = 'error' + AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) AS errors LEFT JOIN ( @@ -6098,7 +5947,7 @@ export class AnalyticsService { ${ parsedOptions?.showResolved ? '' - : "WHERE status.status = 'active' OR status.status = 'regressed'" + : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL" } GROUP BY errors.eid, status.status ORDER BY last_seen DESC @@ -6143,14 +5992,15 @@ export class AnalyticsService { FROM ( SELECT eid, - name, - message, - filename, + error_name AS name, + error_message AS message, + error_filename AS filename, colno, lineno, stackTrace - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ORDER BY created DESC @@ -6173,10 +6023,12 @@ export class AnalyticsService { max(created) AS last_seen, min(created) AS first_seen, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} - AND eid = {eid:FixedString(32)}; + AND type = 'error' + AND eid = {eid:FixedString(32)} + AND created BETWEEN {groupFrom:String} AND {groupTo:String}; ` const queryMetadata = ` @@ -6188,9 +6040,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -6237,8 +6090,8 @@ export class AnalyticsService { timeBucket, groupFrom, groupTo, - `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, - 'AND eid = {eid:FixedString(32)}', + `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, + 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}', paramsData, safeTimezone, ChartRenderMode.PERIODICAL, @@ -6264,17 +6117,23 @@ export class AnalyticsService { groupTo: string, timeBucket: string, showResolved: boolean, + sessionFiltersQuery = '', + sessionFiltersParams: Record = {}, ): Promise { const resolvedFilter = showResolved ? '' : "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)" - // Get total sessions from analytics table for the time range + // Get total sessions from all matching events for the time range const queryTotalSessions = ` SELECT count(DISTINCT psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND psid != '' AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${sessionFiltersQuery} ` // Get error stats: total errors, unique errors, affected sessions, affected users @@ -6284,7 +6143,7 @@ export class AnalyticsService { count(DISTINCT eid) as uniqueErrors, count(DISTINCT psid) as affectedSessions, count(DISTINCT profileId) as affectedUsers - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -6292,6 +6151,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -6301,12 +6161,12 @@ export class AnalyticsService { const queryMostFrequentError = ` SELECT errors.eid as eid, - any(errors.name) as name, - any(errors.message) as message, + any(errors.error_name) as name, + any(errors.error_message) as message, count(*) as count, count(DISTINCT errors.profileId) as usersAffected, max(errors.created) as lastSeen - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -6314,6 +6174,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -6338,7 +6199,7 @@ export class AnalyticsService { SELECT profileId, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -6346,6 +6207,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -6368,7 +6230,10 @@ export class AnalyticsService { clickhouse .query({ query: queryTotalSessions, - query_params: paramsData.params, + query_params: { + ...paramsData.params, + ...sessionFiltersParams, + }, }) .then((resultSet) => resultSet.json()) .then(({ data }) => data[0]), @@ -6457,29 +6322,62 @@ export class AnalyticsService { ): Promise<{ sessions: any[]; total: number }> { const queryCount = ` SELECT count(DISTINCT psid) as total - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` const querySessions = ` - SELECT DISTINCT + SELECT CAST(errors.psid, 'String') as psid, - any(errors.profileId) as profileId, + errors.profileId as profileId, any(analytics.cc) as cc, any(analytics.br) as br, any(analytics.os) as os, - min(errors.created) as firstErrorAt, - max(errors.created) as lastErrorAt, - count(*) as errorCount - FROM errors - LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid - WHERE errors.pid = {pid:FixedString(12)} - AND errors.eid = {eid:FixedString(32)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} - GROUP BY errors.psid - ORDER BY lastErrorAt DESC + errors.firstErrorAt as firstErrorAt, + errors.lastErrorAt as lastErrorAt, + errors.errorCount as errorCount + FROM ( + SELECT + pid, + psid, + any(profileId) AS profileId, + min(created) AS firstErrorAt, + max(created) AS lastErrorAt, + count(*) AS errorCount + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'error' + AND eid = {eid:FixedString(32)} + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY pid, psid + ) AS errors + LEFT JOIN ( + SELECT + pid, + psid, + any(cc) AS cc, + any(br) AS br, + any(os) AS os, + countIf(type = 'pageview') AS pageviews + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'error', 'custom_event') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + GROUP BY pid, psid + ) AS analytics + ON errors.psid = analytics.psid + AND errors.pid = analytics.pid + GROUP BY + errors.psid, + errors.profileId, + errors.firstErrorAt, + errors.lastErrorAt, + errors.errorCount + ORDER BY errors.lastErrorAt DESC LIMIT {take:UInt32} OFFSET {skip:UInt32} ` @@ -6513,6 +6411,10 @@ export class AnalyticsService { } async validateEIDs(eids: string[], pid: string) { + if (_isEmpty(eids)) { + throw new BadRequestException('At least one error ID is required') + } + const params = _reduce( eids, (acc, curr, index) => ({ @@ -6522,7 +6424,7 @@ export class AnalyticsService { {}, ) - const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map( + const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map( params, (val, key) => `{${key}:FixedString(32)}`, ).join(', ')})` @@ -6604,15 +6506,7 @@ export class AnalyticsService { events: number }> { const query = ` - SELECT 'traffic' AS type, count(*) AS count FROM analytics - UNION ALL - SELECT 'customEV' AS type, count(*) AS count FROM customEV - UNION ALL - SELECT 'performance' AS type, count(*) AS count FROM performance - UNION ALL - SELECT 'captcha' AS type, count(*) AS count FROM captcha - UNION ALL - SELECT 'errors' AS type, count(*) AS count FROM errors + SELECT count(*) AS count FROM events ` const users = await this.userService.count({ @@ -6732,12 +6626,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodFormatted:String} AND {now:String} ${kvQuery} ${filtersQuery} @@ -6750,12 +6645,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${kvQuery} ${filtersQuery} diff --git a/backend/apps/cloud/src/analytics/utils/transformers.ts b/backend/apps/cloud/src/analytics/utils/transformers.ts index b420cdac8..7ec1ed491 100644 --- a/backend/apps/cloud/src/analytics/utils/transformers.ts +++ b/backend/apps/cloud/src/analytics/utils/transformers.ts @@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc' dayjs.extend(utc) const processMetaKV = ( - rawMeta: Record, + rawMeta: Record | null | undefined, ): { 'meta.key': string[] | null; 'meta.value': string[] | null } => { if (!rawMeta || _isEmpty(rawMeta)) { return { @@ -23,10 +23,10 @@ const processMetaKV = ( } } -interface TrafficTransformerOptions { - psid: string - profileId: string +interface CommonOptions { pid: string + psid?: string | null + profileId?: string | null host?: string | null pg?: string | null dv?: string | null @@ -52,262 +52,42 @@ interface TrafficTransformerOptions { meta?: Record | null } -export const trafficTransformer = ({ - psid, - profileId, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: TrafficTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } +interface PageviewOptions extends CommonOptions { + type: 'pageview' } -interface CustomEventTransformerOptions { - psid: string - profileId: string - pid: string - host?: string | null +interface CustomEventOptions extends CommonOptions { + type: 'custom_event' ev: string - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - ref?: string | null - so?: string | null - me?: string | null - ca?: string | null - te?: string | null - co?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null - meta?: Record | null -} - -export const customEventTransformer = ({ - psid, - profileId, - pid, - host, - ev, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: CustomEventTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - ev, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface ErrorEventTransformerOptions { - psid: string - profileId: string +interface ErrorEventOptions extends CommonOptions { + type: 'error' eid: string - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null name?: string | null message?: string | null lineno?: number | null colno?: number | null filename?: string | null stackTrace?: string | null - meta?: Record | null -} - -export const errorEventTransformer = ({ - psid, - profileId, - eid, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - name, - message, - lineno, - colno, - filename, - stackTrace, - meta, -}: ErrorEventTransformerOptions) => { - return { - psid, - profileId, - eid, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - name: name || null, - message: message || null, - lineno: lineno || null, - colno: colno || null, - filename: filename || null, - stackTrace: stackTrace || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface PerformanceTransformerOptions { - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null +interface PerformanceOptions extends Omit< + CommonOptions, + | 'psid' + | 'profileId' + | 'os' + | 'osv' + | 'lc' + | 'ref' + | 'so' + | 'me' + | 'ca' + | 'te' + | 'co' + | 'meta' +> { + type: 'performance' dns: number tls: number conn: number @@ -318,53 +98,131 @@ interface PerformanceTransformerOptions { ttfb: number } -export const performanceTransformer = ({ - pid, - host, - pg, - dv, - br, - brv, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - dns, - tls, - conn, - response, - render, - domLoad, - pageLoad, - ttfb, -}: PerformanceTransformerOptions) => { +interface CaptchaOptions { + type: 'captcha' + pid: string + dv?: string | null + br?: string | null + os?: string | null + cc?: string | null + timestamp?: number +} + +type EventTransformerOptions = + | PageviewOptions + | CustomEventOptions + | ErrorEventOptions + | PerformanceOptions + | CaptchaOptions + +const buildCommon = (opts: CommonOptions) => ({ + pid: opts.pid, + psid: opts.psid ?? null, + profileId: opts.profileId ?? null, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + os: opts.os || null, + osv: opts.osv || null, + lc: opts.lc || null, + ref: opts.ref || null, + so: opts.so || null, + me: opts.me || null, + ca: opts.ca || null, + te: opts.te || null, + co: opts.co || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + ...processMetaKV(opts.meta), +}) + +// Single transformer for all event kinds; the discriminator is `type`. +// Renames at the storage boundary: ev -> event_name, name -> error_name, +// message -> error_message, filename -> error_filename. DTO and API field +// names stay as the legacy short forms. +export const eventTransformer = (opts: EventTransformerOptions) => { + const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss') + + if (opts.type === 'pageview') { + return { + type: 'pageview' as const, + ...buildCommon(opts), + created, + } + } + + if (opts.type === 'custom_event') { + return { + type: 'custom_event' as const, + ...buildCommon(opts), + event_name: opts.ev, + created, + } + } + + if (opts.type === 'error') { + return { + type: 'error' as const, + ...buildCommon(opts), + eid: opts.eid, + error_name: opts.name || null, + error_message: opts.message || null, + stackTrace: opts.stackTrace || null, + lineno: opts.lineno ?? null, + colno: opts.colno ?? null, + error_filename: opts.filename || null, + created, + } + } + + if (opts.type === 'performance') { + return { + type: 'performance' as const, + pid: opts.pid, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + dns: _round(opts.dns), + tls: _round(opts.tls), + conn: _round(opts.conn), + response: _round(opts.response), + render: _round(opts.render), + domLoad: _round(opts.domLoad), + pageLoad: _round(opts.pageLoad), + ttfb: _round(opts.ttfb), + created, + } + } + + // captcha return { - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - dns: _round(dns), - tls: _round(tls), - conn: _round(conn), - response: _round(response), - render: _round(render), - domLoad: _round(domLoad), - pageLoad: _round(pageLoad), - ttfb: _round(ttfb), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), + type: 'captcha' as const, + pid: opts.pid, + dv: opts.dv || null, + br: opts.br || null, + os: opts.os || null, + cc: opts.cc || null, + created: + typeof opts.timestamp === 'number' + ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss') + : created, } } diff --git a/backend/apps/cloud/src/captcha/captcha.service.ts b/backend/apps/cloud/src/captcha/captcha.service.ts index 921e6d7b7..99ac3516c 100644 --- a/backend/apps/cloud/src/captcha/captcha.service.ts +++ b/backend/apps/cloud/src/captcha/captcha.service.ts @@ -17,7 +17,7 @@ import { } from '../common/constants' import { getIPDetails } from '../common/utils' import { GeneratedChallenge } from './interfaces/generated-captcha' -import { captchaTransformer } from './utils/transformers' +import { eventTransformer } from '../analytics/utils/transformers' import { clickhouse } from '../common/integrations/clickhouse' dayjs.extend(utc) @@ -152,18 +152,19 @@ export class CaptchaService { const osName = ua.os.name const { country } = getIPDetails(ip) - const transformed = captchaTransformer( + const transformed = eventTransformer({ + type: 'captcha', pid, - deviceType, - browserName, - osName, - country, + dv: deviceType, + br: browserName, + os: osName, + cc: country, timestamp, - ) + }) try { await clickhouse.insert({ - table: 'captcha', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { diff --git a/backend/apps/cloud/src/captcha/utils/transformers.ts b/backend/apps/cloud/src/captcha/utils/transformers.ts deleted file mode 100644 index 3a0d21d43..000000000 --- a/backend/apps/cloud/src/captcha/utils/transformers.ts +++ /dev/null @@ -1,22 +0,0 @@ -import dayjs from 'dayjs' -import utc from 'dayjs/plugin/utc' - -dayjs.extend(utc) - -export const captchaTransformer = ( - pid: string, - dv: string | null, - br: string | null, - os: string | null, - cc: string | null, - timestamp: number, -) => { - return { - pid, - dv: dv || null, - br: br || null, - os: os || null, - cc: cc || null, - created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'), - } -} diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts index 18799910d..019945a24 100644 --- a/backend/apps/cloud/src/data-import/data-import.processor.ts +++ b/backend/apps/cloud/src/data-import/data-import.processor.ts @@ -107,8 +107,7 @@ export class DataImportProcessor extends WorkerHost { let minDate: string | null = null let maxDate: string | null = null - const analyticsBatch: Record[] = [] - const customEVBatch: Record[] = [] + const eventsBatch: Record[] = [] const context = isApiBased ? { @@ -142,28 +141,17 @@ export class DataImportProcessor extends WorkerHost { if (!maxDate || created > maxDate) maxDate = created } - if (row.table === 'analytics') { - analyticsBatch.push(row.data) - } else { - customEVBatch.push(row.data) - } - - if (analyticsBatch.length >= BATCH_SIZE) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - analyticsBatch.length = 0 - } + eventsBatch.push({ type: row.type, ...row.data }) - if (customEVBatch.length >= BATCH_SIZE) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length - customEVBatch.length = 0 + if (eventsBatch.length >= BATCH_SIZE) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length + eventsBatch.length = 0 } if (totalRows % 10000 === 0) { const progress = { - importedRows: - importedRows + analyticsBatch.length + customEVBatch.length, + importedRows: importedRows + eventsBatch.length, totalRows, } @@ -181,14 +169,9 @@ export class DataImportProcessor extends WorkerHost { } } - if (analyticsBatch.length > 0) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - } - - if (customEVBatch.length > 0) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length + if (eventsBatch.length > 0) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length } await this.dataImportService.markCompleted(id, { @@ -235,12 +218,9 @@ export class DataImportProcessor extends WorkerHost { } } - private async flushBatch( - table: 'analytics' | 'customEV', - batch: Record[], - ): Promise { + private async flushBatch(batch: Record[]): Promise { await clickhouse.insert({ - table: `${CLICKHOUSE_DB}.${table}`, + table: `${CLICKHOUSE_DB}.events`, values: batch, format: 'JSONEachRow', }) diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.ts index 63f60b59e..834b53086 100644 --- a/backend/apps/cloud/src/data-import/data-import.service.ts +++ b/backend/apps/cloud/src/data-import/data-import.service.ts @@ -161,11 +161,7 @@ export class DataImportService { importId: number, ): Promise { await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, - query_params: { pid: projectId, importID: importId }, - }) - await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`, query_params: { pid: projectId, importID: importId }, }) } @@ -207,7 +203,7 @@ export class DataImportService { ): Promise { const result = await clickhouse .query({ - query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, + query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, query_params: { pid: projectId, from, to }, }) .then((resultSet) => resultSet.json<{ cnt: string }>()) diff --git a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts index f74c72122..b9dce040d 100644 --- a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts @@ -206,13 +206,13 @@ export class FathomMapper implements ImportMapper { if (fileType === 'pageviews') { for (let i = 0; i < count; i++) { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' - const evData = { ...baseData, ev: eventName } + const evData = { ...baseData, event_name: eventName } for (let i = 0; i < count; i++) { - yield { table: 'customEV', data: evData } + yield { type: 'custom_event', data: evData } } } } diff --git a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts index 5cd625c58..2ee657aa9 100644 --- a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts @@ -357,7 +357,7 @@ export class Ga4Mapper implements ImportMapper { } for (let i = 0; i < count; i++) { - yield { table: 'analytics', data } + yield { type: 'pageview', data } } } @@ -441,7 +441,7 @@ export class Ga4Mapper implements ImportMapper { pid, host: truncate(host, 253), pg: truncate(pagePath, 2048), - ev: truncate(eventName, 256), + event_name: truncate(eventName, 256), dv: device, br: truncate(browser, 30), brv: null, @@ -465,7 +465,7 @@ export class Ga4Mapper implements ImportMapper { } for (let i = 0; i < count; i++) { - yield { table: 'customEV', data } + yield { type: 'custom_event', data } } } diff --git a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts index 0351b00b0..cf19c69c8 100644 --- a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts +++ b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts @@ -6,7 +6,7 @@ export class ImportError extends Error { } export interface AnalyticsImportRow { - table: 'analytics' | 'customEV' + type: 'pageview' | 'custom_event' data: Record } diff --git a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts index 99141bb58..f0c5926d3 100644 --- a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts @@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper { created, }) - yield { table: 'analytics', data } + yield { type: 'pageview', data } } } @@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper { page: null, created, }) - data.ev = truncate(ev.name, 256) || '' - yield { table: 'customEV', data } + data.event_name = truncate(ev.name, 256) || '' + yield { type: 'custom_event', data } } } } diff --git a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts index 880b863b9..b95e37380 100644 --- a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts @@ -176,11 +176,11 @@ export class SimpleAnalyticsMapper implements ImportMapper { const datapoint = normalizeNull(row.datapoint) if (!datapoint || datapoint === 'pageview') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { yield { - table: 'customEV', - data: { ...baseData, ev: truncate(datapoint, 256) || '' }, + type: 'custom_event', + data: { ...baseData, event_name: truncate(datapoint, 256) || '' }, } } } diff --git a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts index 192a9638b..3d8c9d9b8 100644 --- a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts +++ b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts @@ -372,12 +372,12 @@ export class UmamiMapper implements ImportMapper { } if (eventType === '1') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' yield { - table: 'customEV', - data: { ...baseData, ev: eventName }, + type: 'custom_event', + data: { ...baseData, event_name: eventName }, } } } diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts index 52f230267..869176359 100644 --- a/backend/apps/cloud/src/experiment/experiment.controller.ts +++ b/backend/apps/cloud/src/experiment/experiment.controller.ts @@ -60,6 +60,7 @@ import { } from './dto/experiment.dto' import { ExperimentService } from './experiment.service' import { GoalService } from '../goal/goal.service' +import { Goal, GoalMatchType, GoalType } from '../goal/entity/goal.entity' import { FeatureFlagService } from '../feature-flag/feature-flag.service' import { FeatureFlag, @@ -73,6 +74,15 @@ import { Pagination } from '../common/pagination' const EXPERIMENTS_MAXIMUM = 20 // Maximum experiments per project const FEATURE_FLAG_KEY_REGEX = /^[a-zA-Z0-9_-]+$/ +type GoalEventConditions = { + eventType: 'pageview' | 'custom_event' + matchColumn: 'pg' | 'event_name' + matchCondition: string + metaCondition: string + metaParams: Record + goalValue: string +} + @ApiTags('Experiment') @Controller(['experiment', 'v1/experiment']) export class ExperimentController { @@ -881,11 +891,14 @@ export class ExperimentController { timeBucketParam || getLowestPossibleTimeBucket(period, from, to) let allowedTimeBucketForPeriodAll: TimeBucketType[] | undefined let diff: number | undefined + const goalConditions = experiment.goal + ? this.buildGoalEventConditions(experiment.goal) + : null if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( experiment.project.id, - 'analytics', + goalConditions?.eventType || 'pageview', ) diff = res.diff @@ -938,43 +951,21 @@ export class ExperimentController { } let conversionsData: { variantKey: string; conversions: number }[] = [] - if (experiment.goal) { - const goalType = experiment.goal.type - const table = goalType === 'custom_event' ? 'customEV' : 'analytics' - const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg' - const goalValue = experiment.goal.value || '' - - let matchCondition = '' - if (experiment.goal.matchType === 'exact') { - matchCondition = `c.${matchColumn} = {goalValue:String}` - } else { - matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` - } - - let metaCondition = '' - const metaParams: Record = {} - const metadataFilters = experiment.goal.metadataFilters - if (metadataFilters && metadataFilters.length > 0) { - const conditions: string[] = [] - metadataFilters.forEach((filter, index) => { - const keyParam = `metaKey${index}` - const valueParam = `metaValue${index}` - metaParams[keyParam] = filter.key - metaParams[valueParam] = filter.value - conditions.push( - `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`, - ) - }) - - metaCondition = `AND (${conditions.join(' AND ')})` - } + if (goalConditions) { + const { + eventType, + matchCondition, + metaCondition, + metaParams, + goalValue, + } = goalConditions const conversionsQuery = ` SELECT e.variantKey, uniqExact(e.profileId) as conversions FROM experiment_exposures e - INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) + INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}' WHERE e.pid = {pid:FixedString(12)} AND e.experimentId = {experimentId:String} @@ -1105,6 +1096,50 @@ export class ExperimentController { } } + private buildGoalEventConditions(goal: Goal): GoalEventConditions { + const eventType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' + const matchColumn = + goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg' + const goalValue = goal.value || '' + const metaParams: Record = {} + + let matchCondition = '' + if (goal.matchType === GoalMatchType.EXACT) { + matchCondition = `c.${matchColumn} = {goalValue:String}` + } else if (goalValue.trim() === '') { + matchCondition = '1=0' + } else { + matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` + } + + let metaCondition = '' + if (goal.metadataFilters && goal.metadataFilters.length > 0) { + const conditions: string[] = [] + + goal.metadataFilters.forEach((filter, index) => { + const keyParam = `metaKey${index}` + const valueParam = `metaValue${index}` + metaParams[keyParam] = filter.key + metaParams[valueParam] = filter.value + conditions.push( + `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`, + ) + }) + + metaCondition = `AND (${conditions.join(' AND ')})` + } + + return { + eventType, + matchColumn, + matchCondition, + metaCondition, + metaParams, + goalValue, + } + } + /** * Generate time-series chart data for experiment win probabilities */ @@ -1177,39 +1212,18 @@ export class ExperimentController { let conversionsData: any[] = [] if (experiment.goal) { - const goalType = experiment.goal.type - const table = goalType === 'custom_event' ? 'customEV' : 'analytics' - const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg' - const goalValue = experiment.goal.value || '' + const { + eventType, + matchCondition, + metaCondition, + metaParams, + goalValue, + } = this.buildGoalEventConditions(experiment.goal) const conversionsDateColumnsSelect = this.getTimeBucketDateColumnsSelect( timeBucket, 'firstConversion', ) - let matchCondition = '' - if (experiment.goal.matchType === 'exact') { - matchCondition = `c.${matchColumn} = {goalValue:String}` - } else { - matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')` - } - - let metaCondition = '' - const metaParams: Record = {} - const metadataFilters = experiment.goal.metadataFilters - if (metadataFilters && metadataFilters.length > 0) { - const conditions: string[] = [] - metadataFilters.forEach((filter, index) => { - const keyParam = `metaKey${index}` - const valueParam = `metaValue${index}` - metaParams[keyParam] = filter.key - metaParams[valueParam] = filter.value - conditions.push( - `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`, - ) - }) - metaCondition = `AND (${conditions.join(' AND ')})` - } - const conversionsQuery = ` SELECT ${conversionsDateColumnsSelect}, @@ -1221,7 +1235,7 @@ export class ExperimentController { e.profileId as profileId, min(c.created) as firstConversion FROM experiment_exposures e - INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) + INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}' WHERE e.pid = {pid:FixedString(12)} AND e.experimentId = {experimentId:String} diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.ts index 657554968..ed2807bee 100644 --- a/backend/apps/cloud/src/goal/goal.controller.ts +++ b/backend/apps/cloud/src/goal/goal.controller.ts @@ -332,38 +332,38 @@ export class GoalController { }) } - private buildGoalMatchCondition( - goal: Goal, - _table: 'analytics' | 'customEV', - ): { condition: string; params: Record } { + private buildGoalMatchCondition(goal: Goal): { + condition: string + params: Record + } { const params: Record = {} if (goal.type === GoalType.CUSTOM_EVENT) { - // For custom events, match the event name if (goal.matchType === GoalMatchType.EXACT) { params.goalValue = goal.value || '' - return { condition: `ev = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `ev ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + return { condition: `event_name = {goalValue:String}`, params } } - } else { - // For pageview goals, match the page path - if (goal.matchType === GoalMatchType.EXACT) { - params.goalValue = goal.value || '' - return { condition: `pg = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } } + params.goalValue = goal.value || '' + return { + condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`, + params, + } + } + + if (goal.matchType === GoalMatchType.EXACT) { + params.goalValue = goal.value || '' + return { condition: `pg = {goalValue:String}`, params } + } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } + } + params.goalValue = goal.value || '' + return { + condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, + params, } } @@ -429,9 +429,10 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -440,9 +441,10 @@ export class GoalController { SELECT count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -450,6 +452,7 @@ export class GoalController { const queryParams = { pid: goal.project.id, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, ...matchParams, @@ -468,9 +471,10 @@ export class GoalController { // Get total unique sessions for conversion rate const totalSessionsQuery = ` SELECT uniqExact(psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -649,7 +653,8 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const timeBucketFunc = Object.prototype.hasOwnProperty.call( timeBucketConversion, resolvedTimeBucket, @@ -659,7 +664,7 @@ export class GoalController { const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket) const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -671,9 +676,10 @@ export class GoalController { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -684,6 +690,7 @@ export class GoalController { const queryParams = { pid: goal.project.id, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, timezone: safeTimezone, diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.ts index 5e54aab17..13711340a 100644 --- a/backend/apps/cloud/src/project/project.controller.ts +++ b/backend/apps/cloud/src/project/project.controller.ts @@ -745,12 +745,8 @@ export class ProjectController { } const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}', 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}', ] try { @@ -1607,12 +1603,8 @@ export class ProjectController { } const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}', + 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}', 'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}', - 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}', ] try { diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts index 5e5c29c63..f974e49c8 100644 --- a/backend/apps/cloud/src/project/project.service.ts +++ b/backend/apps/cloud/src/project/project.service.ts @@ -489,12 +489,8 @@ export class ProjectService { const projectIds = _map(projects, 'id') const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})', 'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE captcha DELETE WHERE pid IN ({pids:Array(FixedString(12))})', ] await Promise.all( @@ -654,7 +650,7 @@ export class ProjectService { ) } - let query = `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)} AND (` + let query = `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND (` const params = { pid, @@ -684,10 +680,7 @@ export class ProjectService { to: string, ): Promise { const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', + "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}", 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', ] @@ -820,52 +813,37 @@ export class ProjectService { monthEnd, } - const selector = ` + const query = ` + SELECT + countIf(type = 'pageview') AS pageviews, + countIf(type = 'custom_event') AS customEvents, + countIf(type = 'captcha') AS captcha, + countIf(type = 'error') AS errors + FROM events WHERE created BETWEEN {monthStart:String} AND {monthEnd:String} - AND pid IN ({pids:Array(FixedString(12))}) + AND pid IN ({pids:Array(FixedString(12))}) + AND type IN ('pageview', 'custom_event', 'captcha', 'error') ` - const countEVQuery = `SELECT count() FROM analytics ${selector}` - const countCustomEVQuery = `SELECT count() FROM customEV ${selector}` - const countCaptchaQuery = `SELECT count() FROM captcha ${selector}` - const countErrorsQuery = `SELECT count() FROM errors ${selector}` - - const [ - { data: pageviews }, - { data: customEvents }, - { data: captcha }, - { data: errors }, - ] = await Promise.all([ - clickhouse - .query({ - query: countEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCustomEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCaptchaQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countErrorsQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - ]) + const { data: counts } = await clickhouse + .query({ + query, + query_params: params, + }) + .then((resultSet) => + resultSet.json<{ + pageviews: string | number + customEvents: string | number + captcha: string | number + errors: string | number + }>(), + ) - totalPageviews += pageviews[0]['count()'] - totalCustomEvents += customEvents[0]['count()'] - totalCaptcha += captcha[0]['count()'] - totalErrors += errors[0]['count()'] + const row = counts[0] + totalPageviews += Number(row?.pageviews) || 0 + totalCustomEvents += Number(row?.customEvents) || 0 + totalCaptcha += Number(row?.captcha) || 0 + totalErrors += Number(row?.errors) || 0 } count = totalPageviews + totalCustomEvents + totalCaptcha + totalErrors @@ -916,54 +894,43 @@ export class ProjectService { // Process PIDs in chunks for (let i = 0; i < pids.length; i += CHUNK_SIZE) { const pidChunk = pids.slice(i, i + CHUNK_SIZE) - const params = { pids: pidChunk } - - const selector = ` - WHERE pid IN ({pids:Array(FixedString(12))}) - AND created BETWEEN '${monthStart}' AND '${monthEnd}' - ` - - const countEVQuery = `SELECT count() FROM analytics ${selector}` - const countCustomEVQuery = `SELECT count() FROM customEV ${selector}` - const countCaptchaQuery = `SELECT count() FROM captcha ${selector}` - const countErrorsQuery = `SELECT count() FROM errors ${selector}` - - const [ - { data: rawTraffic }, - { data: rawCustomEvents }, - { data: rawCaptcha }, - { data: rawErrors }, - ] = await Promise.all([ - clickhouse - .query({ - query: countEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCustomEVQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countCaptchaQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - clickhouse - .query({ - query: countErrorsQuery, - query_params: params, - }) - .then((resultSet) => resultSet.json()), - ]) + const params = { + pids: pidChunk, + monthStart, + monthEnd, + } - totalTraffic += rawTraffic[0]['count()'] - totalCustomEvents += rawCustomEvents[0]['count()'] - totalCaptcha += rawCaptcha[0]['count()'] - totalErrors += rawErrors[0]['count()'] + const query = ` + SELECT + countIf(type = 'pageview') AS traffic, + countIf(type = 'custom_event') AS customEvents, + countIf(type = 'captcha') AS captcha, + countIf(type = 'error') AS errors + FROM events + WHERE pid IN ({pids:Array(FixedString(12))}) + AND created BETWEEN {monthStart:String} AND {monthEnd:String} + AND type IN ('pageview', 'custom_event', 'captcha', 'error') + ` + + const { data: counts } = await clickhouse + .query({ + query, + query_params: params, + }) + .then((resultSet) => + resultSet.json<{ + traffic: string | number + customEvents: string | number + captcha: string | number + errors: string | number + }>(), + ) + + const row = counts[0] + totalTraffic += Number(row?.traffic) || 0 + totalCustomEvents += Number(row?.customEvents) || 0 + totalCaptcha += Number(row?.captcha) || 0 + totalErrors += Number(row?.errors) || 0 } const total = @@ -1043,36 +1010,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM analytics - WHERE pid IN (${pids}) - ) - OR EXISTS ( - SELECT 1 - FROM customEV - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM - ( - SELECT pid - FROM analytics - WHERE pid IN (${pids}) - UNION ALL - SELECT pid - FROM customEV - WHERE pid IN (${pids}) - ) AS t - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type IN ('pageview', 'custom_event') ` const { data } = await clickhouse @@ -1105,23 +1046,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM errors - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM errors - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'error' ` const { data } = await clickhouse @@ -1154,23 +1082,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM captcha - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM captcha - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'captcha' ` const { data } = await clickhouse diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts index 83b53f2e3..e4b4cb4a7 100644 --- a/backend/apps/cloud/src/task-manager/task-manager.service.ts +++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts @@ -143,13 +143,15 @@ const mapLimit = async ( return results } -// TODO: Count for other tables like captcha, errors, customEV too const generatePlanUsageQueryForUser = (): string => { // NOTE: keep all values parameterized to avoid injection and formatting issues. + // Counts every billable event kind (pageview, custom_event, error, captcha) + // in a single scan over the unified events table. return ` SELECT {uid:String} AS id, count(*) AS "count" - FROM analytics + FROM events WHERE pid IN ({pids:Array(FixedString(12))}) + AND type IN ('pageview', 'custom_event', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String} ` } @@ -385,27 +387,53 @@ export class TaskManagerService { ): { condition: string; params: Record } { const goalValue = (goal.value ?? '').toString() - // If goal value is blank, never match anything (avoid LIKE '%%') - if (goalValue.trim() === '') { - return { condition: '1=0', params: {} } - } - const params: Record = {} - const column = goal.type === GoalType.CUSTOM_EVENT ? 'ev' : 'pg' + const column = goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg' + const appendMetadataFilters = (condition: string) => { + if (!goal.metadataFilters || goal.metadataFilters.length === 0) { + return { condition, params } + } + + const metaConditions: string[] = [] + + goal.metadataFilters.forEach((filter, index) => { + const keyParam = `${paramKey}_metaKey${index}` + const valueParam = `${paramKey}_metaValue${index}` + params[keyParam] = filter.key + params[valueParam] = filter.value + metaConditions.push( + `has(meta.key, {${keyParam}:String}) AND meta.value[indexOf(meta.key, {${keyParam}:String})] = {${valueParam}:String}`, + ) + }) + + return { + condition: `(${condition}) AND (${metaConditions.join(' AND ')})`, + params, + } + } if (goal.matchType === GoalMatchType.EXACT) { params[paramKey] = goalValue - return { condition: `${column} = {${paramKey}:String}`, params } + return appendMetadataFilters(`${column} = {${paramKey}:String}`) } if (goal.matchType === GoalMatchType.CONTAINS) { + // Avoid wildcard goals matching every row via LIKE '%%'. + if (goalValue.trim() === '') { + return { condition: '1=0', params: {} } + } + params[paramKey] = `%${goalValue}%` - return { condition: `${column} LIKE {${paramKey}:String}`, params } + return appendMetadataFilters(`${column} ILIKE {${paramKey}:String}`) } // Regex goal + if (!goalValue || goalValue === '') { + return appendMetadataFilters('false') + } + params[paramKey] = goalValue - return { condition: `match(${column}, {${paramKey}:String})`, params } + return appendMetadataFilters(`match(${column}, {${paramKey}:String})`) } /** @@ -427,7 +455,7 @@ export class TaskManagerService { const total = Number(totalSessions) || 0 const buildUnionQuery = ( - table: 'analytics' | 'customEV', + eventType: 'pageview' | 'custom_event', goals: Goal[], ): { query: string; params: Record } | null => { if (_isEmpty(goals)) { @@ -459,9 +487,10 @@ export class TaskManagerService { {${goalIdKey}:String} as goalId, count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${eventType}' AND ${condition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} `) @@ -488,8 +517,8 @@ export class TaskManagerService { (g) => g.type === GoalType.CUSTOM_EVENT, ) - const analyticsQuery = buildUnionQuery('analytics', analyticsGoals) - const customEventQuery = buildUnionQuery('customEV', customEventGoals) + const analyticsQuery = buildUnionQuery('pageview', analyticsGoals) + const customEventQuery = buildUnionQuery('custom_event', customEventGoals) const [analyticsRes, customEventRes] = await Promise.all([ analyticsQuery @@ -573,24 +602,10 @@ export class TaskManagerService { for (let i = 0; i < pids.length; i += CHUNK_SIZE) { const pidChunk = pids.slice(i, i + CHUNK_SIZE) const query = ` - SELECT sum(cnt) AS totalEvents - FROM ( - SELECT count() AS cnt - FROM analytics - WHERE pid IN ({pids:Array(FixedString(12))}) - UNION ALL - SELECT count() AS cnt - FROM customEV - WHERE pid IN ({pids:Array(FixedString(12))}) - UNION ALL - SELECT count() AS cnt - FROM captcha - WHERE pid IN ({pids:Array(FixedString(12))}) - UNION ALL - SELECT count() AS cnt - FROM errors - WHERE pid IN ({pids:Array(FixedString(12))}) - ) + SELECT count() AS totalEvents + FROM events + WHERE pid IN ({pids:Array(FixedString(12))}) + AND type IN ('pageview', 'custom_event', 'error', 'captcha', 'performance') ` const { data } = await clickhouse @@ -1717,8 +1732,8 @@ export class TaskManagerService { count() FROM ( SELECT eid, min(created) as first_seen - FROM errors - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' GROUP BY eid ) WHERE first_seen >= now() - ${subtractSecondsTimeframe} @@ -1727,9 +1742,10 @@ export class TaskManagerService { query = ` SELECT count() - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created >= now() - ${subtractSecondsTimeframe} ` } @@ -1737,10 +1753,11 @@ export class TaskManagerService { query = ` SELECT count() - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} - AND ev = {ev:String} + AND type = 'custom_event' + AND event_name = {ev:String} AND created >= now() - ${subtractSecondsTimeframe} ` queryParams.ev = alert.queryCustomEvent @@ -1749,9 +1766,10 @@ export class TaskManagerService { query = ` SELECT count(${isUnique ? 'DISTINCT psid' : '*'}) - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created >= now() - ${subtractSecondsTimeframe} ` } @@ -1798,14 +1816,14 @@ export class TaskManagerService { if (alert.alertOnNewErrorsOnly) { detailQuery = ` - SELECT eid, name, message, lineno, colno, filename - FROM errors - WHERE pid = {pid:FixedString(12)} AND eid IN ( + SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN ( SELECT eid FROM ( SELECT eid, min(created) AS first_seen_for_eid - FROM errors - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' GROUP BY eid ) WHERE first_seen_for_eid >= now() - ${subtractSecondsTimeframe} @@ -1815,10 +1833,11 @@ export class TaskManagerService { ` } else { detailQuery = ` - SELECT eid, name, message, lineno, colno, filename - FROM errors + SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created >= now() - ${subtractSecondsTimeframe} ORDER BY created DESC LIMIT 1 @@ -1988,9 +2007,12 @@ export class TaskManagerService { }, select: ['id'], }) - const now = dayjs.utc().format('YYYY-MM-DD') + const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss') // a bit more than 2 months ago - const nineWeeksAgo = dayjs.utc().subtract(9, 'w').format('YYYY-MM-DD') + const nineWeeksAgo = dayjs + .utc() + .subtract(9, 'w') + .format('YYYY-MM-DD HH:mm:ss') await mapLimit(users, REPORTS_USERS_CONCURRENCY, async (user) => { const { id } = user @@ -2002,61 +2024,10 @@ export class TaskManagerService { return } - // No need to check for performance activity because it's not tracked without tracking analytics - const queryAnalytics = `SELECT count() FROM analytics WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` - const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}` + const queryEvents = `SELECT 1 FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error', 'performance') AND created BETWEEN {nineWeeksAgo:String} AND {now:String} LIMIT 1` // Process project IDs in chunks to avoid ClickHouse field value limit - let totalAnalytics = 0 - let totalCaptcha = 0 - let totalCustomEvents = 0 - - for (let i = 0; i < pids.length; i += CHUNK_SIZE) { - const pidChunk = pids.slice(i, i + CHUNK_SIZE) - const queryParams = { - pids: pidChunk, - nineWeeksAgo, - now, - } - - const { data: analyticsResult } = await clickhouse - .query({ - query: queryAnalytics, - query_params: queryParams, - }) - .then((resultSet) => resultSet.json<{ 'count()': number }>()) - - totalAnalytics += analyticsResult[0]['count()'] - - // Early return if we found activity - if (totalAnalytics > 0) { - return - } - } - - for (let i = 0; i < pids.length; i += CHUNK_SIZE) { - const pidChunk = pids.slice(i, i + CHUNK_SIZE) - const queryParams = { - pids: pidChunk, - nineWeeksAgo, - now, - } - - const { data: captchaResult } = await clickhouse - .query({ - query: queryCaptcha, - query_params: queryParams, - }) - .then((resultSet) => resultSet.json<{ 'count()': number }>()) - - totalCaptcha += captchaResult[0]['count()'] - - // Early return if we found activity - if (totalCaptcha > 0) { - return - } - } + let totalEvents = 0 for (let i = 0; i < pids.length; i += CHUNK_SIZE) { const pidChunk = pids.slice(i, i + CHUNK_SIZE) @@ -2066,17 +2037,17 @@ export class TaskManagerService { now, } - const { data: customEventsResult } = await clickhouse + const { data: eventsResult } = await clickhouse .query({ - query: queryCustomEvents, + query: queryEvents, query_params: queryParams, }) - .then((resultSet) => resultSet.json<{ 'count()': number }>()) + .then((resultSet) => resultSet.json>()) - totalCustomEvents += customEventsResult[0]['count()'] + totalEvents += eventsResult.length // Early return if we found activity - if (totalCustomEvents > 0) { + if (totalEvents > 0) { return } } diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts index c6313fad7..1d5b37746 100644 --- a/backend/apps/community/src/analytics/analytics.controller.ts +++ b/backend/apps/community/src/analytics/analytics.controller.ts @@ -73,12 +73,7 @@ import { GetErrorOverviewOptions, } from './dto/get-error-overview.dto' import { PatchStatusDto } from './dto/patch-status.dto' -import { - customEventTransformer, - errorEventTransformer, - performanceTransformer, - trafficTransformer, -} from './utils/transformers' +import { eventTransformer } from './utils/transformers' import { enrichTrafficSource } from './utils/clickIdSources' import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto' import { GetOverallStatsDto } from './dto/get-overall-stats.dto' @@ -250,7 +245,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - isCaptcha ? 'captcha' : 'analytics', + isCaptcha ? 'captcha' : 'pageview', ) diff = res.diff @@ -278,13 +273,11 @@ export class AnalyticsController { diff, ) - let subQuery = `FROM ${ - isCaptcha ? 'captcha' : 'analytics' - } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - - if (customEVFilterApplied && !isCaptcha) { - subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` - } + const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery( + filtersQuery, + customEVFilterApplied, + isCaptcha, + ) const paramsData = { params: { @@ -413,12 +406,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -526,12 +522,15 @@ export class AnalyticsController { let diff if (period === 'all') { - const [analyticsRes, customEVRes] = await Promise.all([ - this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'), - this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'), + const [pageviewRes, customEventRes] = await Promise.all([ + this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'), + this.analyticsService.calculateTimeBucketForAllTime( + pid, + 'custom_event', + ), ]) - diff = Math.max(analyticsRes.diff, customEVRes.diff) + diff = Math.max(pageviewRes.diff, customEventRes.diff) } const safeTimezone = this.analyticsService.getSafeTimezone(timezone) @@ -732,7 +731,7 @@ export class AnalyticsController { diff, ) - const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } } @@ -836,7 +835,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) diff = res.diff @@ -1026,32 +1025,12 @@ export class AnalyticsController { any(os) AS os, any(cc) AS cc, toString(psid) AS psid - FROM - ( - SELECT - psid, - dv, - br, - os, - cc - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT - psid, - dv, - br, - os, - cc - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND created >= {since:DateTime} + AND psid IS NOT NULL GROUP BY psid ` @@ -1151,7 +1130,8 @@ export class AnalyticsController { enrichTrafficSource(eventsDTO) - const transformed = customEventTransformer({ + const transformed = eventTransformer({ + type: 'custom_event', psid, profileId, pid: eventsDTO.pid, @@ -1183,7 +1163,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'customEV', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1321,7 +1301,8 @@ export class AnalyticsController { enrichTrafficSource(logDTO) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1364,7 +1345,8 @@ export class AnalyticsController { ttfb, } = logDTO.perf - perfTransformed = performanceTransformer({ + perfTransformed = eventTransformer({ + type: 'performance', pid: logDTO.pid, host: this.analyticsService.getHostFromOrigin(headers.origin), pg: logDTO.pg, @@ -1392,7 +1374,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1400,7 +1382,7 @@ export class AnalyticsController { if (!_isEmpty(perfTransformed)) { await clickhouse.insert({ - table: 'performance', + table: 'events', format: 'JSONEachRow', values: [perfTransformed], clickhouse_settings: { async_insert: 1 }, @@ -1489,7 +1471,8 @@ export class AnalyticsController { const { deviceType, browserName, browserVersion, osName, osVersion } = await this.analyticsService.getRequestInformation(headers) - const transformed = trafficTransformer({ + const transformed = eventTransformer({ + type: 'pageview', psid, profileId, pid: logDTO.pid, @@ -1520,7 +1503,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'analytics', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1571,7 +1554,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -1675,7 +1658,7 @@ export class AnalyticsController { if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'customEV', + 'custom_event', ) newTimeBucket = _includes(res.timeBucket, timeBucket) @@ -1869,7 +1852,8 @@ export class AnalyticsController { const { name, message, lineno, colno, filename, stackTrace, meta } = errorDTO - const transformed = errorEventTransformer({ + const transformed = eventTransformer({ + type: 'error', psid, profileId, eid: this.analyticsService.getErrorID(errorDTO), @@ -1901,7 +1885,7 @@ export class AnalyticsController { try { await clickhouse.insert({ - table: 'errors', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { async_insert: 1 }, @@ -1986,7 +1970,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) timeBucket = res.timeBucket[0] @@ -2063,7 +2047,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2139,7 +2123,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2151,6 +2135,8 @@ export class AnalyticsController { DataType.ERRORS, true, ) + const [sessionFiltersQuery, sessionFiltersParams] = + this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo( @@ -2180,6 +2166,8 @@ export class AnalyticsController { groupToUTC, newTimeBucket, parsedOptions.showResolved || false, + sessionFiltersQuery, + sessionFiltersParams, ) } @@ -2218,7 +2206,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'errors', + 'error', ) newTimeBucket = res.timeBucket[0] @@ -2291,7 +2279,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] @@ -2368,7 +2356,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - 'analytics', + 'pageview', ) timeBucket = res.timeBucket[0] diff = res.diff @@ -2454,7 +2442,7 @@ export class AnalyticsController { if (period === 'all') { const res = await this.analyticsService.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.analyticsService.getAnalyticsEventType(customEVFilterApplied), ) timeBucket = res.timeBucket[0] diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts index f63a48d82..4215d6bce 100644 --- a/backend/apps/community/src/analytics/analytics.service.ts +++ b/backend/apps/community/src/analytics/analytics.service.ts @@ -270,6 +270,14 @@ const EXCLUDE_NULL_FOR = [ 'ctp', ] +const ERROR_COLUMN_MAP: Record = { + name: 'error_name', + message: 'error_message', + filename: 'error_filename', +} + +const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type + const generateParamsQuery = ( col: string, subQuery: string, @@ -277,7 +285,8 @@ const generateParamsQuery = ( type: 'traffic' | 'performance' | 'errors' | 'captcha', measure?: PerfMeasure, ): string => { - let columns = [`${col} as name`] + const sqlCol = type === 'errors' ? mapErrorColumn(col) : col + let columns = [`${sqlCol} as name`] // For regions and cities we'll return an array of objects, that will also include the country code and region code // We need the conutry code to display the flag next to the region/city name @@ -301,15 +310,15 @@ const generateParamsQuery = ( const fn = MEASURES_MAP[processedMeasure] - if (col === 'pg' || col === 'host') { + if (sqlCol === 'pg' || sqlCol === 'host') { return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}` } - return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'errors') { - return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` + return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}` } if (type === 'captcha') { @@ -330,6 +339,13 @@ export enum DataType { CAPTCHA = 'captcha', } +type AnalyticsEventType = 'pageview' | 'custom_event' +type EventsAllTimeType = + | AnalyticsEventType + | 'performance' + | 'error' + | 'captcha' + const isValidOrigin = (origins: string[], origin: string) => { const escapeRegex = (str: string) => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') @@ -665,7 +681,9 @@ export class AnalyticsService { ) } - const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` + const sqlCol = mapErrorColumn(type) + + const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}` const { data } = await clickhouse .query({ @@ -684,13 +702,13 @@ export class AnalyticsService { type: 'traffic' | 'errors', column: 'br' | 'os', ): Promise> { - const safeTable = type === 'errors' ? 'errors' : 'analytics' + const safeType = type === 'errors' ? 'error' : 'pageview' const safeVersionCol = column === 'br' ? 'brv' : 'osv' const query = ` - SELECT ${column}, ${safeVersionCol} - FROM ${safeTable} - WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL + SELECT ${column}, ${safeVersionCol} + FROM events + WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL GROUP BY ${column}, ${safeVersionCol} ` @@ -723,6 +741,22 @@ export class AnalyticsService { return CAPTCHA_COLUMNS } + getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType { + return customEVFilterApplied ? 'custom_event' : 'pageview' + } + + buildAnalyticsEventsSubQuery( + filtersQuery: string, + customEVFilterApplied: boolean, + isCaptcha = false, + ): string { + const eventType = isCaptcha + ? 'captcha' + : this.getAnalyticsEventType(customEVFilterApplied) + + return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}` + } + getGroupFromTo( from: string, to: string, @@ -871,9 +905,10 @@ export class AnalyticsService { pg, created, lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND pg IS NOT NULL ${filtersQuery} @@ -986,15 +1021,17 @@ export class AnalyticsService { async calculateTimeBucketForAllTime( pid: string, - table: 'analytics' | 'customEV' | 'performance' | 'errors' | 'captcha', + eventTypes: EventsAllTimeType | readonly EventsAllTimeType[], ): Promise<{ timeBucket: TimeBucketType[] diff: number }> { + const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes] + const { data: fromData } = await clickhouse .query({ - query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`, - query_params: { pid }, + query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`, + query_params: { pid, types }, }) .then((res) => res.json<{ firstCreated?: string }>()) @@ -1187,7 +1224,8 @@ export class AnalyticsService { } const { filter, isExclusive, isContains } = converted[column][f] - let sqlColumn = column + let sqlColumn = + dataType === DataType.ERRORS ? mapErrorColumn(column) : column let isArrayDataset = false const param = `qf_${col}_${f}` @@ -1228,8 +1266,8 @@ export class AnalyticsService { ? 'argMin(pg, created)' : 'argMax(pg, created)' const subQueryForPages = isContains - ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` - : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}` + ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')` + : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page = {${param}:String}` // For exclusive filter (isNot) we exclude sessions with matching entry/exit page query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})` @@ -1270,6 +1308,9 @@ export class AnalyticsService { ) { sqlColumn = 'meta.value' isArrayDataset = true + } else if (column === 'ev') { + // Public filter contract `ev` maps to the renamed `event_name` column + sqlColumn = 'event_name' } const isNullFilter = @@ -1586,21 +1627,11 @@ export class AnalyticsService { FROM ( SELECT psid, - pg AS value, + if(type = 'pageview', pg, event_name) AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - - UNION ALL - - SELECT - psid, - ev AS value, - created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -1648,14 +1679,14 @@ export class AnalyticsService { psid, CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1670,17 +1701,11 @@ export class AnalyticsService { psid, argMin(cc, created) AS cc, argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source - FROM ( - SELECT psid, cc, ref, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, cc, ref, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 + AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid ) SELECT step, type, val, cnt FROM ( @@ -1758,14 +1783,14 @@ export class AnalyticsService { psid, windowFunnel(86400)(created, ${pagesStr}) AS level FROM ( - SELECT psid, pg AS value, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid != 0 - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT psid, ev AS value, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid != 0 + SELECT + psid, + if(type = 'pageview', pg, event_name) AS value, + created + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid != 0 AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) GROUP BY psid @@ -1781,19 +1806,12 @@ export class AnalyticsService { any(br) AS br, min(toTimeZone(created, {timezone:String})) AS sessionStart, max(toTimeZone(created, {timezone:String})) AS lastActivity - FROM ( - SELECT psid, pid, cc, os, br, created - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND psid IN (SELECT psid FROM funnel_qualified) - UNION ALL - SELECT psid, pid, cc, os, br, created - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) - ) GROUP BY psidCasted, pid ), pageview_counts AS ( @@ -1801,8 +1819,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1812,8 +1830,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1823,8 +1841,8 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND psid IN (SELECT psid FROM funnel_qualified) GROUP BY psidCasted, pid @@ -1860,7 +1878,7 @@ export class AnalyticsService { COALESCE(errc.count, 0) AS errors, dsf.sessionStart, dsf.lastActivity, - if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, + if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, sda.avg_duration AS sdur, sda.profileId AS profileId, if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified, @@ -1902,8 +1920,9 @@ export class AnalyticsService { const query = ` SELECT count() as c - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` const { data } = await clickhouse @@ -1954,15 +1973,18 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - let queryAll = ` + const allTimeType = this.getAnalyticsEventType(customEVFilterApplied) + + const queryAll = ` WITH analytics_counts AS ( SELECT count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' ${filtersQuery} ), duration_avg AS ( @@ -1973,6 +1995,14 @@ export class AnalyticsService { dateDiff('second', min(firstSeen), max(lastSeen)) as duration FROM sessions WHERE pid = {pid:FixedString(12)} + AND psid IN ( + SELECT DISTINCT psid + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = '${allTimeType}' + AND psid IS NOT NULL + ${filtersQuery} + ) GROUP BY psid ) ) @@ -1982,36 +2012,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryAll = ` - WITH analytics_counts AS ( - SELECT - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const { data } = await clickhouse .query({ query: queryAll, @@ -2055,7 +2055,7 @@ export class AnalyticsService { const { diff: allDiff, timeBucket: allowedBuckets } = await this.calculateTimeBucketForAllTime( pid, - customEVFilterApplied ? 'customEV' : 'analytics', + this.getAnalyticsEventType(customEVFilterApplied), ) const allTimeChartBucket = _includes( allowedBuckets, @@ -2096,16 +2096,19 @@ export class AnalyticsService { ) .format('YYYY-MM-DD HH:mm:ss') - let queryCurrent = ` + const periodType = this.getAnalyticsEventType(customEVFilterApplied) + + const queryCurrent = ` WITH analytics_counts AS ( SELECT 1 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} ), @@ -2119,8 +2122,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} ${filtersQuery} @@ -2134,16 +2138,17 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - let queryPrevious = ` + const queryPrevious = ` WITH analytics_counts AS ( SELECT 2 AS sortOrder, count(*) AS all, count(DISTINCT psid) AS unique, count(DISTINCT profileId) AS users - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} ), @@ -2157,8 +2162,9 @@ export class AnalyticsService { WHERE pid = {pid:FixedString(12)} AND psid IN ( SELECT DISTINCT psid - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = '${periodType}' AND psid IS NOT NULL AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} ${filtersQuery} @@ -2172,83 +2178,6 @@ export class AnalyticsService { FROM analytics_counts, duration_avg ` - if (customEVFilterApplied) { - queryCurrent = ` - WITH analytics_counts AS ( - SELECT - 1 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - queryPrevious = ` - WITH analytics_counts AS ( - SELECT - 2 AS sortOrder, - count(*) AS all, - count(DISTINCT psid) AS unique, - count(DISTINCT profileId) AS users - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ), - duration_avg AS ( - SELECT avgOrNull(duration) as sdur - FROM ( - SELECT - psid, - dateDiff('second', min(firstSeen), max(lastSeen)) as duration - FROM sessions - WHERE pid = {pid:FixedString(12)} - AND psid IN ( - SELECT DISTINCT psid - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND psid IS NOT NULL - AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String} - ${filtersQuery} - ) - GROUP BY psid - ) - ) - SELECT - analytics_counts.*, - duration_avg.sdur - FROM analytics_counts, duration_avg - ` - } - const query = `${queryCurrent} UNION ALL ${queryPrevious}` let { data } = await clickhouse @@ -2343,6 +2272,63 @@ export class AnalyticsService { return result } + private generateAnalyticsAggregationQueryForScope( + timeBucket: TimeBucketType, + filtersQuery: string, + mode: ChartRenderMode, + customEVFilterApplied: boolean, + ): string { + return customEVFilterApplied + ? this.generateCustomEventsAggregationQuery( + timeBucket, + filtersQuery, + mode, + ) + : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode) + } + + private extractAnalyticsChartDataForScope( + result: Array, + xShifted: string[], + customEVFilterApplied: boolean, + mode: ChartRenderMode, + ): IExtractChartData { + if (customEVFilterApplied) { + const customEventsData = this.extractCustomEventsChartData( + result, + xShifted, + ) + const uniques = + customEventsData._uniques || + customEventsData._unknown_event || + Array(_size(xShifted)).fill(0) + const visits = customEventsData._count || Array(_size(xShifted)).fill(0) + const sdur = customEventsData._sdur || Array(_size(xShifted)).fill(0) + + return { + visits, + uniques, + sdur, + } + } + + const chartData = this.extractChartData(result, xShifted) + + // Propagate the previous cumulative value forward for empty buckets + if (mode === ChartRenderMode.CUMULATIVE) { + for (let i = 1; i < chartData.visits.length; ++i) { + if (chartData.visits[i] === 0) { + chartData.visits[i] = chartData.visits[i - 1] + } + if (chartData.uniques[i] === 0) { + chartData.uniques[i] = chartData.uniques[i - 1] + } + } + } + + return chartData + } + /** * Get simplified chart data for dashboard cards * Returns only x (dates) and visits (pageviews) for a lightweight chart @@ -2373,30 +2359,11 @@ export class AnalyticsService { }, } - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - ChartRenderMode.PERIODICAL, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const visits = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - return { x: xShifted, visits } - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, ChartRenderMode.PERIODICAL, + customEVFilterApplied, ) const { data } = await clickhouse @@ -2404,9 +2371,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) + .then((resultSet) => + resultSet.json(), + ) - const { visits } = this.extractChartData(data, xShifted) + const { visits } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + ChartRenderMode.PERIODICAL, + ) return { x: xShifted, visits } } @@ -2456,7 +2430,7 @@ export class AnalyticsService { const promises = pids.map(async (pid) => { try { if (period === 'all') { - const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}` + const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}` const { data } = await clickhouse .query({ query: queryAll, @@ -2518,8 +2492,8 @@ export class AnalyticsService { .format('YYYY-MM-DD HH:mm:ss') } - const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` - const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` + const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}` + const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}` const query = `${queryCurrent} UNION ALL ${queryPrevious}` @@ -2616,8 +2590,8 @@ export class AnalyticsService { const query = ` WITH session_pages AS ( SELECT psid, ${selector} as page - FROM analytics - WHERE pid = {pid:FixedString(12)} + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid ) SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page @@ -2633,11 +2607,10 @@ export class AnalyticsService { return _map(data, 'page') } - let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}` + let query = `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND ${type} IS NOT NULL GROUP BY ${type}` if (type === 'ev') { - query = - 'SELECT ev FROM customEV WHERE pid={pid:FixedString(12)} AND ev IS NOT NULL GROUP BY ev' + query = `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name` } const { data } = await clickhouse @@ -2695,7 +2668,7 @@ export class AnalyticsService { const query = ` WITH ${withClauses.join(',\n')} - SELECT + SELECT column_name, name, count, @@ -2716,7 +2689,7 @@ export class AnalyticsService { } return ` - SELECT + SELECT '${col}' as column_name, name, count, @@ -2914,9 +2887,15 @@ export class AnalyticsService { extractCustomEventsChartData(queryResult, x: string[]): any { const result = {} + const uniques = Array(x.length).fill(0) + const sdur = Array(x.length).fill(0) + let hasUniques = false + let hasSdur = false for (let row = 0; row < _size(queryResult); ++row) { - const { ev = '_unknown_event' } = queryResult[row] + const rowEventName = queryResult[row].ev + const ev = + typeof rowEventName === 'undefined' ? '_unknown_event' : rowEventName const dateString = this.generateDateString(queryResult[row]) @@ -2928,9 +2907,35 @@ export class AnalyticsService { } result[ev][index] = queryResult[row].count + + if (typeof rowEventName === 'undefined') { + if (!result['_count']) { + result['_count'] = Array(x.length).fill(0) + } + + result['_count'][index] = queryResult[row].count || 0 + } + + if (typeof queryResult[row].uniques !== 'undefined') { + uniques[index] = queryResult[row].uniques || 0 + hasUniques = true + } + + if (typeof queryResult[row].sdur !== 'undefined') { + sdur[index] = queryResult[row].sdur || 0 + hasSdur = true + } } } + if (hasUniques) { + result['_uniques'] = uniques + } + + if (hasSdur) { + result['_sdur'] = sdur + } + return result } @@ -3005,8 +3010,8 @@ export class AnalyticsService { pid, psid, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3054,8 +3059,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM analytics - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3082,8 +3087,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3108,8 +3113,8 @@ export class AnalyticsService { FROM ( SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors - PREWHERE pid = {pid:FixedString(12)} + FROM events + PREWHERE pid = {pid:FixedString(12)} AND type = 'error' WHERE created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3131,15 +3136,31 @@ export class AnalyticsService { const baseQuery = ` SELECT ${selector}, - count() as count + count() as count, + countDistinct(subquery.psid) as uniques, + avgOrNull(sessions_data.duration) as sdur FROM ( - SELECT *, - ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + SELECT + pid, + psid, + ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery + LEFT JOIN ( + SELECT + pid, + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) as duration + FROM sessions + WHERE pid = {pid:FixedString(12)} + GROUP BY pid, psid + ) as sessions_data + ON subquery.pid = sessions_data.pid + AND subquery.psid = sessions_data.psid GROUP BY ${groupBy} ORDER BY ${groupBy} ` @@ -3147,8 +3168,10 @@ export class AnalyticsService { if (mode === ChartRenderMode.CUMULATIVE) { return ` SELECT - *, - sum(count) OVER (ORDER BY ${groupBy}) as count + ${groupBy}, + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(uniques) OVER (ORDER BY ${groupBy}) as uniques, + sdur FROM (${baseQuery}) ` } @@ -3179,9 +3202,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3214,9 +3238,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM performance + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'performance' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -3274,9 +3299,10 @@ export class AnalyticsService { FROM ( SELECT pg, dv, br, os, lc, cc, rg, ct, profileId, ${timeBucketFunc}(created) as tz_created - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) as subquery @@ -3287,10 +3313,11 @@ export class AnalyticsService { if (mode === ChartRenderMode.CUMULATIVE) { return ` SELECT - *, - sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews, - sum(uniques) OVER (ORDER BY ${groupBy}) as uniques + ${groupBy}, + sum(count) OVER (ORDER BY ${groupBy}) as count, + sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsers FROM (${baseQuery}) + ORDER BY ${groupBy} ` } @@ -3355,7 +3382,7 @@ export class AnalyticsService { ): Promise<{ name: string; count: number }[]> { const query = ` WITH session_first_pages AS ( - SELECT + SELECT psid, argMin(pg, created) as entry_page ${subQuery} @@ -3450,39 +3477,11 @@ export class AnalyticsService { ): Promise { const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone) - if (customEVFilterApplied) { - const query = this.generateCustomEventsAggregationQuery( - timeBucket, - filtersQuery, - mode, - ) - - const { data } = await clickhouse - .query({ - query, - query_params: { ...paramsData.params, timezone: safeTimezone }, - }) - .then((resultSet) => resultSet.json()) - - const uniques = - this.extractCustomEventsChartData(data, xShifted)?._unknown_event || [] - - const sdur = Array(_size(xShifted)).fill(0) - - return Promise.resolve({ - chart: { - x: xShifted, - visits: uniques, - uniques, - sdur, - }, - }) - } - - const query = this.generateAnalyticsAggregationQuery( + const query = this.generateAnalyticsAggregationQueryForScope( timeBucket, filtersQuery, mode, + customEVFilterApplied, ) const { data } = await clickhouse @@ -3490,21 +3489,16 @@ export class AnalyticsService { query, query_params: { ...paramsData.params, timezone: safeTimezone }, }) - .then((resultSet) => resultSet.json()) - - const { visits, uniques, sdur } = this.extractChartData(data, xShifted) + .then((resultSet) => + resultSet.json(), + ) - // Propagate the previous cumulative value forward for empty buckets - if (mode === ChartRenderMode.CUMULATIVE) { - for (let i = 1; i < visits.length; ++i) { - if (visits[i] === 0) { - visits[i] = visits[i - 1] - } - if (uniques[i] === 0) { - uniques[i] = uniques[i - 1] - } - } - } + const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope( + data, + xShifted, + customEVFilterApplied, + mode, + ) return Promise.resolve({ chart: { @@ -3798,9 +3792,10 @@ export class AnalyticsService { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM captcha + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'captcha' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ) as subquery @@ -4050,7 +4045,7 @@ export class AnalyticsService { filtersQuery: string, params: any, ): Promise { - const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev` + const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name` const result = {} const { data } = await clickhouse @@ -4081,9 +4076,10 @@ export class AnalyticsService { SELECT meta.key AS key FROM - analytics + events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -4137,7 +4133,7 @@ export class AnalyticsService { ) if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'customEV') + const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event') diff = res.diff @@ -4165,12 +4161,12 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND ev = {event:String} + AND event_name = {event:String} ${filtersQuery} ) ARRAY JOIN meta.key, meta.value @@ -4229,7 +4225,7 @@ export class AnalyticsService { const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] = this.getFiltersQuery(filters, DataType.ANALYTICS) - // We cannot make a query to customEV table using analytics table properties + // Page properties are only present on pageview-scoped analytics rows. if (customEVFilterApplied) { return { result: [], @@ -4238,7 +4234,7 @@ export class AnalyticsService { } if (period === 'all') { - const res = await this.calculateTimeBucketForAllTime(pid, 'analytics') + const res = await this.calculateTimeBucketForAllTime(pid, 'pageview') diff = res.diff @@ -4266,10 +4262,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM - analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND indexOf(meta.key, {property:String}) > 0 ${filtersQuery} @@ -4319,17 +4315,11 @@ export class AnalyticsService { const query = ` SELECT uniqExact(psid) as count - FROM ( - SELECT psid FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - UNION ALL - SELECT psid FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created >= {since:DateTime} - AND psid IS NOT NULL - ) + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND created >= {since:DateTime} + AND psid IS NOT NULL ` try { @@ -4367,25 +4357,26 @@ export class AnalyticsService { const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})` const customEventsFilter = customEvents && customEvents.length > 0 - ? 'AND ev IN {customEvents:Array(String)}' + ? 'AND event_name IN {customEvents:Array(String)}' : '' const query = ` SELECT ${selector}, - ev, + event_name AS ev, count() as count FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN ${tzFromDate} AND ${tzToDate} ${filtersQuery} ${customEventsFilter} ) as subquery - GROUP BY ${groupBy}, ev + GROUP BY ${groupBy}, event_name ORDER BY ${groupBy} ` @@ -4430,16 +4421,21 @@ export class AnalyticsService { } return _map(pages, (page: IPageflow) => { - if (!page.metadata) { - return page + const { created_utc: _createdUtc, ...displayPage } = page + + if (!displayPage.metadata) { + return displayPage } return { - ...page, - metadata: _map(page.metadata, ([key, value]: [string, string]) => ({ - key, - value, - })), + ...displayPage, + metadata: _map( + displayPage.metadata, + ([key, value]: [string, string]) => ({ + key, + value, + }), + ), } }) } @@ -4452,70 +4448,69 @@ export class AnalyticsService { const queryPages = ` WITH events_with_meta AS ( SELECT - 'pageview' AS type, - pg AS value, - toTimeZone(analytics.created, {timezone:String}) AS created, - pid, - psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM analytics - WHERE - pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND CAST(analytics.psid AS String) = {psid:String} - - UNION ALL - - SELECT - 'event' AS type, - ev AS value, - toTimeZone(customEV.created, {timezone:String}) AS created, + multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type, + multiIf( + event_type = 'pageview', toString(pg), + event_type = 'custom_event', toString(event_name), + toString(error_name) + ) AS value, + created AS created_utc, + toTimeZone(created, {timezone:String}) AS created_local, pid, - psid, - arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata - FROM customEV - WHERE - pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND CAST(customEV.psid AS String) = {psid:String} - - UNION ALL - - SELECT - 'error' AS type, - errors.name AS value, - toTimeZone(errors.created, {timezone:String}) AS created, - pid, - psid, - [ - tuple('message', COALESCE(errors.message, '')), - tuple('lineno', CAST(COALESCE(errors.lineno, 0), 'String')), - tuple('colno', CAST(COALESCE(errors.colno, 0), 'String')), - tuple('filename', COALESCE(errors.filename, '')) - ] AS metadata - FROM errors - WHERE - pid = {pid:FixedString(12)} - AND errors.psid IS NOT NULL - AND CAST(errors.psid AS String) = {psid:String} + toString(psid) AS psid, + if( + event_type = 'error', + [ + tuple('message', COALESCE(error_message, '')), + tuple('lineno', toString(COALESCE(lineno, 0))), + tuple('colno', toString(COALESCE(colno, 0))), + tuple('filename', COALESCE(error_filename, '')) + ], + arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value)) + ) AS metadata + FROM ( + SELECT + type AS event_type, + pg, + event_name, + error_name, + error_message, + lineno, + colno, + error_filename, + created, + pid, + psid, + meta.key AS meta_key, + meta.value AS meta_value + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} + ) ) SELECT type, value, - created, + created_local AS created, + created_utc, metadata FROM events_with_meta - ORDER BY created ASC + ORDER BY created_utc ASC ` const querySessionDetails = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} - AND CAST(psid, 'String') = {psid:String} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND toString(psid) = {psid:String} ORDER BY created ASC LIMIT 1; ` @@ -4581,7 +4576,7 @@ export class AnalyticsService { } if (!duration && lastSeen && Array.isArray(pages) && pages.length >= 1) { - const first = dayjs(pages[0].created) + const first = dayjs(pages[0].created_utc) const diffSeconds = dayjs(lastSeen).diff(first, 'second') if (diffSeconds > 0) { duration = diffSeconds @@ -4589,8 +4584,8 @@ export class AnalyticsService { } if (!duration && Array.isArray(pages) && pages.length >= 2) { - const first = dayjs(pages[0].created) - const last = dayjs(pages[pages.length - 1].created) + const first = dayjs(pages[0].created_utc) + const last = dayjs(pages[pages.length - 1].created_utc) const diffSeconds = last.diff(first, 'second') if (diffSeconds > 0) { duration = diffSeconds @@ -4601,11 +4596,12 @@ export class AnalyticsService { const querySessionDetailsFromCustomEV = ` SELECT dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('custom_event', 'error') AND psid IS NOT NULL - AND CAST(psid, 'String') = {psid:String} + AND toString(psid) = {psid:String} ORDER BY created ASC LIMIT 1; ` @@ -4626,12 +4622,12 @@ export class AnalyticsService { if (!_isEmpty(pages)) { const from = dayjs - .utc(pages[0].created) + .utc(pages[0].created_utc) .tz(safeTimezone) .startOf('minute') .format('YYYY-MM-DD HH:mm:ss') const to = dayjs - .utc(pages[_size(pages) - 1].created) + .utc(pages[_size(pages) - 1].created_utc) .tz(safeTimezone) .endOf('minute') .format('YYYY-MM-DD HH:mm:ss') @@ -4649,8 +4645,8 @@ export class AnalyticsService { { params: { ...paramsData.params, - groupFrom: pages[0].created, - groupTo: pages[_size(pages) - 1].created, + groupFrom: from, + groupTo: to, }, }, safeTimezone, @@ -4663,7 +4659,7 @@ export class AnalyticsService { let isLive = false if (!_isEmpty(pages)) { - const lastActivityTime = dayjs(pages[pages.length - 1].created) + const lastActivityTime = dayjs(pages[pages.length - 1].created_utc) const liveThresholdTime = dayjs().subtract( LIVE_SESSION_THRESHOLD_SECONDS, @@ -4693,61 +4689,16 @@ export class AnalyticsService { skip = 0, customEVFilterApplied = false, ): Promise { - let primaryEventsSubquery: string - - if (customEVFilterApplied) { - primaryEventsSubquery = ` - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ` - } else { - primaryEventsSubquery = ` - SELECT - CAST(analytics.psid, 'String') AS psidCasted, - analytics.pid, - analytics.cc, - analytics.os, - analytics.br, - toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping - FROM analytics - WHERE - analytics.pid = {pid:FixedString(12)} - AND analytics.psid IS NOT NULL - AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL - SELECT - CAST(customEV.psid, 'String') AS psidCasted, - customEV.pid, - customEV.cc, - customEV.os, - customEV.br, - toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping - FROM customEV - WHERE - customEV.pid = {pid:FixedString(12)} - AND customEV.psid IS NOT NULL - AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ` - } + const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH distinct_sessions_filtered AS ( SELECT psidCasted, - pid, + pid, any(cc) AS cc, any(os) AS os, any(br) AS br, @@ -4757,39 +4708,39 @@ export class AnalyticsService { GROUP BY psidCasted, pid ), pageview_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count - FROM analytics - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + pid, + count() as count + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), event_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count - FROM customEV - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + pid, + count() as count + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), error_counts AS ( - SELECT + SELECT CAST(psid, 'String') AS psidCasted, - pid, - count() as count - FROM errors - WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL + pid, + count() as count + FROM events + WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psidCasted, pid ), session_duration_agg AS ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, + SELECT + CAST(psid, 'String') AS psidCasted, + pid, dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration, argMax(profileId, lastSeen) as profileId FROM sessions @@ -4816,7 +4767,7 @@ export class AnalyticsService { COALESCE(errc.count, 0) AS errors, dsf.sessionStart, dsf.lastActivity, - if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, + if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, sda.avg_duration AS sdur, sda.profileId AS profileId, if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified, @@ -4848,6 +4799,43 @@ export class AnalyticsService { return data } + private buildSessionsListPrimaryEventsSubquery( + filtersQuery: string, + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + ` + : filtersQuery + + return ` + SELECT + CAST(psid, 'String') AS psidCasted, + pid, + cc, + os, + br, + toTimeZone(created, {timezone:String}) AS created_for_grouping + FROM events + WHERE + pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${scopedSessionFilter} + ` + } + async getErrorsList( options: string, filtersQuery: string, @@ -4878,10 +4866,18 @@ export class AnalyticsService { count(DISTINCT psid) as sessions, status.status FROM ( - SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created - FROM errors + SELECT + eid, + error_name AS name, + error_message AS message, + error_filename AS filename, + psid, + profileId, + toTimeZone(created, {timezone:String}) AS created + FROM events WHERE pid = {pid:FixedString(12)} - AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} + AND type = 'error' + AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ) AS errors LEFT JOIN ( @@ -4895,7 +4891,7 @@ export class AnalyticsService { ${ parsedOptions?.showResolved ? '' - : "WHERE status.status = 'active' OR status.status = 'regressed'" + : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL" } GROUP BY errors.eid, status.status ORDER BY last_seen DESC @@ -4940,14 +4936,15 @@ export class AnalyticsService { FROM ( SELECT eid, - name, - message, - filename, + error_name AS name, + error_message AS message, + error_filename AS filename, colno, lineno, stackTrace - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ORDER BY created DESC @@ -4970,9 +4967,10 @@ export class AnalyticsService { max(created) AS last_seen, min(created) AS first_seen, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)}; ` @@ -4985,9 +4983,10 @@ export class AnalyticsService { SELECT meta.key, meta.value - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} ) @@ -5034,8 +5033,8 @@ export class AnalyticsService { timeBucket, groupFrom, groupTo, - `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, - 'AND eid = {eid:FixedString(32)}', + `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`, + 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}', paramsData, safeTimezone, ChartRenderMode.PERIODICAL, @@ -5061,17 +5060,22 @@ export class AnalyticsService { groupTo: string, timeBucket: string, showResolved: boolean, + sessionFiltersQuery = '', + sessionFiltersParams: Record = {}, ): Promise { const resolvedFilter = showResolved ? '' : "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)" - // Get total sessions from analytics table for the time range + // Get total sessions from all matching events for the time range const queryTotalSessions = ` SELECT count(DISTINCT psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${sessionFiltersQuery} ` // Get error stats: total errors, unique errors, affected sessions, affected users @@ -5081,7 +5085,7 @@ export class AnalyticsService { count(DISTINCT eid) as uniqueErrors, count(DISTINCT psid) as affectedSessions, count(DISTINCT profileId) as affectedUsers - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -5089,6 +5093,8 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5098,12 +5104,12 @@ export class AnalyticsService { const queryMostFrequentError = ` SELECT errors.eid as eid, - any(errors.name) as name, - any(errors.message) as message, + any(errors.error_name) as name, + any(errors.error_message) as message, count(*) as count, count(DISTINCT errors.profileId) as usersAffected, max(errors.created) as lastSeen - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -5111,6 +5117,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5135,7 +5142,7 @@ export class AnalyticsService { SELECT profileId, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM errors + FROM events AS errors LEFT JOIN ( SELECT eid, argMax(status, updated) AS status FROM error_statuses @@ -5143,6 +5150,7 @@ export class AnalyticsService { GROUP BY eid ) AS status ON errors.eid = status.eid WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ${filtersQuery} ${resolvedFilter} @@ -5165,7 +5173,10 @@ export class AnalyticsService { clickhouse .query({ query: queryTotalSessions, - query_params: paramsData.params, + query_params: { + ...paramsData.params, + ...sessionFiltersParams, + }, }) .then((resultSet) => resultSet.json()) .then(({ data }) => data[0]), @@ -5254,9 +5265,11 @@ export class AnalyticsService { ): Promise<{ sessions: any[]; total: number }> { const queryCount = ` SELECT count(DISTINCT psid) as total - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND eid = {eid:FixedString(32)} + AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -5264,16 +5277,17 @@ export class AnalyticsService { SELECT DISTINCT CAST(errors.psid, 'String') as psid, any(errors.profileId) as profileId, - any(analytics.cc) as cc, - any(analytics.br) as br, - any(analytics.os) as os, + any(errors.cc) as cc, + any(errors.br) as br, + any(errors.os) as os, min(errors.created) as firstErrorAt, max(errors.created) as lastErrorAt, count(*) as errorCount - FROM errors - LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid + FROM events AS errors WHERE errors.pid = {pid:FixedString(12)} + AND errors.type = 'error' AND errors.eid = {eid:FixedString(32)} + AND errors.psid IS NOT NULL AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY errors.psid ORDER BY lastErrorAt DESC @@ -5319,7 +5333,7 @@ export class AnalyticsService { {}, ) - const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map( + const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map( params, (val, key) => `{${key}:FixedString(32)}`, ).join(', ')})` @@ -5482,12 +5496,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodFormatted:String} AND {now:String} ${kvQuery} ${filtersQuery} @@ -5500,12 +5515,13 @@ export class AnalyticsService { key, round(sum(${aggFunction}), 2) AS sum, round(avg(${aggFunction}), 2) AS avg - FROM customEV + FROM events ARRAY JOIN meta.key AS key, meta.value AS value WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND key = {metricKey:String} - AND ev = {customEventName:String} + AND event_name = {customEventName:String} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${kvQuery} ${filtersQuery} @@ -5599,75 +5615,28 @@ export class AnalyticsService { } } - async getProfilesList( - pid: string, + private buildProfilesListDataCTE( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - profileType: 'all' | 'anonymous' | 'identified' = 'all', - customEVFilterApplied = false, - ): Promise { - let profileTypeFilter = '' - if (profileType === 'anonymous') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` - } else if (profileType === 'identified') { - profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` - } - - let allProfileDataCTE: string - - if (customEVFilterApplied) { - allProfileDataCTE = ` - all_profile_data AS ( - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} + profileTypeFilter: string, + customEVFilterApplied: boolean, + ): string { + const scopedProfileFilter = customEVFilterApplied + ? ` AND profileId IN ( SELECT DISTINCT profileId - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} ${filtersQuery} ) - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} - )` - } else { - allProfileDataCTE = ` + ` + : filtersQuery + + return ` all_profile_data AS ( SELECT profileId, @@ -5677,36 +5646,43 @@ export class AnalyticsService { br, dv, created, - 1 AS isPageview, - 0 AS isEvent - FROM analytics + if(type = 'pageview', 1, 0) AS isPageview, + if(type = 'custom_event', 1, 0) AS isEvent, + if(type = 'error', 1, 0) AS isError + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') AND created BETWEEN {groupFrom:String} AND {groupTo:String} AND profileId IS NOT NULL AND profileId != '' ${profileTypeFilter} - ${filtersQuery} - UNION ALL - SELECT - profileId, - psid, - cc, - os, - br, - dv, - created, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - ${filtersQuery} + ${scopedProfileFilter} )` + } + + async getProfilesList( + pid: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + profileType: 'all' | 'anonymous' | 'identified' = 'all', + customEVFilterApplied = false, + ): Promise { + let profileTypeFilter = '' + if (profileType === 'anonymous') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'` + } else if (profileType === 'identified') { + profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'` } + const allProfileDataCTE = this.buildProfilesListDataCTE( + filtersQuery, + profileTypeFilter, + customEVFilterApplied, + ) + const query = ` WITH ${allProfileDataCTE}, profile_aggregated AS ( @@ -5715,6 +5691,7 @@ export class AnalyticsService { sum(isPageview) AS pageviewsCount, countDistinct(psid) AS sessionsCount, sum(isEvent) AS eventsCount, + sum(isError) AS errorsCount, min(created) AS firstSeen, max(created) AS lastSeen, any(cc) AS cc_agg, @@ -5723,24 +5700,13 @@ export class AnalyticsService { any(dv) AS dv_agg FROM all_profile_data GROUP BY profileId - ), - profile_errors AS ( - SELECT - profileId, - count() AS errorsCount - FROM errors - WHERE pid = {pid:FixedString(12)} - AND profileId IS NOT NULL - AND profileId != '' - ${profileTypeFilter} - GROUP BY profileId ) SELECT pa.profileId AS profileId, pa.sessionsCount AS sessionsCount, pa.pageviewsCount AS pageviewsCount, pa.eventsCount AS eventsCount, - COALESCE(perr.errorsCount, 0) AS errorsCount, + pa.errorsCount AS errorsCount, pa.firstSeen AS firstSeen, pa.lastSeen AS lastSeen, pa.cc_agg AS cc, @@ -5748,7 +5714,6 @@ export class AnalyticsService { pa.br_agg AS br, pa.dv_agg AS dv FROM profile_aggregated AS pa - LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileId ORDER BY pa.lastSeen DESC LIMIT {take:UInt32} OFFSET {skip:UInt32} @@ -5785,54 +5750,83 @@ export class AnalyticsService { AND profileId = {profileId:String} ` - // Query avg duration from analytics table (more accurate than sessions table) + // Prefer pageview timings, but fall back to sessions for event/error-only profiles. const queryAvgDuration = ` - SELECT - avg(session_duration) AS avgDuration - FROM ( + WITH pageview_avg AS ( SELECT - psid, - dateDiff('second', min(created), max(created)) AS session_duration - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - GROUP BY psid - HAVING session_duration > 0 + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(created), max(created)) AS session_duration + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' + AND profileId = {profileId:String} + AND psid IS NOT NULL + GROUP BY psid + HAVING session_duration > 0 + ) + ), + sessions_avg AS ( + SELECT + avgOrNull(session_duration) AS avgDuration + FROM ( + SELECT + psid, + dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration + FROM sessions FINAL + WHERE pid = {pid:FixedString(12)} + AND profileId = {profileId:String} + GROUP BY psid + HAVING session_duration > 0 + ) ) + SELECT + coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration + FROM pageview_avg, sessions_avg ` - // Query analytics for accurate pageview count const queryPageviews = ` SELECT count() AS pageviewsCount - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} ` - // Query customEV for accurate events count const queryEvents = ` SELECT count() AS eventsCount - FROM customEV + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND profileId = {profileId:String} + ` + + const queryErrors = ` + SELECT count() AS errorsCount + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} ` // Query for device/location details const queryDetails = ` SELECT - any(cc) AS cc, - any(rg) AS rg, - any(ct) AS ct, - any(os) AS os, - any(osv) AS osv, - any(br) AS br, - any(brv) AS brv, - any(dv) AS dv, - any(lc) AS lc - FROM analytics + argMax(cc, created) AS cc, + argMax(rg, created) AS rg, + argMax(ct, created) AS ct, + argMax(os, created) AS os, + argMax(osv, created) AS osv, + argMax(br, created) AS br, + argMax(brv, created) AS brv, + argMax(dv, created) AS dv, + argMax(lc, created) AS lc + FROM events WHERE pid = {pid:FixedString(12)} AND profileId = {profileId:String} + AND type IN ('pageview', 'custom_event', 'error') ` const params = { pid, profileId } @@ -5842,6 +5836,7 @@ export class AnalyticsService { avgDurationResult, pageviewsResult, eventsResult, + errorsResult, detailsResult, ] = await Promise.all([ clickhouse @@ -5856,6 +5851,9 @@ export class AnalyticsService { clickhouse .query({ query: queryEvents, query_params: params }) .then((resultSet) => resultSet.json()), + clickhouse + .query({ query: queryErrors, query_params: params }) + .then((resultSet) => resultSet.json()), clickhouse .query({ query: queryDetails, query_params: params }) .then((resultSet) => resultSet.json()), @@ -5868,6 +5866,7 @@ export class AnalyticsService { const avgDuration = (avgDurationResult.data[0] || {}) as Record const pageviews = (pageviewsResult.data[0] || {}) as Record const events = (eventsResult.data[0] || {}) as Record + const errors = (errorsResult.data[0] || {}) as Record const details = (detailsResult.data[0] || {}) as Record return { @@ -5876,6 +5875,7 @@ export class AnalyticsService { sessionsCount: sessionCount.sessionsCount || 0, pageviewsCount: pageviews.pageviewsCount || 0, eventsCount: events.eventsCount || 0, + errorsCount: errors.errorsCount || 0, firstSeen: sessionCount.firstSeen, lastSeen: sessionCount.lastSeen, avgDuration: avgDuration.avgDuration || 0, @@ -5892,8 +5892,9 @@ export class AnalyticsService { SELECT pg AS page, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} GROUP BY pg ORDER BY count DESC @@ -5922,30 +5923,15 @@ export class AnalyticsService { .format('YYYY-MM-DD') const query = ` - WITH all_activity AS ( - SELECT - toDate(created) AS date, - 1 AS isPageview, - 0 AS isEvent - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - UNION ALL - SELECT - toDate(created) AS date, - 0 AS isPageview, - 1 AS isEvent - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND created >= {startDate:Date} - ) SELECT - date, - sum(isPageview) AS pageviews, - sum(isEvent) AS events - FROM all_activity + toDate(created) AS date, + countIf(type = 'pageview') AS pageviews, + countIf(type = 'custom_event') AS events + FROM events + WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event') + AND profileId = {profileId:String} + AND created >= {startDate:Date} GROUP BY date ORDER BY date ASC ` @@ -5980,8 +5966,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -5992,8 +5979,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -6004,8 +5992,9 @@ export class AnalyticsService { SELECT ${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time, count() AS count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND created BETWEEN {from:String} AND {to:String} GROUP BY time @@ -6066,70 +6055,26 @@ export class AnalyticsService { } } - async getProfileSessionsList( - pid: string, - profileId: string, + private buildProfileSessionsEventsCTE( filtersQuery: string, - paramsData: any, - safeTimezone: string, - take = 30, - skip = 0, - customEVFilterApplied = false, - ): Promise { - let allProfileEventsCTE: string + customEVFilterApplied: boolean, + ): string { + const scopedSessionFilter = customEVFilterApplied + ? ` + AND CAST(psid, 'String') IN ( + SELECT DISTINCT CAST(psid, 'String') + FROM events + WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' + AND profileId = {profileId:String} + AND psid IS NOT NULL + AND created BETWEEN {groupFrom:String} AND {groupTo:String} + ${filtersQuery} + ) + ` + : filtersQuery - if (customEVFilterApplied) { - allProfileEventsCTE = ` - all_profile_events AS ( - SELECT - all_events.psidCasted, - all_events.pid, - all_events.profileId, - all_events.cc, - all_events.os, - all_events.br, - all_events.created_tz - FROM ( - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - UNION ALL - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ) AS all_events - WHERE all_events.psidCasted IN ( - SELECT DISTINCT CAST(psid, 'String') - FROM customEV - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - ) - )` - } else { - allProfileEventsCTE = ` + return ` all_profile_events AS ( SELECT CAST(psid, 'String') AS psidCasted, @@ -6139,29 +6084,30 @@ export class AnalyticsService { os, br, toTimeZone(created, {timezone:String}) AS created_tz - FROM analytics - WHERE pid = {pid:FixedString(12)} - AND profileId = {profileId:String} - AND psid IS NOT NULL - AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} - UNION ALL - SELECT - CAST(psid, 'String') AS psidCasted, - pid, - profileId, - cc, - os, - br, - toTimeZone(created, {timezone:String}) AS created_tz - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type IN ('pageview', 'custom_event', 'error') AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} - ${filtersQuery} + ${scopedSessionFilter} )` - } + } + + async getProfileSessionsList( + pid: string, + profileId: string, + filtersQuery: string, + paramsData: any, + safeTimezone: string, + take = 30, + skip = 0, + customEVFilterApplied = false, + ): Promise { + const allProfileEventsCTE = this.buildProfileSessionsEventsCTE( + filtersQuery, + customEVFilterApplied, + ) const query = ` WITH ${allProfileEventsCTE}, @@ -6183,8 +6129,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6195,8 +6142,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM customEV + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'custom_event' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6207,8 +6155,9 @@ export class AnalyticsService { CAST(psid, 'String') AS psidCasted, pid, count() as count - FROM errors + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'error' AND profileId = {profileId:String} AND psid IS NOT NULL AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -6234,7 +6183,7 @@ export class AnalyticsService { COALESCE(errc.count, 0) AS errors, ps.sessionStart, ps.lastActivity, - if(dateDiff('second', ps.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, + if(dateDiff('second', toTimeZone(ps.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive, sda.avg_duration AS sdur FROM profile_sessions ps LEFT JOIN pageview_counts pc ON ps.psidCasted = pc.psidCasted AND ps.pid = pc.pid diff --git a/backend/apps/community/src/analytics/interfaces/index.ts b/backend/apps/community/src/analytics/interfaces/index.ts index 84da70732..2dace48be 100644 --- a/backend/apps/community/src/analytics/interfaces/index.ts +++ b/backend/apps/community/src/analytics/interfaces/index.ts @@ -183,9 +183,10 @@ export interface IOverallPerformance { } export interface IPageflow { - type: 'pageview' | 'event' + type: 'pageview' | 'event' | 'error' value: string created: string + created_utc?: string metadata?: [string, string][] } diff --git a/backend/apps/community/src/analytics/utils/transformers.ts b/backend/apps/community/src/analytics/utils/transformers.ts index b420cdac8..7ec1ed491 100644 --- a/backend/apps/community/src/analytics/utils/transformers.ts +++ b/backend/apps/community/src/analytics/utils/transformers.ts @@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc' dayjs.extend(utc) const processMetaKV = ( - rawMeta: Record, + rawMeta: Record | null | undefined, ): { 'meta.key': string[] | null; 'meta.value': string[] | null } => { if (!rawMeta || _isEmpty(rawMeta)) { return { @@ -23,10 +23,10 @@ const processMetaKV = ( } } -interface TrafficTransformerOptions { - psid: string - profileId: string +interface CommonOptions { pid: string + psid?: string | null + profileId?: string | null host?: string | null pg?: string | null dv?: string | null @@ -52,262 +52,42 @@ interface TrafficTransformerOptions { meta?: Record | null } -export const trafficTransformer = ({ - psid, - profileId, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: TrafficTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } +interface PageviewOptions extends CommonOptions { + type: 'pageview' } -interface CustomEventTransformerOptions { - psid: string - profileId: string - pid: string - host?: string | null +interface CustomEventOptions extends CommonOptions { + type: 'custom_event' ev: string - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - ref?: string | null - so?: string | null - me?: string | null - ca?: string | null - te?: string | null - co?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null - meta?: Record | null -} - -export const customEventTransformer = ({ - psid, - profileId, - pid, - host, - ev, - pg, - dv, - br, - brv, - os, - osv, - lc, - ref, - so, - me, - ca, - te, - co, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - meta, -}: CustomEventTransformerOptions) => { - return { - psid, - profileId, - pid, - host: host || null, - ev, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - ref: ref || null, - so: so || null, - me: me || null, - ca: ca || null, - te: te || null, - co: co || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface ErrorEventTransformerOptions { - psid: string - profileId: string +interface ErrorEventOptions extends CommonOptions { + type: 'error' eid: string - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - os?: string | null - osv?: string | null - lc?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null name?: string | null message?: string | null lineno?: number | null colno?: number | null filename?: string | null stackTrace?: string | null - meta?: Record | null -} - -export const errorEventTransformer = ({ - psid, - profileId, - eid, - pid, - host, - pg, - dv, - br, - brv, - os, - osv, - lc, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - name, - message, - lineno, - colno, - filename, - stackTrace, - meta, -}: ErrorEventTransformerOptions) => { - return { - psid, - profileId, - eid, - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - os: os || null, - osv: osv || null, - lc: lc || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - name: name || null, - message: message || null, - lineno: lineno || null, - colno: colno || null, - filename: filename || null, - stackTrace: stackTrace || null, - ...processMetaKV(meta), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), - } } -interface PerformanceTransformerOptions { - pid: string - host?: string | null - pg?: string | null - dv?: string | null - br?: string | null - brv?: string | null - cc?: string | null - rg?: string | null - rgc?: string | null - ct?: string | null - isp?: string | null - og?: string | null - ut?: string | null - ctp?: string | null +interface PerformanceOptions extends Omit< + CommonOptions, + | 'psid' + | 'profileId' + | 'os' + | 'osv' + | 'lc' + | 'ref' + | 'so' + | 'me' + | 'ca' + | 'te' + | 'co' + | 'meta' +> { + type: 'performance' dns: number tls: number conn: number @@ -318,53 +98,131 @@ interface PerformanceTransformerOptions { ttfb: number } -export const performanceTransformer = ({ - pid, - host, - pg, - dv, - br, - brv, - cc, - rg, - rgc, - ct, - isp, - og, - ut, - ctp, - dns, - tls, - conn, - response, - render, - domLoad, - pageLoad, - ttfb, -}: PerformanceTransformerOptions) => { +interface CaptchaOptions { + type: 'captcha' + pid: string + dv?: string | null + br?: string | null + os?: string | null + cc?: string | null + timestamp?: number +} + +type EventTransformerOptions = + | PageviewOptions + | CustomEventOptions + | ErrorEventOptions + | PerformanceOptions + | CaptchaOptions + +const buildCommon = (opts: CommonOptions) => ({ + pid: opts.pid, + psid: opts.psid ?? null, + profileId: opts.profileId ?? null, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + os: opts.os || null, + osv: opts.osv || null, + lc: opts.lc || null, + ref: opts.ref || null, + so: opts.so || null, + me: opts.me || null, + ca: opts.ca || null, + te: opts.te || null, + co: opts.co || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + ...processMetaKV(opts.meta), +}) + +// Single transformer for all event kinds; the discriminator is `type`. +// Renames at the storage boundary: ev -> event_name, name -> error_name, +// message -> error_message, filename -> error_filename. DTO and API field +// names stay as the legacy short forms. +export const eventTransformer = (opts: EventTransformerOptions) => { + const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss') + + if (opts.type === 'pageview') { + return { + type: 'pageview' as const, + ...buildCommon(opts), + created, + } + } + + if (opts.type === 'custom_event') { + return { + type: 'custom_event' as const, + ...buildCommon(opts), + event_name: opts.ev, + created, + } + } + + if (opts.type === 'error') { + return { + type: 'error' as const, + ...buildCommon(opts), + eid: opts.eid, + error_name: opts.name || null, + error_message: opts.message || null, + stackTrace: opts.stackTrace || null, + lineno: opts.lineno ?? null, + colno: opts.colno ?? null, + error_filename: opts.filename || null, + created, + } + } + + if (opts.type === 'performance') { + return { + type: 'performance' as const, + pid: opts.pid, + host: opts.host || null, + pg: opts.pg || null, + dv: opts.dv || null, + br: opts.br || null, + brv: opts.brv || null, + cc: opts.cc || null, + rg: opts.rg || null, + rgc: opts.rgc || null, + ct: opts.ct || null, + isp: opts.isp || null, + og: opts.og || null, + ut: opts.ut || null, + ctp: opts.ctp || null, + dns: _round(opts.dns), + tls: _round(opts.tls), + conn: _round(opts.conn), + response: _round(opts.response), + render: _round(opts.render), + domLoad: _round(opts.domLoad), + pageLoad: _round(opts.pageLoad), + ttfb: _round(opts.ttfb), + created, + } + } + + // captcha return { - pid, - host: host || null, - pg: pg || null, - dv: dv || null, - br: br || null, - brv: brv || null, - cc: cc || null, - rg: rg || null, - rgc: rgc || null, - ct: ct || null, - isp: isp || null, - og: og || null, - ut: ut || null, - ctp: ctp || null, - dns: _round(dns), - tls: _round(tls), - conn: _round(conn), - response: _round(response), - render: _round(render), - domLoad: _round(domLoad), - pageLoad: _round(pageLoad), - ttfb: _round(ttfb), - created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'), + type: 'captcha' as const, + pid: opts.pid, + dv: opts.dv || null, + br: opts.br || null, + os: opts.os || null, + cc: opts.cc || null, + created: + typeof opts.timestamp === 'number' + ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss') + : created, } } diff --git a/backend/apps/community/src/captcha/captcha.service.ts b/backend/apps/community/src/captcha/captcha.service.ts index 921e6d7b7..99ac3516c 100644 --- a/backend/apps/community/src/captcha/captcha.service.ts +++ b/backend/apps/community/src/captcha/captcha.service.ts @@ -17,7 +17,7 @@ import { } from '../common/constants' import { getIPDetails } from '../common/utils' import { GeneratedChallenge } from './interfaces/generated-captcha' -import { captchaTransformer } from './utils/transformers' +import { eventTransformer } from '../analytics/utils/transformers' import { clickhouse } from '../common/integrations/clickhouse' dayjs.extend(utc) @@ -152,18 +152,19 @@ export class CaptchaService { const osName = ua.os.name const { country } = getIPDetails(ip) - const transformed = captchaTransformer( + const transformed = eventTransformer({ + type: 'captcha', pid, - deviceType, - browserName, - osName, - country, + dv: deviceType, + br: browserName, + os: osName, + cc: country, timestamp, - ) + }) try { await clickhouse.insert({ - table: 'captcha', + table: 'events', format: 'JSONEachRow', values: [transformed], clickhouse_settings: { diff --git a/backend/apps/community/src/captcha/utils/transformers.ts b/backend/apps/community/src/captcha/utils/transformers.ts deleted file mode 100644 index 3a0d21d43..000000000 --- a/backend/apps/community/src/captcha/utils/transformers.ts +++ /dev/null @@ -1,22 +0,0 @@ -import dayjs from 'dayjs' -import utc from 'dayjs/plugin/utc' - -dayjs.extend(utc) - -export const captchaTransformer = ( - pid: string, - dv: string | null, - br: string | null, - os: string | null, - cc: string | null, - timestamp: number, -) => { - return { - pid, - dv: dv || null, - br: br || null, - os: os || null, - cc: cc || null, - created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'), - } -} diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts index 0d0646d1b..5f3ebfe97 100644 --- a/backend/apps/community/src/data-import/data-import.processor.ts +++ b/backend/apps/community/src/data-import/data-import.processor.ts @@ -8,6 +8,7 @@ import { Job } from 'bullmq' import { DataImportService } from './data-import.service' import { DataImportStatus } from './entity/data-import.entity' import { getMapper } from './mappers' +import { ImportError } from './mappers/mapper.interface' import { clickhouse } from '../common/integrations/clickhouse' const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics' @@ -85,8 +86,7 @@ export class DataImportProcessor extends WorkerHost { let minDate: string | null = null let maxDate: string | null = null - const analyticsBatch: Record[] = [] - const customEVBatch: Record[] = [] + const eventsBatch: Record[] = [] try { for await (const row of mapper.createRowStream( @@ -102,28 +102,17 @@ export class DataImportProcessor extends WorkerHost { if (!maxDate || created > maxDate) maxDate = created } - if (row.table === 'analytics') { - analyticsBatch.push(row.data) - } else { - customEVBatch.push(row.data) - } - - if (analyticsBatch.length >= BATCH_SIZE) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - analyticsBatch.length = 0 - } + eventsBatch.push({ type: row.type, ...row.data }) - if (customEVBatch.length >= BATCH_SIZE) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length - customEVBatch.length = 0 + if (eventsBatch.length >= BATCH_SIZE) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length + eventsBatch.length = 0 } if (totalRows % 10000 === 0) { const progress = { - importedRows: - importedRows + analyticsBatch.length + customEVBatch.length, + importedRows: importedRows + eventsBatch.length, totalRows, } @@ -141,14 +130,9 @@ export class DataImportProcessor extends WorkerHost { } } - if (analyticsBatch.length > 0) { - await this.flushBatch('analytics', analyticsBatch) - importedRows += analyticsBatch.length - } - - if (customEVBatch.length > 0) { - await this.flushBatch('customEV', customEVBatch) - importedRows += customEVBatch.length + if (eventsBatch.length > 0) { + await this.flushBatch(eventsBatch) + importedRows += eventsBatch.length } await this.dataImportService.markCompleted(importId, projectId, { @@ -176,10 +160,15 @@ export class DataImportProcessor extends WorkerHost { ) } + const userMessage = + error instanceof ImportError + ? error.message + : 'An unexpected error occurred while processing the import. Please try again or contact support.' + await this.dataImportService.markFailed( importId, projectId, - error.message, + userMessage, ) } } finally { @@ -187,12 +176,9 @@ export class DataImportProcessor extends WorkerHost { } } - private async flushBatch( - table: 'analytics' | 'customEV', - batch: Record[], - ): Promise { + private async flushBatch(batch: Record[]): Promise { await clickhouse.insert({ - table: `${CLICKHOUSE_DB}.${table}`, + table: `${CLICKHOUSE_DB}.events`, values: batch, format: 'JSONEachRow', }) diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.ts index ac9be0c50..e1453fbe0 100644 --- a/backend/apps/community/src/data-import/data-import.service.ts +++ b/backend/apps/community/src/data-import/data-import.service.ts @@ -281,11 +281,7 @@ export class DataImportService { importId: number, ): Promise { await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, - query_params: { pid: projectId, importID: importId }, - }) - await clickhouse.command({ - query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`, + query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`, query_params: { pid: projectId, importID: importId }, }) } @@ -330,7 +326,7 @@ export class DataImportService { ): Promise { const result = await clickhouse .query({ - query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, + query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`, query_params: { pid: projectId, from, to }, }) .then((rs) => rs.json<{ cnt: string }>()) diff --git a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts index 55c578f7c..1ae5397ca 100644 --- a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts @@ -1,7 +1,11 @@ import * as fs from 'fs' import { parse } from 'csv-parse' -import { ImportMapper, AnalyticsImportRow } from './mapper.interface' +import { + ImportMapper, + AnalyticsImportRow, + ImportError, +} from './mapper.interface' import { normalizeNull, truncate, @@ -41,7 +45,7 @@ function detectFileType(headers: string[]): { const missing = COMMON_REQUIRED.filter((c) => !normalized.includes(c)) if (missing.length > 0) { - throw new Error( + throw new ImportError( `CSV does not appear to be a Fathom Analytics export. Missing required columns: ${missing.join(', ')}.`, ) } @@ -54,7 +58,7 @@ function detectFileType(headers: string[]): { return { type: 'events', normalized } } - throw new Error( + throw new ImportError( 'CSV does not appear to be a Fathom Analytics export. Expected a pageviews export (with "pageviews" column) or an events export (with "event_name" column).', ) } @@ -202,19 +206,19 @@ export class FathomMapper implements ImportMapper { if (fileType === 'pageviews') { for (let i = 0; i < count; i++) { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' - const evData = { ...baseData, ev: eventName } + const evData = { ...baseData, event_name: eventName } for (let i = 0; i < count; i++) { - yield { table: 'customEV', data: evData } + yield { type: 'custom_event', data: evData } } } } if (!headerChecked) { - throw new Error( + throw new ImportError( 'CSV appears empty or is missing a header row. Please upload a CSV export from Fathom Analytics.', ) } diff --git a/backend/apps/community/src/data-import/mappers/mapper.interface.ts b/backend/apps/community/src/data-import/mappers/mapper.interface.ts index b7bdc9bf0..73ea8538b 100644 --- a/backend/apps/community/src/data-import/mappers/mapper.interface.ts +++ b/backend/apps/community/src/data-import/mappers/mapper.interface.ts @@ -6,7 +6,7 @@ export class ImportError extends Error { } export interface AnalyticsImportRow { - table: 'analytics' | 'customEV' + type: 'pageview' | 'custom_event' data: Record } diff --git a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts index 99141bb58..f0c5926d3 100644 --- a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts @@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper { created, }) - yield { table: 'analytics', data } + yield { type: 'pageview', data } } } @@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper { page: null, created, }) - data.ev = truncate(ev.name, 256) || '' - yield { table: 'customEV', data } + data.event_name = truncate(ev.name, 256) || '' + yield { type: 'custom_event', data } } } } diff --git a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts index e95c04049..b95e37380 100644 --- a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts @@ -1,7 +1,11 @@ import * as fs from 'fs' import { parse } from 'csv-parse' -import { ImportMapper, AnalyticsImportRow } from './mapper.interface' +import { + ImportMapper, + ImportError, + AnalyticsImportRow, +} from './mapper.interface' import { normalizeNull, truncate, @@ -20,7 +24,7 @@ function validateHeaders(headers: string[]): string[] { ) if (missingColumns.length > 0) { - throw new Error( + throw new ImportError( `CSV does not appear to be a Simple Analytics export. Missing required columns: ${missingColumns.join(', ')}.`, ) } @@ -133,7 +137,7 @@ export class SimpleAnalyticsMapper implements ImportMapper { hostname, ) - const cc = normalizeNull(row.country_code) + const cc = normalizeNull(row.country_code)?.toUpperCase() || null const validCC = cc && /^[A-Z]{2}$/.test(cc) ? cc : null const locale = buildLocale( @@ -172,17 +176,17 @@ export class SimpleAnalyticsMapper implements ImportMapper { const datapoint = normalizeNull(row.datapoint) if (!datapoint || datapoint === 'pageview') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { yield { - table: 'customEV', - data: { ...baseData, ev: truncate(datapoint, 256) || '' }, + type: 'custom_event', + data: { ...baseData, event_name: truncate(datapoint, 256) || '' }, } } } if (!headerChecked) { - throw new Error( + throw new ImportError( 'CSV appears empty or is missing a header row. Please upload the raw CSV export from Simple Analytics.', ) } diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.ts index a1514ae46..3d8c9d9b8 100644 --- a/backend/apps/community/src/data-import/mappers/umami.mapper.ts +++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts @@ -4,7 +4,11 @@ import * as os from 'os' import { Unzip, UnzipInflate } from 'fflate' import { parse } from 'csv-parse' -import { ImportMapper, AnalyticsImportRow } from './mapper.interface' +import { + ImportMapper, + ImportError, + AnalyticsImportRow, +} from './mapper.interface' import { normalizeNull, truncate, @@ -188,7 +192,7 @@ export class UmamiMapper implements ImportMapper { file.originalSize > MAX_UMAMI_CSV_BYTES ) { fail( - new Error( + new ImportError( `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, ), ) @@ -213,7 +217,7 @@ export class UmamiMapper implements ImportMapper { extractedBytes += chunk.length if (extractedBytes > MAX_UMAMI_CSV_BYTES) { fail( - new Error( + new ImportError( `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`, ), ) @@ -263,7 +267,7 @@ export class UmamiMapper implements ImportMapper { if (!foundEntry) { fail( - new Error( + new ImportError( `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`, ), ) @@ -368,12 +372,12 @@ export class UmamiMapper implements ImportMapper { } if (eventType === '1') { - yield { table: 'analytics', data: baseData } + yield { type: 'pageview', data: baseData } } else { const eventName = truncate(normalizeNull(row.event_name), 256) || '' yield { - table: 'customEV', - data: { ...baseData, ev: eventName }, + type: 'custom_event', + data: { ...baseData, event_name: eventName }, } } } diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.ts index 0d9e12b71..37f85f30a 100644 --- a/backend/apps/community/src/goal/goal.controller.ts +++ b/backend/apps/community/src/goal/goal.controller.ts @@ -252,38 +252,38 @@ export class GoalController { await this.goalService.delete(id) } - private buildGoalMatchCondition( - goal: Goal, - _table: 'analytics' | 'customEV', - ): { condition: string; params: Record } { + private buildGoalMatchCondition(goal: Goal): { + condition: string + params: Record + } { const params: Record = {} if (goal.type === GoalType.CUSTOM_EVENT) { - // For custom events, match the event name if (goal.matchType === GoalMatchType.EXACT) { params.goalValue = goal.value || '' - return { condition: `ev = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `ev ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + return { condition: `event_name = {goalValue:String}`, params } } - } else { - // For pageview goals, match the page path - if (goal.matchType === GoalMatchType.EXACT) { - params.goalValue = goal.value || '' - return { condition: `pg = {goalValue:String}`, params } - } else { - // Contains match - params.goalValue = goal.value || '' - return { - condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, - params, - } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } } + params.goalValue = goal.value || '' + return { + condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`, + params, + } + } + + if (goal.matchType === GoalMatchType.EXACT) { + params.goalValue = goal.value || '' + return { condition: `pg = {goalValue:String}`, params } + } + if ((goal.value || '').trim() === '') { + return { condition: '1=0', params: {} } + } + params.goalValue = goal.value || '' + return { + condition: `pg ILIKE concat('%', {goalValue:String}, '%')`, + params, } } @@ -347,9 +347,10 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -358,9 +359,10 @@ export class GoalController { SELECT count(*) as conversions, uniqExact(psid) as uniqueSessions - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -368,6 +370,7 @@ export class GoalController { const queryParams = { pid: goal.projectId, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, ...matchParams, @@ -386,9 +389,10 @@ export class GoalController { // Get total unique sessions for conversion rate const totalSessionsQuery = ` SELECT uniqExact(psid) as totalSessions - FROM analytics + FROM events WHERE pid = {pid:FixedString(12)} + AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} ` @@ -566,7 +570,8 @@ export class GoalController { safeTimezone, ) - const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics' + const goalType = + goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview' const timeBucketFunc = Object.prototype.hasOwnProperty.call( timeBucketConversion, resolvedTimeBucket, @@ -576,7 +581,7 @@ export class GoalController { const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket) const { condition: matchCondition, params: matchParams } = - this.buildGoalMatchCondition(goal, table) + this.buildGoalMatchCondition(goal) const { condition: metaCondition, params: metaParams } = this.buildMetadataCondition(goal) @@ -588,9 +593,10 @@ export class GoalController { FROM ( SELECT *, ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created - FROM ${table} + FROM events WHERE pid = {pid:FixedString(12)} + AND type = {goalType:String} AND ${matchCondition} ${metaCondition} AND created BETWEEN {groupFrom:String} AND {groupTo:String} @@ -601,6 +607,7 @@ export class GoalController { const queryParams = { pid: goal.projectId, + goalType, groupFrom: groupFromUTC, groupTo: groupToUTC, timezone: safeTimezone, diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.ts index 757444527..1c98b7939 100644 --- a/backend/apps/community/src/project/project.controller.ts +++ b/backend/apps/community/src/project/project.controller.ts @@ -576,11 +576,7 @@ export class ProjectController { try { await clickhouse.command({ - query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`, - query_params: { pid: id }, - }) - await clickhouse.command({ - query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`, + query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event')`, query_params: { pid: id }, }) return 'Project reset successfully' @@ -1021,11 +1017,11 @@ export class ProjectController { try { await deleteProjectSharesByProjectClickhouse(id) await clickhouse.command({ - query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`, + query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error', 'performance', 'captcha')`, query_params: { pid: id }, }) await clickhouse.command({ - query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`, + query: `ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}`, query_params: { pid: id }, }) await deleteProjectRedis(id) diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.ts index 0d6147406..160c46ac3 100644 --- a/backend/apps/community/src/project/project.service.ts +++ b/backend/apps/community/src/project/project.service.ts @@ -248,36 +248,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM analytics - WHERE pid IN (${pids}) - ) - OR EXISTS ( - SELECT 1 - FROM customEV - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM - ( - SELECT pid - FROM analytics - WHERE pid IN (${pids}) - UNION ALL - SELECT pid - FROM customEV - WHERE pid IN (${pids}) - ) AS t - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type IN ('pageview', 'custom_event') ` const { data } = await clickhouse @@ -310,23 +284,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM errors - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM errors - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'error' ` const { data } = await clickhouse @@ -359,23 +320,10 @@ export class ProjectService { ) const query = ` - SELECT - pid, - CASE - WHEN EXISTS ( - SELECT 1 - FROM captcha - WHERE pid IN (${pids}) - ) - THEN 1 - ELSE 0 - END AS exists - FROM - ( - SELECT DISTINCT pid - FROM captcha - WHERE pid IN (${pids}) - ); + SELECT DISTINCT pid + FROM events + WHERE pid IN (${pids}) + AND type = 'captcha' ` const { data } = await clickhouse @@ -394,10 +342,7 @@ export class ProjectService { to: string, ): Promise { const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', - 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', + "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}", 'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}', ] diff --git a/backend/apps/community/src/user/user.controller.ts b/backend/apps/community/src/user/user.controller.ts index 64dd758fd..cb3e4fe8b 100644 --- a/backend/apps/community/src/user/user.controller.ts +++ b/backend/apps/community/src/user/user.controller.ts @@ -181,10 +181,7 @@ export class UserController { if (!_isEmpty(projects)) { const pidArray = projects.map((el) => el.id) const queries = [ - 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})', - 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})', + 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})', 'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})', ] await deleteProjectsByUserIdClickhouse(id) diff --git a/backend/meta/clickhouse/generate-dummy-data.js b/backend/meta/clickhouse/generate-dummy-data.js index 1b5f9b8fa..004d81d45 100644 --- a/backend/meta/clickhouse/generate-dummy-data.js +++ b/backend/meta/clickhouse/generate-dummy-data.js @@ -115,10 +115,10 @@ const chalk = { cyan: text => `\x1b[36m${text}\x1b[0m`, } -const insertData = async (pid, rowCount, processedRecords, table) => { +const insertData = async (pid, rowCount, processedRecords) => { try { await clickhouse.insert({ - table, + table: 'events', format: 'JSONEachRow', values: processedRecords, clickhouse_settings: { @@ -154,6 +154,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'pageview', psid: Math.random() < 0.05 ? faker.helpers.arrayElement(PSIDS) @@ -193,7 +194,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'analytics') + insertData(pid, rowCount, records) } const generateCaptchaData = async (pid, rowCount, from, to) => { @@ -201,6 +202,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'captcha', pid, dv: faker.helpers.arrayElement(DEVICES), br: faker.helpers.arrayElement(BROWSERS), @@ -223,7 +225,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'captcha') + insertData(pid, rowCount, records) } const generateCustomEventsData = async (pid, rowCount, from, to) => { @@ -231,8 +233,9 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'custom_event', pid, - ev: faker.helpers.arrayElement(CUSTOM_EVENTS), + event_name: faker.helpers.arrayElement(CUSTOM_EVENTS), pg: faker.helpers.arrayElement(PAGES), dv: faker.helpers.arrayElement(DEVICES), br: faker.helpers.arrayElement(BROWSERS), @@ -267,7 +270,7 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'customEV') + insertData(pid, rowCount, records) } const generatePerformanceData = async (pid, rowCount, from, to) => { @@ -275,6 +278,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => { for (let i = 0; i < rowCount; ++i) { records.push({ + type: 'performance', pid, pg: faker.helpers.arrayElement(PAGES), dv: faker.helpers.arrayElement(DEVICES), @@ -308,7 +312,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => { ), ) - insertData(pid, rowCount, records, 'performance') + insertData(pid, rowCount, records) } const main = () => { diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.js new file mode 100644 index 000000000..164c012af --- /dev/null +++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js @@ -0,0 +1,97 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + `DROP TABLE IF EXISTS ${dbName}.events_tmp;`, + `DROP TABLE IF EXISTS ${dbName}.events_backup;`, + + `CREATE TABLE IF NOT EXISTS ${dbName}.events + ( + type LowCardinality(String), + pid FixedString(12), + psid Nullable(UInt64), + profileId Nullable(String) CODEC(ZSTD(3)), + host Nullable(String) CODEC(ZSTD(3)), + pg Nullable(String) CODEC(ZSTD(3)), + dv LowCardinality(Nullable(String)), + br LowCardinality(Nullable(String)), + brv Nullable(String) CODEC(ZSTD(3)), + os LowCardinality(Nullable(String)), + osv Nullable(String) CODEC(ZSTD(3)), + lc LowCardinality(Nullable(String)), + ref Nullable(String) CODEC(ZSTD(3)), + so Nullable(String) CODEC(ZSTD(3)), + me Nullable(String) CODEC(ZSTD(3)), + ca Nullable(String) CODEC(ZSTD(3)), + te Nullable(String) CODEC(ZSTD(3)), + co Nullable(String) CODEC(ZSTD(3)), + cc Nullable(FixedString(2)), + rg LowCardinality(Nullable(String)), + rgc LowCardinality(Nullable(String)), + ct Nullable(String) CODEC(ZSTD(3)), + isp LowCardinality(Nullable(String)), + og Nullable(String) CODEC(ZSTD(3)), + ut LowCardinality(Nullable(String)), + ctp LowCardinality(Nullable(String)), + \`meta.key\` Array(String) CODEC(ZSTD(3)), + \`meta.value\` Array(String) CODEC(ZSTD(3)), + importID Nullable(UInt8), + event_name Nullable(String) CODEC(ZSTD(3)), + eid Nullable(FixedString(32)), + error_name Nullable(String) CODEC(ZSTD(3)), + error_message Nullable(String) CODEC(ZSTD(3)), + stackTrace Nullable(String) CODEC(ZSTD(3)), + lineno Nullable(UInt32), + colno Nullable(UInt32), + error_filename Nullable(String) CODEC(ZSTD(3)), + dns Nullable(UInt32), + tls Nullable(UInt32), + conn Nullable(UInt32), + response Nullable(UInt32), + render Nullable(UInt32), + domLoad Nullable(UInt32), + pageLoad Nullable(UInt32), + ttfb Nullable(UInt32), + created DateTime('UTC') CODEC(Delta(4), LZ4) + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(created) + ORDER BY (pid, type, created);`, + + `CREATE TABLE ${dbName}.events_tmp AS ${dbName}.events;`, + + `INSERT INTO ${dbName}.events_tmp + (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created) + SELECT + 'pageview', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created + FROM ${dbName}.analytics;`, + + `INSERT INTO ${dbName}.events_tmp + (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, event_name, created) + SELECT + 'custom_event', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, ev, created + FROM ${dbName}.customEV;`, + + `INSERT INTO ${dbName}.events_tmp + (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, error_name, error_message, stackTrace, lineno, colno, error_filename, created) + SELECT + 'error', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, name, message, stackTrace, lineno, colno, filename, created + FROM ${dbName}.errors;`, + + `INSERT INTO ${dbName}.events_tmp + (type, pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created) + SELECT + 'performance', pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created + FROM ${dbName}.performance;`, + + `INSERT INTO ${dbName}.events_tmp + (type, pid, dv, br, os, cc, created) + SELECT + 'captcha', pid, dv, br, os, cc, created + FROM ${dbName}.captcha;`, + + `RENAME TABLE ${dbName}.events TO ${dbName}.events_backup, ${dbName}.events_tmp TO ${dbName}.events;`, + + `DROP TABLE IF EXISTS ${dbName}.events_backup;`, +] + +queriesRunner(queries) diff --git a/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js new file mode 100644 index 000000000..4964e865c --- /dev/null +++ b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js @@ -0,0 +1,18 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + `SELECT throwIf( + count() = 0, + 'The ${dbName}.events table does not exist. Run 2026_05_01_unify_events.js before dropping legacy tables.' + ) + FROM system.tables + WHERE database = '${dbName}' AND name = 'events';`, + + `DROP TABLE IF EXISTS ${dbName}.analytics;`, + `DROP TABLE IF EXISTS ${dbName}.customEV;`, + `DROP TABLE IF EXISTS ${dbName}.errors;`, + `DROP TABLE IF EXISTS ${dbName}.performance;`, + `DROP TABLE IF EXISTS ${dbName}.captcha;`, +] + +queriesRunner(queries) diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js index 5018ca040..beb10a635 100644 --- a/backend/migrations/clickhouse/initialise_database.js +++ b/backend/migrations/clickhouse/initialise_database.js @@ -4,51 +4,14 @@ const { queriesRunner, dbName, databaselessQueriesRunner } = require('./setup') const CLICKHOUSE_DB_INIT_QUERIES = [`CREATE DATABASE IF NOT EXISTS ${dbName}`] const CLICKHOUSE_INIT_QUERIES = [ - // The traffic data table - `CREATE TABLE IF NOT EXISTS ${dbName}.analytics - ( - psid Nullable(UInt64), - profileId Nullable(String) CODEC(ZSTD(3)), - pid FixedString(12), - host Nullable(String) CODEC(ZSTD(3)), - pg Nullable(String) CODEC(ZSTD(3)), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - brv Nullable(String) CODEC(ZSTD(3)), - os LowCardinality(Nullable(String)), - osv Nullable(String) CODEC(ZSTD(3)), - lc LowCardinality(Nullable(String)), - ref Nullable(String) CODEC(ZSTD(3)), - so Nullable(String) CODEC(ZSTD(3)), - me Nullable(String) CODEC(ZSTD(3)), - ca Nullable(String) CODEC(ZSTD(3)), - te Nullable(String) CODEC(ZSTD(3)), - co Nullable(String) CODEC(ZSTD(3)), - cc Nullable(FixedString(2)), - rg LowCardinality(Nullable(String)), - rgc LowCardinality(Nullable(String)), - ct Nullable(String) CODEC(ZSTD(3)), - isp LowCardinality(Nullable(String)), - og Nullable(String) CODEC(ZSTD(3)), - ut LowCardinality(Nullable(String)), - ctp LowCardinality(Nullable(String)), - \`meta.key\` Array(String) CODEC(ZSTD(3)), - \`meta.value\` Array(String) CODEC(ZSTD(3)), - importID Nullable(UInt8), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - // Custom events table - `CREATE TABLE IF NOT EXISTS ${dbName}.customEV + `CREATE TABLE IF NOT EXISTS ${dbName}.events ( + type LowCardinality(String), + pid FixedString(12), psid Nullable(UInt64), profileId Nullable(String) CODEC(ZSTD(3)), - pid FixedString(12), host Nullable(String) CODEC(ZSTD(3)), - ev String CODEC(ZSTD(3)), pg Nullable(String) CODEC(ZSTD(3)), dv LowCardinality(Nullable(String)), br LowCardinality(Nullable(String)), @@ -73,29 +36,14 @@ const CLICKHOUSE_INIT_QUERIES = [ \`meta.key\` Array(String) CODEC(ZSTD(3)), \`meta.value\` Array(String) CODEC(ZSTD(3)), importID Nullable(UInt8), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - - // The performance data table - `CREATE TABLE IF NOT EXISTS ${dbName}.performance - ( - pid FixedString(12), - host Nullable(String) CODEC(ZSTD(3)), - pg Nullable(String) CODEC(ZSTD(3)), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - brv Nullable(String) CODEC(ZSTD(3)), - cc Nullable(FixedString(2)), - rg LowCardinality(Nullable(String)), - rgc LowCardinality(Nullable(String)), - ct Nullable(String) CODEC(ZSTD(3)), - isp LowCardinality(Nullable(String)), - og Nullable(String) CODEC(ZSTD(3)), - ut LowCardinality(Nullable(String)), - ctp LowCardinality(Nullable(String)), + event_name Nullable(String) CODEC(ZSTD(3)), + eid Nullable(FixedString(32)), + error_name Nullable(String) CODEC(ZSTD(3)), + error_message Nullable(String) CODEC(ZSTD(3)), + stackTrace Nullable(String) CODEC(ZSTD(3)), + lineno Nullable(UInt32), + colno Nullable(UInt32), + error_filename Nullable(String) CODEC(ZSTD(3)), dns Nullable(UInt32), tls Nullable(UInt32), conn Nullable(UInt32), @@ -108,44 +56,7 @@ const CLICKHOUSE_INIT_QUERIES = [ ) ENGINE = MergeTree() PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - - // Error events table - `CREATE TABLE IF NOT EXISTS ${dbName}.errors - ( - psid Nullable(UInt64), - profileId Nullable(String) CODEC(ZSTD(3)), - eid FixedString(32), - pid FixedString(12), - host Nullable(String) CODEC(ZSTD(3)), - pg Nullable(String) CODEC(ZSTD(3)), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - brv Nullable(String) CODEC(ZSTD(3)), - os LowCardinality(Nullable(String)), - osv Nullable(String) CODEC(ZSTD(3)), - lc LowCardinality(Nullable(String)), - cc LowCardinality(Nullable(FixedString(2))), - rg LowCardinality(Nullable(String)), - rgc LowCardinality(Nullable(String)), - ct Nullable(String) CODEC(ZSTD(3)), - isp LowCardinality(Nullable(String)), - og Nullable(String) CODEC(ZSTD(3)), - ut LowCardinality(Nullable(String)), - ctp LowCardinality(Nullable(String)), - name String CODEC(ZSTD(3)), - message Nullable(String) CODEC(ZSTD(3)), - stackTrace Nullable(String) CODEC(ZSTD(3)), - \`meta.key\` Array(String) CODEC(ZSTD(3)), - \`meta.value\` Array(String) CODEC(ZSTD(3)), - lineno Nullable(UInt32), - colno Nullable(UInt32), - filename Nullable(String) CODEC(ZSTD(3)), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, + ORDER BY (pid, type, created);`, // Error events status table `CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses ( @@ -201,20 +112,6 @@ const CLICKHOUSE_INIT_QUERIES = [ ORDER BY (pid, experimentId, created) TTL created + INTERVAL 1 YEAR;`, - // The CAPTCHA data table - `CREATE TABLE IF NOT EXISTS ${dbName}.captcha - ( - pid FixedString(12), - dv LowCardinality(Nullable(String)), - br LowCardinality(Nullable(String)), - os Nullable(String) CODEC(ZSTD(3)), - cc Nullable(FixedString(2)), - created DateTime('UTC') CODEC(Delta(4), LZ4) - ) - ENGINE = MergeTree() - PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);`, - // Bot block events table for advanced bot protection reporting `CREATE TABLE IF NOT EXISTS ${dbName}.bot_blocks ( diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.ts index 3eb66d8d5..052d09680 100644 --- a/web/app/api/api.server.ts +++ b/web/app/api/api.server.ts @@ -1036,10 +1036,12 @@ export async function getFunnelSessionsServer( } interface PageflowItem { - type: 'pageview' | 'event' | 'error' + type: 'pageview' | 'event' | 'error' | 'sale' | 'refund' value: string created: string metadata?: { key: string; value: string }[] + amount?: number + currency?: string } export interface SessionDetailsResponse { @@ -1067,6 +1069,7 @@ export interface SessionDetailsResponse { refunds?: number created: string sdur?: number + isLive?: boolean profileId: string | null isIdentified: 1 | 0 isFirstSession: 1 | 0 diff --git a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx index 0dd99425a..7a3573cc7 100644 --- a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx +++ b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx @@ -42,10 +42,12 @@ import { Pageflow } from './Pageflow' import { SessionChart } from './SessionChart' interface PageflowItem { - type: 'pageview' | 'event' | 'error' + type: 'pageview' | 'event' | 'error' | 'sale' | 'refund' value: string created: string metadata?: { key: string; value: string }[] + amount?: number + currency?: string } interface SessionDetailViewProps { diff --git a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx index c9631ed51..e43ad3018 100644 --- a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx +++ b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx @@ -85,10 +85,12 @@ class SessionsErrorBoundary extends Component< const SESSIONS_TAKE = 30 interface PageflowItem { - type: 'pageview' | 'event' | 'error' + type: 'pageview' | 'event' | 'error' | 'sale' | 'refund' value: string created: string metadata?: { key: string; value: string }[] + amount?: number + currency?: string } interface ActiveSession { @@ -201,7 +203,9 @@ const SessionsViewInner = ({ dv: apiDetails.dv, profileId: apiDetails.profileId, sdur: apiDetails.sdur, - isLive: matchingSession?.isLive === 1, + isLive: matchingSession + ? matchingSession.isLive === 1 + : apiDetails.isLive, } return { details,