From bffb98cd5147e4f9daa03e21a1adc95384172568 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 15 Oct 2024 11:29:27 -0700 Subject: [PATCH] add tag migrations and update send / saveEvent --- db/clickhouse/migrations/04_add_tag.sql | 78 +++++++++++++++++++ db/mysql/migrations/07_add_tag/migration.sql | 5 ++ db/mysql/schema.prisma | 2 + .../migrations/07_add_tag/migration.sql | 5 ++ db/postgresql/schema.prisma | 3 + src/pages/api/send.ts | 3 +- src/queries/analytics/events/saveEvent.ts | 7 ++ 7 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 db/clickhouse/migrations/04_add_tag.sql create mode 100644 db/mysql/migrations/07_add_tag/migration.sql create mode 100644 db/postgresql/migrations/07_add_tag/migration.sql diff --git a/db/clickhouse/migrations/04_add_tag.sql b/db/clickhouse/migrations/04_add_tag.sql new file mode 100644 index 0000000000..cfe0de20bd --- /dev/null +++ b/db/clickhouse/migrations/04_add_tag.sql @@ -0,0 +1,78 @@ +-- add tag column +ALTER TABLE umami.website_event ADD COLUMN "tag" String AFTER "event_name"; +ALTER TABLE umami.website_event_stats_hourly ADD COLUMN "tag" String AFTER "max_time"; + +-- update materialized view +DROP TABLE umami.website_event_stats_hourly_mv; + +CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv +TO umami.website_event_stats_hourly +AS +SELECT + website_id, + session_id, + visit_id, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + entry_url, + exit_url, + url_paths as url_path, + url_query, + referrer_domain, + page_title, + event_type, + event_name, + views, + min_time, + max_time, + tag, + timestamp as created_at +FROM (SELECT + website_id, + session_id, + visit_id, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + argMinState(url_path, created_at) entry_url, + argMaxState(url_path, created_at) exit_url, + arrayFilter(x -> x != '', groupArray(url_path)) as url_paths, + arrayFilter(x -> x != '', groupArray(url_query)) url_query, + arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain, + arrayFilter(x -> x != '', groupArray(page_title)) page_title, + event_type, + if(event_type = 2, groupArray(event_name), []) event_name, + sumIf(1, event_type = 1) views, + min(created_at) min_time, + max(created_at) max_time, + tag, + toStartOfHour(created_at) timestamp +FROM umami.website_event +GROUP BY website_id, + session_id, + visit_id, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + event_type, + tag, + timestamp); \ No newline at end of file diff --git a/db/mysql/migrations/07_add_tag/migration.sql b/db/mysql/migrations/07_add_tag/migration.sql new file mode 100644 index 0000000000..f1c349bd42 --- /dev/null +++ b/db/mysql/migrations/07_add_tag/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE `website_event` ADD COLUMN `tag` VARCHAR(50) NULL; + +-- CreateIndex +CREATE INDEX `website_event_website_id_created_at_tag_idx` ON `website_event`(`website_id`, `created_at`, `tag`); diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index a7722accdc..5f9d4dff59 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -102,6 +102,7 @@ model WebsiteEvent { pageTitle String? @map("page_title") @db.VarChar(500) eventType Int @default(1) @map("event_type") @db.UnsignedInt eventName String? @map("event_name") @db.VarChar(50) + tag String? @db.VarChar(50) eventData EventData[] session Session @relation(fields: [sessionId], references: [id]) @@ -116,6 +117,7 @@ model WebsiteEvent { @@index([websiteId, createdAt, referrerDomain]) @@index([websiteId, createdAt, pageTitle]) @@index([websiteId, createdAt, eventName]) + @@index([websiteId, createdAt, tag]) @@index([websiteId, sessionId, createdAt]) @@index([websiteId, visitId, createdAt]) @@map("website_event") diff --git a/db/postgresql/migrations/07_add_tag/migration.sql b/db/postgresql/migrations/07_add_tag/migration.sql new file mode 100644 index 0000000000..ef465f838f --- /dev/null +++ b/db/postgresql/migrations/07_add_tag/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "website_event" ADD COLUMN "tag" VARCHAR(50); + +-- CreateIndex +CREATE INDEX "website_event_website_id_created_at_tag_idx" ON "website_event"("website_id", "created_at", "tag"); diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index b9009e0fb3..364986dcd2 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -102,6 +102,7 @@ model WebsiteEvent { pageTitle String? @map("page_title") @db.VarChar(500) eventType Int @default(1) @map("event_type") @db.Integer eventName String? @map("event_name") @db.VarChar(50) + tag String? @db.VarChar(50) eventData EventData[] session Session @relation(fields: [sessionId], references: [id]) @@ -111,11 +112,13 @@ model WebsiteEvent { @@index([visitId]) @@index([websiteId]) @@index([websiteId, createdAt]) + @@index([websiteId, createdAt, urlPath]) @@index([websiteId, createdAt, urlQuery]) @@index([websiteId, createdAt, referrerDomain]) @@index([websiteId, createdAt, pageTitle]) @@index([websiteId, createdAt, eventName]) + @@index([websiteId, createdAt, tag]) @@index([websiteId, sessionId, createdAt]) @@index([websiteId, visitId, createdAt]) @@map("website_event") diff --git a/src/pages/api/send.ts b/src/pages/api/send.ts index 23640de9f0..fb4e90c744 100644 --- a/src/pages/api/send.ts +++ b/src/pages/api/send.ts @@ -96,7 +96,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { } const { type, payload } = req.body; - const { url, referrer, name: eventName, data, title } = payload; + const { url, referrer, name: eventName, data, title, tag } = payload; const pageTitle = safeDecodeURI(title); await useSession(req, res); @@ -143,6 +143,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => { eventData: data, ...session, sessionId: session.id, + tag, }); } else if (type === COLLECTION_TYPE.identify) { if (!data) { diff --git a/src/queries/analytics/events/saveEvent.ts b/src/queries/analytics/events/saveEvent.ts index 6c0f917baa..6a3391d674 100644 --- a/src/queries/analytics/events/saveEvent.ts +++ b/src/queries/analytics/events/saveEvent.ts @@ -28,6 +28,7 @@ export async function saveEvent(args: { subdivision1?: string; subdivision2?: string; city?: string; + tag?: string; }) { return runQuery({ [PRISMA]: () => relationalQuery(args), @@ -47,6 +48,7 @@ async function relationalQuery(data: { pageTitle?: string; eventName?: string; eventData?: any; + tag?: string; }) { const { websiteId, @@ -60,6 +62,7 @@ async function relationalQuery(data: { eventName, eventData, pageTitle, + tag, } = data; const websiteEventId = uuid(); @@ -77,6 +80,7 @@ async function relationalQuery(data: { pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH), eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, + tag, }, }); @@ -116,6 +120,7 @@ async function clickhouseQuery(data: { subdivision1?: string; subdivision2?: string; city?: string; + tag?: string; }) { const { websiteId, @@ -133,6 +138,7 @@ async function clickhouseQuery(data: { subdivision1, subdivision2, city, + tag, ...args } = data; const { insert, getUTCString } = clickhouse; @@ -163,6 +169,7 @@ async function clickhouseQuery(data: { page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH), event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, + tag: tag, created_at: createdAt, };