From b93f3499cd69787b5144e1dbb5a872ded65311e8 Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Mon, 12 Dec 2022 04:30:02 -0800 Subject: [PATCH 01/12] feat:Referrer --- app/page_event.html | 9 +++ src/event-schemas/page-view-event.json | 4 +- src/sessions/PageManager.ts | 42 +++++++++++ src/sessions/__integ__/PageManager.test.ts | 31 ++++++++ src/sessions/__tests__/PageManager.test.ts | 86 +++++++++++++++++++++- 5 files changed, 170 insertions(+), 2 deletions(-) diff --git a/app/page_event.html b/app/page_event.html index 467c7784..2bc82368 100644 --- a/app/page_event.html +++ b/app/page_event.html @@ -71,6 +71,12 @@ cwr('recordPageView', '/page_view_two'); } + function createReferrer() { + Object.defineProperty(document, 'referrer', { + value: 'http://amazon.com/searchresults/1/' + }); + } + function recordPageViewWithPageTagAttribute() { cwr('recordPageView', { pageId: '/page_view_two', @@ -179,6 +185,9 @@ +
diff --git a/src/event-schemas/page-view-event.json b/src/event-schemas/page-view-event.json index 60bd02b8..41cad259 100644 --- a/src/event-schemas/page-view-event.json +++ b/src/event-schemas/page-view-event.json @@ -12,7 +12,9 @@ "pageId": { "type": "string" }, "pageInteractionId": { "type": "string" }, "interaction": { "type": "number" }, - "parentPageInteractionId": { "type": "string" } + "parentPageInteractionId": { "type": "string" }, + "referrer": { "type": "string" }, + "referrerDomain": { "type": "string" } }, "additionalProperties": false, "required": ["pageId"] diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index 46f286ca..de24e88b 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -8,6 +8,8 @@ export type Page = { pageId: string; interaction: number; parentPageId?: string; + referrer?: string | null; + referrerDomain?: string | null; start: number; }; @@ -129,6 +131,11 @@ export class PageManager { pageId, parentPageId: resumed.pageId, interaction: resumed.interaction + 1, + referrer: + document.referrer !== undefined && document.referrer !== '' + ? document.referrer + : null, + referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; this.resumed = undefined; @@ -165,6 +172,11 @@ export class PageManager { pageId, parentPageId: currentPage.pageId, interaction: currentPage.interaction + 1, + referrer: + document.referrer !== undefined && document.referrer !== '' + ? document.referrer + : null, + referrerDomain: this.getDomainFromReferrer(), start: startTime }; } @@ -173,6 +185,11 @@ export class PageManager { this.page = { pageId, interaction: 0, + referrer: + document.referrer !== undefined && document.referrer !== '' + ? document.referrer + : null, + referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; } @@ -220,6 +237,14 @@ export class PageManager { pageViewEvent.parentPageInteractionId = page.parentPageId + '-' + (page.interaction - 1); } + + if (page.referrer !== null) { + pageViewEvent.referrer = page.referrer; + + if (page.referrerDomain !== null) { + pageViewEvent.referrerDomain = page.referrerDomain; + } + } } return pageViewEvent; @@ -235,4 +260,21 @@ export class PageManager { private useCookies() { return navigator.cookieEnabled && this.config.allowCookies; } + + /* + Parses the domain from the referrer, if it is available + */ + private getDomainFromReferrer() { + if (document.referrer !== undefined || document.referrer !== '') { + const domainName = document.referrer.match( + '([A-Za-z]+://[^/]+)[/]?' + ); + + if (domainName) { + return domainName[1]; + } + } else { + return null; + } + } } diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index ecc61fb3..9cc4c0c7 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -1,6 +1,7 @@ import { Selector } from 'testcafe'; import { REQUEST_BODY } from '../../test-utils/integ-test-utils'; import { PAGE_VIEW_EVENT_TYPE } from '../../plugins/utils/constant'; +import { create } from 'lodash'; const recordPageView: Selector = Selector(`#recordPageView`); const recordPageViewWithPageTagAttribute: Selector = Selector( @@ -12,6 +13,7 @@ const recordPageViewWithCustomPageAttributes: Selector = Selector( const dispatch: Selector = Selector(`#dispatch`); const clear: Selector = Selector(`#clearRequestResponse`); const doNotRecordPageView = Selector(`#doNotRecordPageView`); +const createReferrer: Selector = Selector(`#createReferrer`); fixture('PageViewEventPlugin').page('http://localhost:8080/page_event.html'); @@ -206,3 +208,32 @@ test('when custom page attributes are set when manually recording page view even customPageAttributeBoolean: true }); }); + +test('when referrer exists, then page view event details records it', async (t: TestController) => { + // If we click too soon, the client/event collector plugin will not be loaded and will not record the click. + // This could be a symptom of an issue with RUM web client load speed, or prioritization of script execution. + + await t + .wait(300) + .click(dispatch) + .expect(REQUEST_BODY.textContent) + .contains('BatchId') + .click(clear) + .wait(300) + .click(createReferrer) + .click(recordPageView) + .click(dispatch) + .expect(REQUEST_BODY.textContent) + .contains('BatchId'); + + const requestBody = JSON.parse(await REQUEST_BODY.textContent); + + const pages = requestBody.RumEvents.filter( + (e) => e.type === PAGE_VIEW_EVENT_TYPE + ).map((e) => JSON.parse(e.details)); + + await t.expect(pages.length).eql(1).expect(pages[0]).contains({ + referrer: 'http://amazon.com/searchresults/1/', + referrerDomain: 'http://amazon.com' + }); +}); diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 15af59a2..4109ad9e 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -11,7 +11,8 @@ const record = jest.fn(); declare const jsdom: any; Object.defineProperty(document, 'referrer', { - value: 'https://console.aws.amazon.com' + value: 'https://console.aws.amazon.com', + configurable: true }); Object.defineProperty(document, 'title', { value: 'Amazon AWS Console' }); global.fetch = mockFetch; @@ -487,3 +488,86 @@ describe('PageManager tests', () => { expect(pageManager.getPage().start).toEqual(2500); }); }); + +test('when complete referrer is available from the DOM', async () => { + // Init + const config: Config = { + ...DEFAULT_CONFIG, + allowCookies: true + }; + const pageManager: PageManager = new PageManager(config, record); + + Object.defineProperty(document, 'referrer', { + value: 'http://abc.com/consoles', + configurable: true + }); + + // Run + pageManager.recordPageView('/console/home'); + + // Assert + expect(pageManager.getPage()).toMatchObject({ + referrer: 'http://abc.com/consoles', + referrerDomain: 'http://abc.com', + pageId: '/console/home' + }); + + window.removeEventListener( + 'popstate', + (pageManager as any).popstateListener + ); +}); + +test('when only domain level referrer is available from the DOM', async () => { + // Init + const config: Config = { + ...DEFAULT_CONFIG, + allowCookies: true + }; + const pageManager: PageManager = new PageManager(config, record); + + Object.defineProperty(document, 'referrer', { + value: 'http://abc.com', + configurable: true + }); + // Run + pageManager.recordPageView('/console/home'); + + // Assert + expect(pageManager.getPage()).toMatchObject({ + referrer: 'http://abc.com', + referrerDomain: 'http://abc.com', + pageId: '/console/home' + }); + + window.removeEventListener( + 'popstate', + (pageManager as any).popstateListener + ); +}); + +test('when referrer is not available from the DOM', async () => { + // Init + const config: Config = { + ...DEFAULT_CONFIG, + allowCookies: true + }; + const pageManager: PageManager = new PageManager(config, record); + + Object.defineProperty(document, 'referrer', { + value: '', + configurable: true + }); + // Run + pageManager.recordPageView('/console/home'); + + // Assert + expect(pageManager.getPage()).toMatchObject({ + pageId: '/console/home' + }); + + window.removeEventListener( + 'popstate', + (pageManager as any).popstateListener + ); +}); From a6d93aa8d1534be3e2ecc19ecd25ddfd33810a73 Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 14 Dec 2022 01:11:18 -0800 Subject: [PATCH 02/12] Revision 2 --- src/sessions/PageManager.ts | 28 ++++++++++++---------- src/sessions/__tests__/PageManager.test.ts | 6 ++--- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index de24e88b..175e0960 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -131,10 +131,7 @@ export class PageManager { pageId, parentPageId: resumed.pageId, interaction: resumed.interaction + 1, - referrer: - document.referrer !== undefined && document.referrer !== '' - ? document.referrer - : null, + referrer: this.getReferrer(), referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; @@ -172,10 +169,7 @@ export class PageManager { pageId, parentPageId: currentPage.pageId, interaction: currentPage.interaction + 1, - referrer: - document.referrer !== undefined && document.referrer !== '' - ? document.referrer - : null, + referrer: this.getReferrer(), referrerDomain: this.getDomainFromReferrer(), start: startTime }; @@ -185,10 +179,7 @@ export class PageManager { this.page = { pageId, interaction: 0, - referrer: - document.referrer !== undefined && document.referrer !== '' - ? document.referrer - : null, + referrer: this.getReferrer(), referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; @@ -265,7 +256,7 @@ export class PageManager { Parses the domain from the referrer, if it is available */ private getDomainFromReferrer() { - if (document.referrer !== undefined || document.referrer !== '') { + if (document.referrer !== undefined && document.referrer !== '') { const domainName = document.referrer.match( '([A-Za-z]+://[^/]+)[/]?' ); @@ -277,4 +268,15 @@ export class PageManager { return null; } } + + /* + Get the referrer, if it can be read from the DOM + */ + private getReferrer() { + if (document.referrer !== undefined && document.referrer !== '') { + return document.referrer; + } else { + return null; + } + } } diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 4109ad9e..1f360162 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -489,7 +489,7 @@ describe('PageManager tests', () => { }); }); -test('when complete referrer is available from the DOM', async () => { +test('when complete referrer is available from the DOM then is recorded in page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -518,7 +518,7 @@ test('when complete referrer is available from the DOM', async () => { ); }); -test('when only domain level referrer is available from the DOM', async () => { +test('when only domain level referrer is available from the DOM then is recorded in page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -546,7 +546,7 @@ test('when only domain level referrer is available from the DOM', async () => { ); }); -test('when referrer is not available from the DOM', async () => { +test('when referrer from the DOM is not valid then it is not recorded in page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, From ca4b499d0d5b31fb980f62e2555b449b0c221a2c Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 14 Dec 2022 11:45:01 -0800 Subject: [PATCH 03/12] Revision 3 --- src/sessions/PageManager.ts | 31 +++++----------------- src/sessions/__integ__/PageManager.test.ts | 2 +- src/sessions/__tests__/PageManager.test.ts | 10 ++++--- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index 175e0960..d1a6c989 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -131,7 +131,7 @@ export class PageManager { pageId, parentPageId: resumed.pageId, interaction: resumed.interaction + 1, - referrer: this.getReferrer(), + referrer: document.referrer, referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; @@ -169,7 +169,7 @@ export class PageManager { pageId, parentPageId: currentPage.pageId, interaction: currentPage.interaction + 1, - referrer: this.getReferrer(), + referrer: document.referrer, referrerDomain: this.getDomainFromReferrer(), start: startTime }; @@ -179,7 +179,7 @@ export class PageManager { this.page = { pageId, interaction: 0, - referrer: this.getReferrer(), + referrer: document.referrer, referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; @@ -256,27 +256,10 @@ export class PageManager { Parses the domain from the referrer, if it is available */ private getDomainFromReferrer() { - if (document.referrer !== undefined && document.referrer !== '') { - const domainName = document.referrer.match( - '([A-Za-z]+://[^/]+)[/]?' - ); - - if (domainName) { - return domainName[1]; - } - } else { - return null; - } - } - - /* - Get the referrer, if it can be read from the DOM - */ - private getReferrer() { - if (document.referrer !== undefined && document.referrer !== '') { - return document.referrer; - } else { - return null; + try { + return new URL(document.referrer).hostname; + } catch (e) { + return ''; } } } diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index 9cc4c0c7..3565c96c 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -234,6 +234,6 @@ test('when referrer exists, then page view event details records it', async (t: await t.expect(pages.length).eql(1).expect(pages[0]).contains({ referrer: 'http://amazon.com/searchresults/1/', - referrerDomain: 'http://amazon.com' + referrerDomain: 'amazon.com' }); }); diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 1f360162..37d077f0 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -508,7 +508,7 @@ test('when complete referrer is available from the DOM then is recorded in page // Assert expect(pageManager.getPage()).toMatchObject({ referrer: 'http://abc.com/consoles', - referrerDomain: 'http://abc.com', + referrerDomain: 'abc.com', pageId: '/console/home' }); @@ -536,7 +536,7 @@ test('when only domain level referrer is available from the DOM then is recorded // Assert expect(pageManager.getPage()).toMatchObject({ referrer: 'http://abc.com', - referrerDomain: 'http://abc.com', + referrerDomain: 'abc.com', pageId: '/console/home' }); @@ -546,7 +546,7 @@ test('when only domain level referrer is available from the DOM then is recorded ); }); -test('when referrer from the DOM is not valid then it is not recorded in page view event', async () => { +test('when referrer from the DOM is empty then it is recorded as empty in the page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -563,7 +563,9 @@ test('when referrer from the DOM is not valid then it is not recorded in page vi // Assert expect(pageManager.getPage()).toMatchObject({ - pageId: '/console/home' + pageId: '/console/home', + referrer: '', + referrerDomain: '' }); window.removeEventListener( From 2efa9343ea0b9e0448f696d29ab64d85f169f859 Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 4 Jan 2023 08:30:51 -0800 Subject: [PATCH 04/12] Adding referrer to metadata instead of event details --- src/event-schemas/meta-data.json | 5 ++++- src/event-schemas/page-view-event.json | 4 +--- src/sessions/PageManager.ts | 22 ++++++------------- src/sessions/__integ__/PageManager.test.ts | 4 ++-- src/sessions/__tests__/PageManager.test.ts | 25 +++++++++++++++------- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/event-schemas/meta-data.json b/src/event-schemas/meta-data.json index 4708838d..bfd094da 100644 --- a/src/event-schemas/meta-data.json +++ b/src/event-schemas/meta-data.json @@ -45,7 +45,10 @@ "interaction": { "type": "number" }, - "referrerUrl": { + "referrer": { + "type": "string" + }, + "referrerDomain": { "type": "string" }, "pageTitle": { diff --git a/src/event-schemas/page-view-event.json b/src/event-schemas/page-view-event.json index 41cad259..60bd02b8 100644 --- a/src/event-schemas/page-view-event.json +++ b/src/event-schemas/page-view-event.json @@ -12,9 +12,7 @@ "pageId": { "type": "string" }, "pageInteractionId": { "type": "string" }, "interaction": { "type": "number" }, - "parentPageInteractionId": { "type": "string" }, - "referrer": { "type": "string" }, - "referrerDomain": { "type": "string" } + "parentPageInteractionId": { "type": "string" } }, "additionalProperties": false, "required": ["pageId"] diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index d1a6c989..a233dd94 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -8,8 +8,6 @@ export type Page = { pageId: string; interaction: number; parentPageId?: string; - referrer?: string | null; - referrerDomain?: string | null; start: number; }; @@ -19,6 +17,8 @@ export type Attributes = { parentPageId?: string; interaction?: number; pageTags?: string[]; + referrer?: string; + referrerDomain?: string; // The value types of custom attributes are restricted to the types: string | number | boolean // However, given that pageTags is a string array, we need to include it as a valid type // Events will be verified by our service to validate attribute value types where @@ -131,8 +131,6 @@ export class PageManager { pageId, parentPageId: resumed.pageId, interaction: resumed.interaction + 1, - referrer: document.referrer, - referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; this.resumed = undefined; @@ -169,8 +167,6 @@ export class PageManager { pageId, parentPageId: currentPage.pageId, interaction: currentPage.interaction + 1, - referrer: document.referrer, - referrerDomain: this.getDomainFromReferrer(), start: startTime }; } @@ -179,8 +175,6 @@ export class PageManager { this.page = { pageId, interaction: 0, - referrer: document.referrer, - referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; } @@ -199,6 +193,10 @@ export class PageManager { if (page.parentPageId !== undefined) { this.attributes.parentPageId = page.parentPageId; } + if (document.referrer !== undefined) { + this.attributes.referrer = document.referrer; + this.attributes.referrerDomain = this.getDomainFromReferrer(); + } } if (customPageAttributes?.pageTags) { @@ -228,14 +226,6 @@ export class PageManager { pageViewEvent.parentPageInteractionId = page.parentPageId + '-' + (page.interaction - 1); } - - if (page.referrer !== null) { - pageViewEvent.referrer = page.referrer; - - if (page.referrerDomain !== null) { - pageViewEvent.referrerDomain = page.referrerDomain; - } - } } return pageViewEvent; diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index 3565c96c..86bca10d 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -209,7 +209,7 @@ test('when custom page attributes are set when manually recording page view even }); }); -test('when referrer exists, then page view event details records it', async (t: TestController) => { +test('when referrer exists, then metadata records it', async (t: TestController) => { // If we click too soon, the client/event collector plugin will not be loaded and will not record the click. // This could be a symptom of an issue with RUM web client load speed, or prioritization of script execution. @@ -230,7 +230,7 @@ test('when referrer exists, then page view event details records it', async (t: const pages = requestBody.RumEvents.filter( (e) => e.type === PAGE_VIEW_EVENT_TYPE - ).map((e) => JSON.parse(e.details)); + ).map((e) => JSON.parse(e.metadata)); await t.expect(pages.length).eql(1).expect(pages[0]).contains({ referrer: 'http://amazon.com/searchresults/1/', diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 37d077f0..add3802e 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -489,7 +489,7 @@ describe('PageManager tests', () => { }); }); -test('when complete referrer is available from the DOM then is recorded in page view event', async () => { +test('when complete referrer is available from the DOM then is recorded in the metadata', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -507,18 +507,21 @@ test('when complete referrer is available from the DOM then is recorded in page // Assert expect(pageManager.getPage()).toMatchObject({ - referrer: 'http://abc.com/consoles', - referrerDomain: 'abc.com', pageId: '/console/home' }); + expect(pageManager.getAttributes()).toMatchObject({ + referrer: 'http://abc.com/consoles', + referrerDomain: 'abc.com' + }); + window.removeEventListener( 'popstate', (pageManager as any).popstateListener ); }); -test('when only domain level referrer is available from the DOM then is recorded in page view event', async () => { +test('when only domain level referrer is available from the DOM then is recorded in the metadata', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -535,18 +538,21 @@ test('when only domain level referrer is available from the DOM then is recorded // Assert expect(pageManager.getPage()).toMatchObject({ - referrer: 'http://abc.com', - referrerDomain: 'abc.com', pageId: '/console/home' }); + expect(pageManager.getAttributes()).toMatchObject({ + referrer: 'http://abc.com', + referrerDomain: 'abc.com' + }); + window.removeEventListener( 'popstate', (pageManager as any).popstateListener ); }); -test('when referrer from the DOM is empty then it is recorded as empty in the page view event', async () => { +test('when referrer from the DOM is empty then it is recorded as empty in the metadata', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -563,7 +569,10 @@ test('when referrer from the DOM is empty then it is recorded as empty in the pa // Assert expect(pageManager.getPage()).toMatchObject({ - pageId: '/console/home', + pageId: '/console/home' + }); + + expect(pageManager.getAttributes()).toMatchObject({ referrer: '', referrerDomain: '' }); From fbc10acb9b5736b20f5b1d8f76eb42b6a8ff342f Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 11 Jan 2023 12:20:55 -0800 Subject: [PATCH 05/12] Updated referrer field names with new convention --- src/event-schemas/meta-data.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/event-schemas/meta-data.json b/src/event-schemas/meta-data.json index bfd094da..1ca89706 100644 --- a/src/event-schemas/meta-data.json +++ b/src/event-schemas/meta-data.json @@ -45,10 +45,10 @@ "interaction": { "type": "number" }, - "referrer": { + "aws:referrer": { "type": "string" }, - "referrerDomain": { + "aws:referrerDomain": { "type": "string" }, "pageTitle": { From 5a1190dd81cd8b85d994c2e352c9a4db518e8780 Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 11 Jan 2023 12:34:13 -0800 Subject: [PATCH 06/12] Updated referrer field names with new convention --- src/sessions/PageManager.ts | 10 ++++++---- src/sessions/__integ__/PageManager.test.ts | 4 ++-- src/sessions/__tests__/PageManager.test.ts | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index a233dd94..3c3bbd2f 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -17,8 +17,8 @@ export type Attributes = { parentPageId?: string; interaction?: number; pageTags?: string[]; - referrer?: string; - referrerDomain?: string; + 'aws:referrer'?: string; + 'aws:referrerDomain'?: string; // The value types of custom attributes are restricted to the types: string | number | boolean // However, given that pageTags is a string array, we need to include it as a valid type // Events will be verified by our service to validate attribute value types where @@ -194,8 +194,10 @@ export class PageManager { this.attributes.parentPageId = page.parentPageId; } if (document.referrer !== undefined) { - this.attributes.referrer = document.referrer; - this.attributes.referrerDomain = this.getDomainFromReferrer(); + this.attributes['aws:referrer'] = document.referrer; + this.attributes[ + 'aws:referrerDomain' + ] = this.getDomainFromReferrer(); } } diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index 86bca10d..ada30547 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -233,7 +233,7 @@ test('when referrer exists, then metadata records it', async (t: TestController) ).map((e) => JSON.parse(e.metadata)); await t.expect(pages.length).eql(1).expect(pages[0]).contains({ - referrer: 'http://amazon.com/searchresults/1/', - referrerDomain: 'amazon.com' + 'aws:referrer': 'http://amazon.com/searchresults/1/', + 'aws:referrerDomain': 'amazon.com' }); }); diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index add3802e..7fc0b35f 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -511,8 +511,8 @@ test('when complete referrer is available from the DOM then is recorded in the m }); expect(pageManager.getAttributes()).toMatchObject({ - referrer: 'http://abc.com/consoles', - referrerDomain: 'abc.com' + 'aws:referrer': 'http://abc.com/consoles', + 'aws:referrerDomain': 'abc.com' }); window.removeEventListener( @@ -542,8 +542,8 @@ test('when only domain level referrer is available from the DOM then is recorded }); expect(pageManager.getAttributes()).toMatchObject({ - referrer: 'http://abc.com', - referrerDomain: 'abc.com' + 'aws:referrer': 'http://abc.com', + 'aws:referrerDomain': 'abc.com' }); window.removeEventListener( @@ -573,8 +573,8 @@ test('when referrer from the DOM is empty then it is recorded as empty in the me }); expect(pageManager.getAttributes()).toMatchObject({ - referrer: '', - referrerDomain: '' + 'aws:referrer': '', + 'aws:referrerDomain': '' }); window.removeEventListener( From 4e283df22f638bafab7a443760f93198f272df8b Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Thu, 12 Jan 2023 04:28:00 -0800 Subject: [PATCH 07/12] handle localhost referrer --- src/sessions/PageManager.ts | 4 +++ src/sessions/__integ__/PageManager.test.ts | 1 - src/sessions/__tests__/PageManager.test.ts | 31 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index 3c3bbd2f..bc8562a1 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -251,6 +251,10 @@ export class PageManager { try { return new URL(document.referrer).hostname; } catch (e) { + if (document.referrer === 'localhost') { + // Handle special case for localhost + return 'localhost'; + } return ''; } } diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index ada30547..88e4e51e 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -1,7 +1,6 @@ import { Selector } from 'testcafe'; import { REQUEST_BODY } from '../../test-utils/integ-test-utils'; import { PAGE_VIEW_EVENT_TYPE } from '../../plugins/utils/constant'; -import { create } from 'lodash'; const recordPageView: Selector = Selector(`#recordPageView`); const recordPageViewWithPageTagAttribute: Selector = Selector( diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 7fc0b35f..49c45019 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -582,3 +582,34 @@ test('when referrer from the DOM is empty then it is recorded as empty in the me (pageManager as any).popstateListener ); }); + +test('when referrer is localhost then it is recorded in the metadata', async () => { + // Init + const config: Config = { + ...DEFAULT_CONFIG, + allowCookies: true + }; + const pageManager: PageManager = new PageManager(config, record); + + Object.defineProperty(document, 'referrer', { + value: 'localhost', + configurable: true + }); + // Run + pageManager.recordPageView('/console/home'); + + // Assert + expect(pageManager.getPage()).toMatchObject({ + pageId: '/console/home' + }); + + expect(pageManager.getAttributes()).toMatchObject({ + 'aws:referrer': 'localhost', + 'aws:referrerDomain': 'localhost' + }); + + window.removeEventListener( + 'popstate', + (pageManager as any).popstateListener + ); +}); From b251bde961881f5bc389cdfea77dc38c58e2318f Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Thu, 19 Jan 2023 09:59:20 -0800 Subject: [PATCH 08/12] Revert metadata changes --- src/event-schemas/meta-data.json | 5 +- src/event-schemas/page-view-event.json | 4 +- src/sessions/PageManager.ts | 28 +++++----- src/sessions/__integ__/PageManager.test.ts | 9 ++-- src/sessions/__tests__/PageManager.test.ts | 60 ++++------------------ 5 files changed, 35 insertions(+), 71 deletions(-) diff --git a/src/event-schemas/meta-data.json b/src/event-schemas/meta-data.json index 1ca89706..4708838d 100644 --- a/src/event-schemas/meta-data.json +++ b/src/event-schemas/meta-data.json @@ -45,10 +45,7 @@ "interaction": { "type": "number" }, - "aws:referrer": { - "type": "string" - }, - "aws:referrerDomain": { + "referrerUrl": { "type": "string" }, "pageTitle": { diff --git a/src/event-schemas/page-view-event.json b/src/event-schemas/page-view-event.json index 60bd02b8..41cad259 100644 --- a/src/event-schemas/page-view-event.json +++ b/src/event-schemas/page-view-event.json @@ -12,7 +12,9 @@ "pageId": { "type": "string" }, "pageInteractionId": { "type": "string" }, "interaction": { "type": "number" }, - "parentPageInteractionId": { "type": "string" } + "parentPageInteractionId": { "type": "string" }, + "referrer": { "type": "string" }, + "referrerDomain": { "type": "string" } }, "additionalProperties": false, "required": ["pageId"] diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index bc8562a1..d1a6c989 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -8,6 +8,8 @@ export type Page = { pageId: string; interaction: number; parentPageId?: string; + referrer?: string | null; + referrerDomain?: string | null; start: number; }; @@ -17,8 +19,6 @@ export type Attributes = { parentPageId?: string; interaction?: number; pageTags?: string[]; - 'aws:referrer'?: string; - 'aws:referrerDomain'?: string; // The value types of custom attributes are restricted to the types: string | number | boolean // However, given that pageTags is a string array, we need to include it as a valid type // Events will be verified by our service to validate attribute value types where @@ -131,6 +131,8 @@ export class PageManager { pageId, parentPageId: resumed.pageId, interaction: resumed.interaction + 1, + referrer: document.referrer, + referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; this.resumed = undefined; @@ -167,6 +169,8 @@ export class PageManager { pageId, parentPageId: currentPage.pageId, interaction: currentPage.interaction + 1, + referrer: document.referrer, + referrerDomain: this.getDomainFromReferrer(), start: startTime }; } @@ -175,6 +179,8 @@ export class PageManager { this.page = { pageId, interaction: 0, + referrer: document.referrer, + referrerDomain: this.getDomainFromReferrer(), start: Date.now() }; } @@ -193,12 +199,6 @@ export class PageManager { if (page.parentPageId !== undefined) { this.attributes.parentPageId = page.parentPageId; } - if (document.referrer !== undefined) { - this.attributes['aws:referrer'] = document.referrer; - this.attributes[ - 'aws:referrerDomain' - ] = this.getDomainFromReferrer(); - } } if (customPageAttributes?.pageTags) { @@ -228,6 +228,14 @@ export class PageManager { pageViewEvent.parentPageInteractionId = page.parentPageId + '-' + (page.interaction - 1); } + + if (page.referrer !== null) { + pageViewEvent.referrer = page.referrer; + + if (page.referrerDomain !== null) { + pageViewEvent.referrerDomain = page.referrerDomain; + } + } } return pageViewEvent; @@ -251,10 +259,6 @@ export class PageManager { try { return new URL(document.referrer).hostname; } catch (e) { - if (document.referrer === 'localhost') { - // Handle special case for localhost - return 'localhost'; - } return ''; } } diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index 88e4e51e..3565c96c 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -1,6 +1,7 @@ import { Selector } from 'testcafe'; import { REQUEST_BODY } from '../../test-utils/integ-test-utils'; import { PAGE_VIEW_EVENT_TYPE } from '../../plugins/utils/constant'; +import { create } from 'lodash'; const recordPageView: Selector = Selector(`#recordPageView`); const recordPageViewWithPageTagAttribute: Selector = Selector( @@ -208,7 +209,7 @@ test('when custom page attributes are set when manually recording page view even }); }); -test('when referrer exists, then metadata records it', async (t: TestController) => { +test('when referrer exists, then page view event details records it', async (t: TestController) => { // If we click too soon, the client/event collector plugin will not be loaded and will not record the click. // This could be a symptom of an issue with RUM web client load speed, or prioritization of script execution. @@ -229,10 +230,10 @@ test('when referrer exists, then metadata records it', async (t: TestController) const pages = requestBody.RumEvents.filter( (e) => e.type === PAGE_VIEW_EVENT_TYPE - ).map((e) => JSON.parse(e.metadata)); + ).map((e) => JSON.parse(e.details)); await t.expect(pages.length).eql(1).expect(pages[0]).contains({ - 'aws:referrer': 'http://amazon.com/searchresults/1/', - 'aws:referrerDomain': 'amazon.com' + referrer: 'http://amazon.com/searchresults/1/', + referrerDomain: 'amazon.com' }); }); diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 49c45019..37d077f0 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -489,7 +489,7 @@ describe('PageManager tests', () => { }); }); -test('when complete referrer is available from the DOM then is recorded in the metadata', async () => { +test('when complete referrer is available from the DOM then is recorded in page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -507,21 +507,18 @@ test('when complete referrer is available from the DOM then is recorded in the m // Assert expect(pageManager.getPage()).toMatchObject({ + referrer: 'http://abc.com/consoles', + referrerDomain: 'abc.com', pageId: '/console/home' }); - expect(pageManager.getAttributes()).toMatchObject({ - 'aws:referrer': 'http://abc.com/consoles', - 'aws:referrerDomain': 'abc.com' - }); - window.removeEventListener( 'popstate', (pageManager as any).popstateListener ); }); -test('when only domain level referrer is available from the DOM then is recorded in the metadata', async () => { +test('when only domain level referrer is available from the DOM then is recorded in page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -538,21 +535,18 @@ test('when only domain level referrer is available from the DOM then is recorded // Assert expect(pageManager.getPage()).toMatchObject({ + referrer: 'http://abc.com', + referrerDomain: 'abc.com', pageId: '/console/home' }); - expect(pageManager.getAttributes()).toMatchObject({ - 'aws:referrer': 'http://abc.com', - 'aws:referrerDomain': 'abc.com' - }); - window.removeEventListener( 'popstate', (pageManager as any).popstateListener ); }); -test('when referrer from the DOM is empty then it is recorded as empty in the metadata', async () => { +test('when referrer from the DOM is empty then it is recorded as empty in the page view event', async () => { // Init const config: Config = { ...DEFAULT_CONFIG, @@ -569,43 +563,9 @@ test('when referrer from the DOM is empty then it is recorded as empty in the me // Assert expect(pageManager.getPage()).toMatchObject({ - pageId: '/console/home' - }); - - expect(pageManager.getAttributes()).toMatchObject({ - 'aws:referrer': '', - 'aws:referrerDomain': '' - }); - - window.removeEventListener( - 'popstate', - (pageManager as any).popstateListener - ); -}); - -test('when referrer is localhost then it is recorded in the metadata', async () => { - // Init - const config: Config = { - ...DEFAULT_CONFIG, - allowCookies: true - }; - const pageManager: PageManager = new PageManager(config, record); - - Object.defineProperty(document, 'referrer', { - value: 'localhost', - configurable: true - }); - // Run - pageManager.recordPageView('/console/home'); - - // Assert - expect(pageManager.getPage()).toMatchObject({ - pageId: '/console/home' - }); - - expect(pageManager.getAttributes()).toMatchObject({ - 'aws:referrer': 'localhost', - 'aws:referrerDomain': 'localhost' + pageId: '/console/home', + referrer: '', + referrerDomain: '' }); window.removeEventListener( From e05dbd59d068319413db01d0d71af1dad74585dc Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Thu, 19 Jan 2023 11:38:54 -0800 Subject: [PATCH 09/12] Handle localhost --- src/sessions/PageManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index d1a6c989..63dc8194 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -259,6 +259,10 @@ export class PageManager { try { return new URL(document.referrer).hostname; } catch (e) { + if (document.referrer === 'localhost') { + // Handle special case for localhost + return 'localhost'; + } return ''; } } From f2f5fc225927687d9c69449720a213dda8c6d4bb Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 25 Jan 2023 01:39:30 -0800 Subject: [PATCH 10/12] Address comments --- src/sessions/PageManager.ts | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/sessions/PageManager.ts b/src/sessions/PageManager.ts index 63dc8194..b7ec6efc 100644 --- a/src/sessions/PageManager.ts +++ b/src/sessions/PageManager.ts @@ -229,13 +229,8 @@ export class PageManager { page.parentPageId + '-' + (page.interaction - 1); } - if (page.referrer !== null) { - pageViewEvent.referrer = page.referrer; - - if (page.referrerDomain !== null) { - pageViewEvent.referrerDomain = page.referrerDomain; - } - } + pageViewEvent.referrer = document.referrer; + pageViewEvent.referrerDomain = this.getDomainFromReferrer(); } return pageViewEvent; @@ -259,11 +254,7 @@ export class PageManager { try { return new URL(document.referrer).hostname; } catch (e) { - if (document.referrer === 'localhost') { - // Handle special case for localhost - return 'localhost'; - } - return ''; + return document.referrer === 'localhost' ? document.referrer : ''; } } } From 72bee17e21606ffb452e40f6902de3c0c3320816 Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Wed, 25 Jan 2023 21:02:32 -0800 Subject: [PATCH 11/12] remove unneeded import --- src/sessions/__integ__/PageManager.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sessions/__integ__/PageManager.test.ts b/src/sessions/__integ__/PageManager.test.ts index 3565c96c..944c45fc 100644 --- a/src/sessions/__integ__/PageManager.test.ts +++ b/src/sessions/__integ__/PageManager.test.ts @@ -1,7 +1,6 @@ import { Selector } from 'testcafe'; import { REQUEST_BODY } from '../../test-utils/integ-test-utils'; import { PAGE_VIEW_EVENT_TYPE } from '../../plugins/utils/constant'; -import { create } from 'lodash'; const recordPageView: Selector = Selector(`#recordPageView`); const recordPageViewWithPageTagAttribute: Selector = Selector( From 9a52c10a27832751f0bf7fafba2cb0b53f4fab83 Mon Sep 17 00:00:00 2001 From: Paras Sakharkar Date: Thu, 26 Jan 2023 11:27:33 -0800 Subject: [PATCH 12/12] Adding localhost unit test for referrer --- src/sessions/__tests__/PageManager.test.ts | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/sessions/__tests__/PageManager.test.ts b/src/sessions/__tests__/PageManager.test.ts index 37d077f0..178c2ad4 100644 --- a/src/sessions/__tests__/PageManager.test.ts +++ b/src/sessions/__tests__/PageManager.test.ts @@ -573,3 +573,31 @@ test('when referrer from the DOM is empty then it is recorded as empty in the pa (pageManager as any).popstateListener ); }); + +test('when referrer from the DOM is localhost then referrerDomain is also recorded as localhost', async () => { + // Init + const config: Config = { + ...DEFAULT_CONFIG, + allowCookies: true + }; + const pageManager: PageManager = new PageManager(config, record); + + Object.defineProperty(document, 'referrer', { + value: 'localhost', + configurable: true + }); + // Run + pageManager.recordPageView('/console/home'); + + // Assert + expect(pageManager.getPage()).toMatchObject({ + pageId: '/console/home', + referrer: 'localhost', + referrerDomain: 'localhost' + }); + + window.removeEventListener( + 'popstate', + (pageManager as any).popstateListener + ); +});