From 384a83a4bff1fd8abdebb2c6825cba7e3d03b36f Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 22 Apr 2024 16:50:19 +0200 Subject: [PATCH 1/5] fix: improved elasticsearch query --- .../configuration/datasetFieldMapping.ts | 6 ++++++ src/elastic-search/providers/fields.enum.ts | 16 ++++++++-------- .../providers/query-builder.service.ts | 17 ++++++++--------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 24c78ec15..58ffd702e 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -49,6 +49,9 @@ export const datasetMappings: MappingObject = { proposalId: { type: "keyword", }, + sampleId: { + type: "keyword", + }, sourceFolder: { type: "keyword", }, @@ -73,4 +76,7 @@ export const datasetMappings: MappingObject = { sharedWith: { type: "keyword", }, + ownerEmail: { + type: "keyword", + }, }; diff --git a/src/elastic-search/providers/fields.enum.ts b/src/elastic-search/providers/fields.enum.ts index 13718e921..e33715dbc 100644 --- a/src/elastic-search/providers/fields.enum.ts +++ b/src/elastic-search/providers/fields.enum.ts @@ -12,14 +12,6 @@ export enum FilterFields { Mode = "mode", } -export enum FacetFields { - Type = "type", - CreationLocation = "creationLocation", - OwnerGroup = "ownerGroup", - AccessGroups = "accessGroups", - Keywords = "keywords", -} - export enum QueryFields { DatasetName = "datasetName", Description = "description", @@ -30,6 +22,14 @@ export enum ShouldFields { UserGroups = "userGroups", } +export enum FacetFields { + Type = "type", + CreationLocation = "creationLocation", + OwnerGroup = "ownerGroup", + AccessGroups = "accessGroups", + Keywords = "keywords", +} + export enum SortFields { DatasetName = "datasetName", DatasetNameKeyword = "datasetName.keyword", diff --git a/src/elastic-search/providers/query-builder.service.ts b/src/elastic-search/providers/query-builder.service.ts index 08070f3e1..5879d30f3 100644 --- a/src/elastic-search/providers/query-builder.service.ts +++ b/src/elastic-search/providers/query-builder.service.ts @@ -44,15 +44,14 @@ export class SearchQueryService { private buildFilterFields(fields: Partial): IFilter[] { const filter: IFilter[] = []; - for (const fieldName of this.filterFields) { - if (fields[fieldName]) { - const filterQueries = this.buildTermsFilter( - fieldName, - fields[fieldName], - ); - filter.push(...filterQueries); + Object.entries(fields).forEach(([key, value]) => { + if (this.shouldFields.includes(key as ShouldFields) || key === "text") { + return; } - } + + const filterQueries = this.buildTermsFilter(key, value); + filter.push(...filterQueries); + }); return filter; } @@ -173,7 +172,7 @@ export class SearchQueryService { if (typeof values === "string") { filterArray.push({ match: { - [`${fieldName}.keyword`]: values as string, + [fieldName]: values as string, }, }); } From cc956635cbd9d572b638e8fc13213a68d0d8319a Mon Sep 17 00:00:00 2001 From: junjiequan Date: Wed, 24 Apr 2024 15:20:54 +0200 Subject: [PATCH 2/5] In elasticsearch datasetMapping ,ignore_above 256 option for keywords type is added --- .../configuration/datasetFieldMapping.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 58ffd702e..054b2f57d 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -25,9 +25,11 @@ export const datasetMappings: MappingObject = { }, pid: { type: "keyword", + ignore_above: 256, }, creationTime: { type: "date", + ignore_above: 256, }, scientificMetadata: { type: "nested", @@ -45,38 +47,49 @@ export const datasetMappings: MappingObject = { history: { type: "nested", dynamic: false, + ignore_above: 256, }, proposalId: { type: "keyword", + ignore_above: 256, }, sampleId: { type: "keyword", + ignore_above: 256, }, sourceFolder: { type: "keyword", + ignore_above: 256, }, isPublished: { type: "boolean", }, type: { type: "keyword", + ignore_above: 256, }, keywords: { type: "keyword", + ignore_above: 256, }, creationLocation: { type: "keyword", + ignore_above: 256, }, ownerGroup: { type: "keyword", + ignore_above: 256, }, accessGroups: { type: "keyword", + ignore_above: 256, }, sharedWith: { type: "keyword", + ignore_above: 256, }, ownerEmail: { type: "keyword", + ignore_above: 256, }, }; From 9b3472edf96bdfcf401d2088375febd28669ccc4 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Fri, 26 Apr 2024 16:46:29 +0200 Subject: [PATCH 3/5] minor fix --- src/elastic-search/configuration/datasetFieldMapping.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 054b2f57d..3bb571441 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -29,7 +29,6 @@ export const datasetMappings: MappingObject = { }, creationTime: { type: "date", - ignore_above: 256, }, scientificMetadata: { type: "nested", From c4daf420fb765b3458f9453b4d90e361cd26bd84 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 16 May 2024 11:15:32 +0200 Subject: [PATCH 4/5] fix e2e test fail --- src/elastic-search/configuration/datasetFieldMapping.ts | 1 - src/elastic-search/elastic-search.service.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 3bb571441..e28d8b983 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -46,7 +46,6 @@ export const datasetMappings: MappingObject = { history: { type: "nested", dynamic: false, - ignore_above: 256, }, proposalId: { type: "keyword", diff --git a/src/elastic-search/elastic-search.service.ts b/src/elastic-search/elastic-search.service.ts index 59d9b4244..fc3dca187 100644 --- a/src/elastic-search/elastic-search.service.ts +++ b/src/elastic-search/elastic-search.service.ts @@ -428,7 +428,7 @@ export class ElasticSearchService implements OnModuleInit { ]; }, onDrop(doc) { - console.debug(`${doc.document._id}`, doc.error?.reason); + console.debug(`${doc.document.pid}`, doc.error?.reason); }, }); } From 20844f1cc68c0128e8a9ece51f2f7dec77788f2b Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 16 May 2024 13:22:37 +0200 Subject: [PATCH 5/5] elasticsearch variable naming improvement --- src/elastic-search/providers/fields.enum.ts | 2 +- .../providers/query-builder.service.ts | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/elastic-search/providers/fields.enum.ts b/src/elastic-search/providers/fields.enum.ts index e33715dbc..8030407ab 100644 --- a/src/elastic-search/providers/fields.enum.ts +++ b/src/elastic-search/providers/fields.enum.ts @@ -12,7 +12,7 @@ export enum FilterFields { Mode = "mode", } -export enum QueryFields { +export enum MustFields { DatasetName = "datasetName", Description = "description", } diff --git a/src/elastic-search/providers/query-builder.service.ts b/src/elastic-search/providers/query-builder.service.ts index 5879d30f3..4813479e6 100644 --- a/src/elastic-search/providers/query-builder.service.ts +++ b/src/elastic-search/providers/query-builder.service.ts @@ -10,7 +10,7 @@ import { } from "../interfaces/es-common.type"; import { FilterFields, - QueryFields, + MustFields, FacetFields, ShouldFields, } from "./fields.enum"; @@ -22,7 +22,7 @@ import { convertToElasticSearchQuery } from "../helpers/utils"; @Injectable() export class SearchQueryService { readonly filterFields = [...Object.values(FilterFields)]; - readonly queryFields = [...Object.values(QueryFields)]; + readonly mustFields = [...Object.values(MustFields)]; readonly shouldFields = [...Object.values(ShouldFields)]; readonly facetFields = [...Object.values(FacetFields)]; readonly textQuerySplitMethod = /[ ,]+/; @@ -33,9 +33,13 @@ export class SearchQueryService { const filter = this.buildFilterFields(fields); const should = this.buildShouldFields(fields); - const query = this.buildTextQuery(fields); + const must = this.buildTextQuery(fields); - return this.constructFinalQuery(filter, should, query); + // NOTE: The final query flow is as follows: + // step 1. Build filter fields conditions must match all filter fields + // step 2. Build should fields conditions must match at least one should field + // step 3. Build text query conditions must match all text query fields + return this.constructFinalQuery(filter, should, must); } catch (err) { Logger.error("Elastic search build search query failed"); throw err; @@ -100,7 +104,7 @@ export class SearchQueryService { private buildWildcardQueries(text: string): QueryDslQueryContainer[] { const terms = this.splitSearchText(text); return terms.flatMap((term) => - this.queryFields.map((fieldName) => ({ + this.mustFields.map((fieldName) => ({ wildcard: { [fieldName]: { value: `*${term}*` } }, })), );