Merge pull request #14694 from Budibase/view-merge-v3-ui

View changes - merge to V3 UI branch

packages/builder/src/stores/builder/viewsV2.js

@@ -1,6 +1,24 @@
 import { writable, derived, get } from "svelte/store"
 import { tables } from "./tables"
 import { API } from "api"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
+
+function convertToSearchFilterGroup(view) {
+  if (view.queryUI) {
+    view.query = view.queryUI
+    delete view.queryUI
+  }
+  return view
+}
 
 export function createViewsV2Store() {
   const store = writable({
@@ -12,7 +30,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views)
+      list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
     })
     return {
       ...$store,
@@ -34,13 +52,15 @@ export function createViewsV2Store() {
   }
 
   const create = async view => {
+    view = convertToSearchFilters(view)
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
     replaceView(savedView.id, savedView)
     return savedView
   }
 
   const save = async view => {
+    view = convertToSearchFilters(view)
     const res = await API.viewV2.update(view)
     const savedView = res?.data
     replaceView(view.id, savedView)
@@ -51,6 +71,7 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
+    view = convertToSearchFilterGroup(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,12 +1,22 @@
 import { get } from "svelte/store"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
 
 const SuppressErrors = true
 
 export const createActions = context => {
   const { API, datasource, columns } = context
 
   const saveDefinition = async newDefinition => {
-    await API.viewV2.update(newDefinition)
+    await API.viewV2.update(convertToSearchFilters(newDefinition))
   }
 
   const saveRow = async row => {
@@ -125,7 +135,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          filter.set($definition.query)
+          filter.set($definition.queryUI || $definition.query)
         }
       })
     )

packages/server/src/api/controllers/row/views.ts

@@ -3,6 +3,8 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
+  RequiredKeys,
+  RowSearchParams,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
@@ -20,30 +22,34 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
+  const viewFields = Object.entries(view.schema || {})
+    .filter(([_, value]) => value.visible)
+    .map(([key]) => key)
   const { body } = ctx.request
 
   await context.ensureSnippetContext(true)
 
-  const result = await sdk.rows.search(
-    {
-      viewId: view.id,
-      tableId: view.tableId,
-      query: body.query,
-      ...getSortOptions(body, view),
-      limit: body.limit,
-      bookmark: body.bookmark,
-      paginate: body.paginate,
-      countRows: body.countRows,
-    },
-    {
-      user: sdk.users.getUserContextBindings(ctx.user),
-    }
-  )
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
+    RequiredKeys<
+      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
+    > = {
+    tableId: view.tableId,
+    viewId: view.id,
+    query: body.query,
+    fields: viewFields,
+    ...getSortOptions(body, view),
+    limit: body.limit,
+    bookmark: body.bookmark,
+    paginate: body.paginate,
+    countRows: body.countRows,
+  }
 
+  const result = await sdk.rows.search(searchOptions, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
-
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
   if (request.sort) {
     return {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -103,6 +103,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
+    queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
@@ -138,6 +139,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     version: view.version,
     tableId: view.tableId,
     query: view.query,
+    queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -155,7 +155,7 @@ describe.each([
       })
 
       it("can persist views with all fields", async () => {
-        const newView: Required<CreateViewRequest> = {
+        const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
@@ -612,7 +612,7 @@ describe.each([
       it("can update all fields", async () => {
         const tableId = table._id!
 
-        const updatedData: Required<UpdateViewRequest> = {
+        const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
           version: view.version,
           id: view.id,
           tableId,

packages/server/src/sdk/app/rows/search.ts

@@ -1,12 +1,10 @@
 import {
   EmptyFilterOption,
+  LegacyFilter,
   LogicalOperator,
   Row,
   RowSearchParams,
-  SearchFilter,
-  SearchFilterGroup,
   SearchFilterKey,
-  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -63,7 +61,6 @@ export async function search(
     if (options.viewId) {
       source = await sdk.views.get(options.viewId)
       table = await sdk.views.getTable(source)
-      options = searchInputMapping(table, options)
     } else if (options.tableId) {
       source = await sdk.tables.getTable(options.tableId)
       table = source
@@ -76,48 +73,62 @@ export async function search(
     if (options.query) {
       const visibleFields = (
         options.fields || Object.keys(table.schema)
-      ).filter(field => table.schema[field].visible !== false)
+      ).filter(field => table.schema[field]?.visible !== false)
 
       const queryableFields = await getQueryableFields(table, visibleFields)
       options.query = removeInvalidFilters(options.query, queryableFields)
     } else {
       options.query = {}
     }
 
+    // need to make sure filters in correct shape before checking for view
+    options = searchInputMapping(table, options)
+
     if (options.viewId) {
-      const view = await sdk.views.get(options.viewId)
+      // Delete extraneous search params that cannot be overridden
+      delete options.query.onEmptyFilter
+
+      const view = source as ViewV2
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQuery(view.query || [])
-
-      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
-        // Lucene does not accept conditional filters, so we need to keep the old logic
-        const query: SearchFilters = viewQuery || {}
-        const viewFilters = view.query as SearchFilter[]
+      let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
+      delete viewQuery?.onEmptyFilter
+
+      const sqsEnabled = await features.flags.isEnabled("SQS")
+      const supportsLogicalOperators =
+        isExternalTableID(view.tableId) || sqsEnabled
+      if (!supportsLogicalOperators) {
+        // In the unlikely event that a Grouped Filter is in a non-SQS environment
+        // It needs to be ignored entirely
+        let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+          ? view.query
+          : []
 
         delete options.query.onEmptyFilter
 
         // Extract existing fields
         const existingFields =
-          viewFilters
+          queryFilters
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
+        viewQuery ??= {}
         // Carry over filters for unused fields
-        Object.keys(options.query || {}).forEach(key => {
+        Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              query[operator]![field] = options.query[operator]![field]
+              viewQuery![operator]![field] = options.query[operator]![field]
             }
           })
         })
-        options.query = query
+        options.query = viewQuery
       } else {
+        const conditions = viewQuery ? [viewQuery] : []
         options.query = {
           $and: {
-            conditions: [viewQuery as SearchFilterGroup, options.query],
+            conditions: [...conditions, options.query],
           },
         }
         if (viewQuery.onEmptyFilter) {
@@ -151,8 +162,6 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    options = searchInputMapping(table, options)
-
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/utils.ts

@@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
     }
     return dataFilters.recurseLogicalOperators(filters, checkFilters)
   }
-  options.query = checkFilters(options.query)
+  if (options.query) {
+    options.query = checkFilters(options.query)
+  }
   return options
 }
 

packages/shared-core/src/filters.ts

@@ -3,7 +3,7 @@ import {
   BBReferenceFieldSubType,
   FieldType,
   FormulaType,
-  SearchFilter,
+  LegacyFilter,
   SearchFilters,
   SearchQueryFields,
   ArrayOperator,
@@ -127,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters?.[logical]) {
+    if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )
@@ -163,9 +163,6 @@ export function recurseSearchFilters(
  * https://github.com/Budibase/budibase/issues/10118
  */
 export const cleanupQuery = (query: SearchFilters) => {
-  if (!query) {
-    return query
-  }
   for (let filterField of NoEmptyFilterStrings) {
     if (!query[filterField]) {
       continue
@@ -311,7 +308,7 @@ export class ColumnSplitter {
  * @param filter the builder filter structure
  */
 
-const buildCondition = (expression: SearchFilter) => {
+const buildCondition = (expression: LegacyFilter) => {
   // Filter body
   let query: SearchFilters = {
     string: {},
@@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => {
 }
 
 export const buildQueryLegacy = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: LegacyFilter[] | SearchFilters
 ): SearchFilters | undefined => {
+  // this is of type SearchFilters or is undefined
+  if (!Array.isArray(filter)) {
+    return filter
+  }
+
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -572,7 +574,7 @@ export const buildQueryLegacy = (
  */
 
 export const buildQuery = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: SearchFilterGroup | LegacyFilter[]
 ): SearchFilters | undefined => {
   const parsedFilter: SearchFilterGroup | undefined =
     processSearchFilters(filter)
@@ -594,7 +596,7 @@ export const buildQuery = (
   const globalOperator: LogicalOperator =
     operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
 
-  const coreRequest: SearchFilters = {
+  return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
       conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@@ -608,7 +610,6 @@ export const buildQuery = (
       }),
     },
   }
-  return coreRequest
 }
 
 // The frontend can send single values for array fields sometimes, so to handle