From 913cefaf175d0c112c4132c973ce726b7b165648 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 31 Dec 2024 13:24:26 +0100 Subject: [PATCH 01/63] Type index --- .../src/fetch/{index.js => index.ts} | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) rename packages/frontend-core/src/fetch/{index.js => index.ts} (67%) diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.ts similarity index 67% rename from packages/frontend-core/src/fetch/index.js rename to packages/frontend-core/src/fetch/index.ts index 903810ac25b..d9eac9482e2 100644 --- a/packages/frontend-core/src/fetch/index.js +++ b/packages/frontend-core/src/fetch/index.ts @@ -10,6 +10,7 @@ import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch.js" import CustomFetch from "./CustomFetch.js" import QueryArrayFetch from "./QueryArrayFetch.js" +import { UIDatasource, UIFetchAPI } from "@budibase/types" const DataFetchMap = { table: TableFetch, @@ -29,15 +30,30 @@ const DataFetchMap = { } // Constructs a new fetch model for a certain datasource -export const fetchData = ({ API, datasource, options }) => { - const Fetch = DataFetchMap[datasource?.type] || TableFetch +export const fetchData = ({ + API, + datasource, + options, +}: { + API: UIFetchAPI + datasource: UIDatasource + options: {} +}) => { + const Fetch = + DataFetchMap[datasource?.type as keyof typeof DataFetchMap] || TableFetch return new Fetch({ API, datasource, ...options }) } // Creates an empty fetch instance with no datasource configured, so no data // will initially be loaded -const createEmptyFetchInstance = ({ API, datasource }) => { - const handler = DataFetchMap[datasource?.type] +const createEmptyFetchInstance = ({ + API, + datasource, +}: { + API: UIFetchAPI + datasource: UIDatasource +}) => { + const handler = DataFetchMap[datasource?.type as keyof typeof DataFetchMap] if (!handler) { return null } @@ -45,13 +61,27 @@ const createEmptyFetchInstance = ({ API, datasource }) => { } // Fetches the definition of any type of datasource -export const getDatasourceDefinition = async ({ API, datasource }) => { +export const getDatasourceDefinition = async ({ + API, + datasource, +}: { + API: UIFetchAPI + datasource: UIDatasource +}) => { const instance = createEmptyFetchInstance({ API, datasource }) return await instance?.getDefinition(datasource) } // Fetches the schema of any type of datasource -export const getDatasourceSchema = ({ API, datasource, definition }) => { +export const getDatasourceSchema = ({ + API, + datasource, + definition, +}: { + API: UIFetchAPI + datasource: UIDatasource + definition: {} +}) => { const instance = createEmptyFetchInstance({ API, datasource }) return instance?.getSchema(datasource, definition) } From ecfc248e606f85ddd3a60e462e07427003216855 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 31 Dec 2024 15:43:08 +0100 Subject: [PATCH 02/63] Type datafetch --- .../src/fetch/{DataFetch.js => DataFetch.ts} | 178 ++++++++++++------ packages/frontend-core/src/fetch/index.ts | 4 +- packages/shared-core/src/filters.ts | 8 +- packages/types/src/ui/stores/grid/fetch.ts | 2 + 4 files changed, 128 insertions(+), 64 deletions(-) rename packages/frontend-core/src/fetch/{DataFetch.js => DataFetch.ts} (76%) diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.ts similarity index 76% rename from packages/frontend-core/src/fetch/DataFetch.js rename to packages/frontend-core/src/fetch/DataFetch.ts index 175365a442c..d8a8e152458 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -1,25 +1,86 @@ -import { writable, derived, get } from "svelte/store" +import { writable, derived, get, Writable, Readable } from "svelte/store" import { cloneDeep } from "lodash/fp" import { QueryUtils } from "../utils" import { convertJSONSchemaToTableSchema } from "../utils/json" -import { FieldType, SortOrder, SortType } from "@budibase/types" +import { + FieldType, + LegacyFilter, + SearchFilters, + SortOrder, + SortType, + Table, + TableSchema, + UIDatasource, + UIFetchAPI, + UIRow, + UISearchFilter, +} from "@budibase/types" const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils +interface DataFetchStore { + rows: UIRow[] + info: null + schema: TableSchema | null + loading: boolean + loaded: boolean + query: SearchFilters | null + pageNumber: number + cursor: null + cursors: any[] + resetKey: number + error: null +} + +interface DataFetchDerivedStore extends DataFetchStore { + hasNextPage: boolean + hasPrevPage: boolean + supportsSearch: boolean + supportsSort: boolean + supportsPagination: boolean +} + /** * Parent class which handles the implementation of fetching data from an * internal table or datasource plus. * For other types of datasource, this class is overridden and extended. */ -export default class DataFetch { +export default abstract class DataFetch { + API: UIFetchAPI + features: { + supportsSearch: boolean + supportsSort: boolean + supportsPagination: boolean + } + options: { + datasource: UIDatasource | null + limit: number + // Search config + filter: UISearchFilter | LegacyFilter[] | null + query: SearchFilters | null + // Sorting config + sortColumn: string | null + sortOrder: SortOrder + sortType: SortType | null + // Pagination config + paginate: boolean + // Client side feature customisation + clientSideSearching: boolean + clientSideSorting: boolean + clientSideLimiting: boolean + } + store: Writable + derivedStore: Readable + /** * Constructs a new DataFetch instance. * @param opts the fetch options */ - constructor(opts) { - // API client - this.API = null - + constructor(opts: { + API: UIFetchAPI + datasource?: UIDatasource + options?: {} + }) { // Feature flags this.features = { supportsSearch: false, @@ -118,7 +179,10 @@ export default class DataFetch { /** * Gets the default sort column for this datasource */ - getDefaultSortColumn(definition, schema) { + getDefaultSortColumn( + definition: { primaryDisplay?: string } | null, + schema: Record + ) { if (definition?.primaryDisplay && schema[definition.primaryDisplay]) { return definition.primaryDisplay } else { @@ -144,7 +208,7 @@ export default class DataFetch { } // Fetch and enrich schema - let schema = this.getSchema(datasource, definition) + let schema = this.getSchema(datasource, definition) ?? null schema = this.enrichSchema(schema) if (!schema) { return @@ -172,7 +236,7 @@ export default class DataFetch { if ( fieldSchema?.type === FieldType.NUMBER || fieldSchema?.type === FieldType.BIGINT || - fieldSchema?.calculationType + ("calculationType" in fieldSchema && fieldSchema?.calculationType) ) { this.options.sortType = SortType.NUMBER } @@ -185,7 +249,7 @@ export default class DataFetch { // Build the query let query = this.options.query if (!query) { - query = buildQuery(filter) + query = buildQuery(filter ?? undefined) } // Update store @@ -239,7 +303,7 @@ export default class DataFetch { // If we don't support sorting, do a client-side sort if (!this.features.supportsSort && clientSideSorting) { - rows = sort(rows, sortColumn, sortOrder, sortType) + rows = sort(rows, sortColumn as any, sortOrder, sortType) } // If we don't support pagination, do a client-side limit @@ -256,18 +320,13 @@ export default class DataFetch { } } - /** - * Fetches a single page of data from the remote resource. - * Must be overridden by a datasource specific child class. - */ - async getData() { - return { - rows: [], - info: null, - hasNextPage: false, - cursor: null, - } - } + abstract getData(): Promise<{ + rows: UIRow[] + info: any + hasNextPage: boolean + cursor: any + error: any + }> /** * Gets the definition for this datasource. @@ -275,13 +334,13 @@ export default class DataFetch { * @param datasource * @return {object} the definition */ - async getDefinition(datasource) { + async getDefinition(datasource: UIDatasource | null) { if (!datasource?.tableId) { return null } try { return await this.API.fetchTableDefinition(datasource.tableId) - } catch (error) { + } catch (error: any) { this.store.update(state => ({ ...state, error, @@ -293,11 +352,11 @@ export default class DataFetch { /** * Gets the schema definition for a datasource. * Defaults to getting the "schema" property of the definition. - * @param datasource the datasource + * @param _datasource the datasource * @param definition the datasource definition * @return {object} the schema */ - getSchema(datasource, definition) { + getSchema(_datasource: UIDatasource | null, definition: Table | null) { return definition?.schema } @@ -307,44 +366,48 @@ export default class DataFetch { * @param schema the datasource schema * @return {object} the enriched datasource schema */ - enrichSchema(schema) { + enrichSchema(schema: TableSchema | null): TableSchema | null { if (schema == null) { return null } // Check for any JSON fields so we can add any top level properties - let jsonAdditions = {} - Object.keys(schema).forEach(fieldKey => { + let jsonAdditions: Record = {} + for (const fieldKey of Object.keys(schema)) { const fieldSchema = schema[fieldKey] if (fieldSchema?.type === FieldType.JSON) { const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, { squashObjects: true, - }) - Object.keys(jsonSchema).forEach(jsonKey => { - jsonAdditions[`${fieldKey}.${jsonKey}`] = { - type: jsonSchema[jsonKey].type, - nestedJSON: true, + }) as Record | null // TODO: remove when convertJSONSchemaToTableSchema is typed + if (jsonSchema) { + for (const jsonKey of Object.keys(jsonSchema)) { + jsonAdditions[`${fieldKey}.${jsonKey}`] = { + type: jsonSchema[jsonKey].type, + nestedJSON: true, + } } - }) + } } - }) - schema = { ...schema, ...jsonAdditions } + } // Ensure schema is in the correct structure - let enrichedSchema = {} - Object.entries(schema).forEach(([fieldName, fieldSchema]) => { - if (typeof fieldSchema === "string") { - enrichedSchema[fieldName] = { - type: fieldSchema, - name: fieldName, - } - } else { - enrichedSchema[fieldName] = { - ...fieldSchema, - name: fieldName, + let enrichedSchema: TableSchema = {} + Object.entries({ ...schema, ...jsonAdditions }).forEach( + ([fieldName, fieldSchema]) => { + if (typeof fieldSchema === "string") { + enrichedSchema[fieldName] = { + type: fieldSchema, + name: fieldName, + } + } else { + enrichedSchema[fieldName] = { + ...fieldSchema, + type: fieldSchema.type as any, // TODO: check type union definition conflicts + name: fieldName, + } } } - }) + ) return enrichedSchema } @@ -353,7 +416,7 @@ export default class DataFetch { * Determine the feature flag for this datasource definition * @param definition */ - determineFeatureFlags(_definition) { + determineFeatureFlags(_definition: Table | null) { return { supportsSearch: false, supportsSort: false, @@ -365,12 +428,11 @@ export default class DataFetch { * Resets the data set and updates options * @param newOptions any new options */ - async update(newOptions) { + async update(newOptions: any) { // Check if any settings have actually changed let refresh = false - const entries = Object.entries(newOptions || {}) - for (let [key, value] of entries) { - const oldVal = this.options[key] == null ? null : this.options[key] + for (const [key, value] of Object.entries(newOptions || {})) { + const oldVal = this.options[key as keyof typeof this.options] ?? null const newVal = value == null ? null : value if (JSON.stringify(newVal) !== JSON.stringify(oldVal)) { refresh = true @@ -437,7 +499,7 @@ export default class DataFetch { * @param state the current store state * @return {boolean} whether there is a next page of data or not */ - hasNextPage(state) { + hasNextPage(state: DataFetchStore): boolean { return state.cursors[state.pageNumber + 1] != null } @@ -447,7 +509,7 @@ export default class DataFetch { * @param state the current store state * @return {boolean} whether there is a previous page of data or not */ - hasPrevPage(state) { + hasPrevPage(state: { pageNumber: number }): boolean { return state.pageNumber > 0 } diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index d9eac9482e2..a08748a77ea 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -10,7 +10,7 @@ import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch.js" import CustomFetch from "./CustomFetch.js" import QueryArrayFetch from "./QueryArrayFetch.js" -import { UIDatasource, UIFetchAPI } from "@budibase/types" +import { Table, UIDatasource, UIFetchAPI } from "@budibase/types" const DataFetchMap = { table: TableFetch, @@ -80,7 +80,7 @@ export const getDatasourceSchema = ({ }: { API: UIFetchAPI datasource: UIDatasource - definition: {} + definition: Table }) => { const instance = createEmptyFetchInstance({ API, datasource }) return instance?.getSchema(datasource, definition) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index a023015b7e9..a1e8534a95e 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -552,7 +552,7 @@ export function search>( */ export function runQuery>( docs: T[], - query: SearchFilters + query: SearchFilters | null ): T[] { if (!docs || !Array.isArray(docs)) { return [] @@ -876,7 +876,7 @@ export function sort>( docs: T[], sort: keyof T, sortOrder: SortOrder, - sortType = SortType.STRING + sortType: SortType | null = SortType.STRING ): T[] { if (!sort || !sortOrder || !sortType) { return docs @@ -911,8 +911,8 @@ export function sort>( * @param docs the data * @param limit the number of docs to limit to */ -export function limit(docs: T[], limit: string): T[] { - const numLimit = parseFloat(limit) +export function limit(docs: T[], limit: string | number): T[] { + const numLimit = typeof limit === "number" ? limit : parseFloat(limit) if (isNaN(numLimit)) { return docs } diff --git a/packages/types/src/ui/stores/grid/fetch.ts b/packages/types/src/ui/stores/grid/fetch.ts index 8901acc08b3..0be9ca17b4a 100644 --- a/packages/types/src/ui/stores/grid/fetch.ts +++ b/packages/types/src/ui/stores/grid/fetch.ts @@ -1,12 +1,14 @@ import { Row, SortOrder, + Table, UIDatasource, UILegacyFilter, UISearchFilter, } from "@budibase/types" export interface UIFetchAPI { + fetchTableDefinition(tableId: string): Promise definition: UIDatasource getInitialData: () => Promise From d7cfd51caf50ba6603a711e8a0295e1ee3f7417b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 10:23:39 +0100 Subject: [PATCH 03/63] Type tablefetch --- packages/frontend-core/src/fetch/DataFetch.ts | 6 +++--- .../src/fetch/{TableFetch.js => TableFetch.ts} | 2 +- packages/types/src/ui/stores/grid/fetch.ts | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) rename packages/frontend-core/src/fetch/{TableFetch.js => TableFetch.ts} (96%) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index d8a8e152458..67ed07a835f 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -322,10 +322,10 @@ export default abstract class DataFetch { abstract getData(): Promise<{ rows: UIRow[] - info: any + info?: any hasNextPage: boolean - cursor: any - error: any + cursor?: any + error?: any }> /** diff --git a/packages/frontend-core/src/fetch/TableFetch.js b/packages/frontend-core/src/fetch/TableFetch.ts similarity index 96% rename from packages/frontend-core/src/fetch/TableFetch.js rename to packages/frontend-core/src/fetch/TableFetch.ts index 777d16aa458..0615f83dc25 100644 --- a/packages/frontend-core/src/fetch/TableFetch.js +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -14,7 +14,7 @@ export default class TableFetch extends DataFetch { async getData() { const { datasource, limit, sortColumn, sortOrder, sortType, paginate } = this.options - const { tableId } = datasource + const { tableId } = datasource! const { cursor, query } = get(this.store) // Search table diff --git a/packages/types/src/ui/stores/grid/fetch.ts b/packages/types/src/ui/stores/grid/fetch.ts index 0be9ca17b4a..772e68eb14e 100644 --- a/packages/types/src/ui/stores/grid/fetch.ts +++ b/packages/types/src/ui/stores/grid/fetch.ts @@ -1,6 +1,8 @@ import { Row, + SearchFilters, SortOrder, + SortType, Table, UIDatasource, UILegacyFilter, @@ -15,6 +17,19 @@ export interface UIFetchAPI { loading: any loaded: boolean + searchTable( + tableId: string, + arg1: { + query: SearchFilters | null + limit: number + sort: string | null + sortOrder: string + sortType: SortType | null + paginate: boolean + bookmark: null + } + ): any + resetKey: string | null error: any From 89eba3189703454ed1055251713b98629cb53a14 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 12:16:53 +0100 Subject: [PATCH 04/63] Use tempaltes --- packages/frontend-core/src/fetch/DataFetch.ts | 14 ++++------- .../frontend-core/src/fetch/TableFetch.ts | 6 ++--- packages/types/src/ui/stores/grid/fetch.ts | 23 +++++++++---------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 67ed07a835f..7355afd967f 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -45,7 +45,7 @@ interface DataFetchDerivedStore extends DataFetchStore { * internal table or datasource plus. * For other types of datasource, this class is overridden and extended. */ -export default abstract class DataFetch { +export default abstract class DataFetch { API: UIFetchAPI features: { supportsSearch: boolean @@ -53,7 +53,7 @@ export default abstract class DataFetch { supportsPagination: boolean } options: { - datasource: UIDatasource | null + datasource: T limit: number // Search config filter: UISearchFilter | LegacyFilter[] | null @@ -76,11 +76,7 @@ export default abstract class DataFetch { * Constructs a new DataFetch instance. * @param opts the fetch options */ - constructor(opts: { - API: UIFetchAPI - datasource?: UIDatasource - options?: {} - }) { + constructor(opts: { API: UIFetchAPI; datasource: T; options?: {} }) { // Feature flags this.features = { supportsSearch: false, @@ -90,7 +86,7 @@ export default abstract class DataFetch { // Config this.options = { - datasource: null, + datasource: opts.datasource, limit: 10, // Search config @@ -182,7 +178,7 @@ export default abstract class DataFetch { getDefaultSortColumn( definition: { primaryDisplay?: string } | null, schema: Record - ) { + ): string | null { if (definition?.primaryDisplay && schema[definition.primaryDisplay]) { return definition.primaryDisplay } else { diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 0615f83dc25..e3a2e317beb 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -1,8 +1,8 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" -import { SortOrder } from "@budibase/types" +import { SortOrder, UITable } from "@budibase/types" -export default class TableFetch extends DataFetch { +export default class TableFetch extends DataFetch { determineFeatureFlags() { return { supportsSearch: true, @@ -14,7 +14,7 @@ export default class TableFetch extends DataFetch { async getData() { const { datasource, limit, sortColumn, sortOrder, sortType, paginate } = this.options - const { tableId } = datasource! + const { tableId } = datasource const { cursor, query } = get(this.store) // Search table diff --git a/packages/types/src/ui/stores/grid/fetch.ts b/packages/types/src/ui/stores/grid/fetch.ts index 772e68eb14e..a81f436fde4 100644 --- a/packages/types/src/ui/stores/grid/fetch.ts +++ b/packages/types/src/ui/stores/grid/fetch.ts @@ -9,6 +9,16 @@ import { UISearchFilter, } from "@budibase/types" +interface SearchOptions { + query: SearchFilters | null + limit: number + sort: string | null + sortOrder: string + sortType: SortType | null + paginate: boolean + bookmark: null +} + export interface UIFetchAPI { fetchTableDefinition(tableId: string): Promise
definition: UIDatasource @@ -17,18 +27,7 @@ export interface UIFetchAPI { loading: any loaded: boolean - searchTable( - tableId: string, - arg1: { - query: SearchFilters | null - limit: number - sort: string | null - sortOrder: string - sortType: SortType | null - paginate: boolean - bookmark: null - } - ): any + searchTable(tableId: string, options: SearchOptions): any resetKey: string | null error: any From 97eb4a2e79824923b41c47d3a51f9f2d76175a2a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 12:33:54 +0100 Subject: [PATCH 05/63] Type view fetch --- packages/frontend-core/src/fetch/DataFetch.ts | 22 ++++++++++------- .../fetch/{ViewV2Fetch.js => ViewV2Fetch.ts} | 24 +++++++++++-------- .../types/src/ui/stores/grid/datasource.ts | 4 +--- packages/types/src/ui/stores/grid/fetch.ts | 9 +++++-- packages/types/src/ui/stores/grid/view.ts | 3 ++- 5 files changed, 37 insertions(+), 25 deletions(-) rename packages/frontend-core/src/fetch/{ViewV2Fetch.js => ViewV2Fetch.ts} (70%) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 7355afd967f..ea28cd72402 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -8,7 +8,6 @@ import { SearchFilters, SortOrder, SortType, - Table, TableSchema, UIDatasource, UIFetchAPI, @@ -18,7 +17,7 @@ import { const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils -interface DataFetchStore { +interface DataFetchStore { rows: UIRow[] info: null schema: TableSchema | null @@ -30,9 +29,11 @@ interface DataFetchStore { cursors: any[] resetKey: number error: null + definition?: T | null } -interface DataFetchDerivedStore extends DataFetchStore { +interface DataFetchDerivedStore + extends DataFetchStore { hasNextPage: boolean hasPrevPage: boolean supportsSearch: boolean @@ -69,8 +70,8 @@ export default abstract class DataFetch { clientSideSorting: boolean clientSideLimiting: boolean } - store: Writable - derivedStore: Readable + store: Writable> + derivedStore: Readable> /** * Constructs a new DataFetch instance. @@ -335,7 +336,7 @@ export default abstract class DataFetch { return null } try { - return await this.API.fetchTableDefinition(datasource.tableId) + return (await this.API.fetchTableDefinition(datasource.tableId)) as T } catch (error: any) { this.store.update(state => ({ ...state, @@ -352,7 +353,10 @@ export default abstract class DataFetch { * @param definition the datasource definition * @return {object} the schema */ - getSchema(_datasource: UIDatasource | null, definition: Table | null) { + getSchema( + _datasource: UIDatasource | null, + definition: T | null + ): TableSchema | undefined { return definition?.schema } @@ -412,7 +416,7 @@ export default abstract class DataFetch { * Determine the feature flag for this datasource definition * @param definition */ - determineFeatureFlags(_definition: Table | null) { + determineFeatureFlags(_definition: T | null) { return { supportsSearch: false, supportsSort: false, @@ -495,7 +499,7 @@ export default abstract class DataFetch { * @param state the current store state * @return {boolean} whether there is a next page of data or not */ - hasNextPage(state: DataFetchStore): boolean { + hasNextPage(state: DataFetchStore): boolean { return state.cursors[state.pageNumber + 1] != null } diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.ts similarity index 70% rename from packages/frontend-core/src/fetch/ViewV2Fetch.js rename to packages/frontend-core/src/fetch/ViewV2Fetch.ts index 84366460774..337c090c665 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.js +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -1,8 +1,9 @@ -import { ViewV2Type } from "@budibase/types" +import { SortOrder, UIView, ViewV2Type } from "@budibase/types" import DataFetch from "./DataFetch.js" import { get } from "svelte/store" +import { isCalculationField } from "packages/shared-core/src/helpers/views.js" -export default class ViewV2Fetch extends DataFetch { +export default class ViewV2Fetch extends DataFetch { determineFeatureFlags() { return { supportsSearch: true, @@ -11,18 +12,18 @@ export default class ViewV2Fetch extends DataFetch { } } - getSchema(datasource, definition) { + getSchema(_datasource: UIView | null, definition: UIView | null) { return definition?.schema } - async getDefinition(datasource) { + async getDefinition(datasource: UIView | null): Promise { if (!datasource?.id) { return null } try { const res = await this.API.viewV2.fetchDefinition(datasource.id) return res?.data - } catch (error) { + } catch (error: any) { this.store.update(state => ({ ...state, error, @@ -31,7 +32,10 @@ export default class ViewV2Fetch extends DataFetch { } } - getDefaultSortColumn() { + getDefaultSortColumn( + _definition: { primaryDisplay?: string } | null, + _schema: Record + ) { return null } @@ -42,8 +46,8 @@ export default class ViewV2Fetch extends DataFetch { // If this is a calculation view and we have no calculations, return nothing if ( - definition.type === ViewV2Type.CALCULATION && - !Object.values(definition.schema || {}).some(x => x.calculationType) + definition?.type === ViewV2Type.CALCULATION && + !Object.values(definition.schema || {}).some(isCalculationField) ) { return { rows: [], @@ -56,9 +60,9 @@ export default class ViewV2Fetch extends DataFetch { // If sort/filter params are not defined, update options to store the // params built in to this view. This ensures that we can accurately // compare old and new params and skip a redundant API call. - if (!sortColumn && definition.sort?.field) { + if (!sortColumn && definition?.sort?.field) { this.options.sortColumn = definition.sort.field - this.options.sortOrder = definition.sort.order + this.options.sortOrder = definition.sort.order || SortOrder.ASCENDING } try { diff --git a/packages/types/src/ui/stores/grid/datasource.ts b/packages/types/src/ui/stores/grid/datasource.ts index 9533bbb8f08..99275181330 100644 --- a/packages/types/src/ui/stores/grid/datasource.ts +++ b/packages/types/src/ui/stores/grid/datasource.ts @@ -1,8 +1,6 @@ import { UITable, UIView } from "@budibase/types" -export type UIDatasource = (UITable | UIView) & { - type: string -} +export type UIDatasource = UITable | UIView export interface UIFieldMutation { visible?: boolean diff --git a/packages/types/src/ui/stores/grid/fetch.ts b/packages/types/src/ui/stores/grid/fetch.ts index a81f436fde4..a8732c66e38 100644 --- a/packages/types/src/ui/stores/grid/fetch.ts +++ b/packages/types/src/ui/stores/grid/fetch.ts @@ -10,10 +10,10 @@ import { } from "@budibase/types" interface SearchOptions { - query: SearchFilters | null + query?: SearchFilters | null | undefined limit: number sort: string | null - sortOrder: string + sortOrder: string | undefined sortType: SortType | null paginate: boolean bookmark: null @@ -29,6 +29,11 @@ export interface UIFetchAPI { searchTable(tableId: string, options: SearchOptions): any + viewV2: { + fetchDefinition: (datasourceId: string) => Promise + fetch: (datasourceId: string, options: SearchOptions) => any + } + resetKey: string | null error: any diff --git a/packages/types/src/ui/stores/grid/view.ts b/packages/types/src/ui/stores/grid/view.ts index f81cc34aaf5..270faaa1606 100644 --- a/packages/types/src/ui/stores/grid/view.ts +++ b/packages/types/src/ui/stores/grid/view.ts @@ -1,6 +1,7 @@ import { ViewV2 } from "@budibase/types" import { UIFieldSchema } from "./table" -export interface UIView extends ViewV2 { +export interface UIView extends Omit { + type: string schema: Record } From 163ca349a91d335c8344867761f892bcac994da1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 13:21:36 +0100 Subject: [PATCH 06/63] Create apis --- packages/types/src/ui/stores/grid/fetch.ts | 35 +++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/types/src/ui/stores/grid/fetch.ts b/packages/types/src/ui/stores/grid/fetch.ts index a8732c66e38..5f42db24b0c 100644 --- a/packages/types/src/ui/stores/grid/fetch.ts +++ b/packages/types/src/ui/stores/grid/fetch.ts @@ -19,20 +19,41 @@ interface SearchOptions { bookmark: null } -export interface UIFetchAPI { +interface TableAPI { fetchTableDefinition(tableId: string): Promise
+ searchTable(tableId: string, options: SearchOptions): any +} + +interface ViewV2API { + fetchDefinition: (datasourceId: string) => Promise + fetch: (datasourceId: string, options: SearchOptions) => any +} + +interface UserAPI { + searchUsers: (opts: { + bookmark: null + query: + | SearchFilters + | { + string: { + email: null + } + } + | null + appId: string + paginate: boolean + limit: number + }) => Promise +} + +export interface UIFetchAPI extends TableAPI, UserAPI { definition: UIDatasource getInitialData: () => Promise loading: any loaded: boolean - searchTable(tableId: string, options: SearchOptions): any - - viewV2: { - fetchDefinition: (datasourceId: string) => Promise - fetch: (datasourceId: string, options: SearchOptions) => any - } + viewV2: ViewV2API resetKey: string | null error: any From 1899af919077c8ca87030c2c27517ddb4565f549 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 13:33:24 +0100 Subject: [PATCH 07/63] More types --- packages/frontend-core/src/fetch/DataFetch.ts | 57 ++++++++----------- .../frontend-core/src/fetch/TableFetch.ts | 23 +++++++- .../frontend-core/src/fetch/ViewV2Fetch.ts | 8 +-- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index ea28cd72402..389ddd2f17a 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -17,7 +17,7 @@ import { const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils -interface DataFetchStore { +interface DataFetchStore { rows: UIRow[] info: null schema: TableSchema | null @@ -32,8 +32,7 @@ interface DataFetchStore { definition?: T | null } -interface DataFetchDerivedStore - extends DataFetchStore { +interface DataFetchDerivedStore extends DataFetchStore { hasNextPage: boolean hasPrevPage: boolean supportsSearch: boolean @@ -46,7 +45,10 @@ interface DataFetchDerivedStore * internal table or datasource plus. * For other types of datasource, this class is overridden and extended. */ -export default abstract class DataFetch { +export default abstract class DataFetch< + TDatasource extends UIDatasource | null, + TDefinition extends { primaryDisplay?: string } +> { API: UIFetchAPI features: { supportsSearch: boolean @@ -54,7 +56,7 @@ export default abstract class DataFetch { supportsPagination: boolean } options: { - datasource: T + datasource: TDatasource limit: number // Search config filter: UISearchFilter | LegacyFilter[] | null @@ -70,14 +72,18 @@ export default abstract class DataFetch { clientSideSorting: boolean clientSideLimiting: boolean } - store: Writable> - derivedStore: Readable> + store: Writable> + derivedStore: Readable> /** * Constructs a new DataFetch instance. * @param opts the fetch options */ - constructor(opts: { API: UIFetchAPI; datasource: T; options?: {} }) { + constructor(opts: { + API: UIFetchAPI + datasource: TDatasource + options?: {} + }) { // Feature flags this.features = { supportsSearch: false, @@ -327,38 +333,23 @@ export default abstract class DataFetch { /** * Gets the definition for this datasource. - * Defaults to fetching a table definition. * @param datasource * @return {object} the definition */ - async getDefinition(datasource: UIDatasource | null) { - if (!datasource?.tableId) { - return null - } - try { - return (await this.API.fetchTableDefinition(datasource.tableId)) as T - } catch (error: any) { - this.store.update(state => ({ - ...state, - error, - })) - return null - } - } + abstract getDefinition( + datasource: UIDatasource | null + ): Promise /** * Gets the schema definition for a datasource. - * Defaults to getting the "schema" property of the definition. - * @param _datasource the datasource + * @param datasource the datasource * @param definition the datasource definition * @return {object} the schema */ - getSchema( - _datasource: UIDatasource | null, - definition: T | null - ): TableSchema | undefined { - return definition?.schema - } + abstract getSchema( + datasource: UIDatasource | null, + definition: TDefinition | null + ): any /** * Enriches a datasource schema with nested fields and ensures the structure @@ -416,7 +407,7 @@ export default abstract class DataFetch { * Determine the feature flag for this datasource definition * @param definition */ - determineFeatureFlags(_definition: T | null) { + determineFeatureFlags(_definition: TDefinition | null) { return { supportsSearch: false, supportsSort: false, @@ -499,7 +490,7 @@ export default abstract class DataFetch { * @param state the current store state * @return {boolean} whether there is a next page of data or not */ - hasNextPage(state: DataFetchStore): boolean { + hasNextPage(state: DataFetchStore): boolean { return state.cursors[state.pageNumber + 1] != null } diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index e3a2e317beb..3c4a1b7abc8 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -1,8 +1,8 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" -import { SortOrder, UITable } from "@budibase/types" +import { SortOrder, Table, UITable } from "@budibase/types" -export default class TableFetch extends DataFetch { +export default class TableFetch extends DataFetch { determineFeatureFlags() { return { supportsSearch: true, @@ -11,6 +11,25 @@ export default class TableFetch extends DataFetch { } } + async getDefinition(datasource: UITable | null) { + if (!datasource?.tableId) { + return null + } + try { + return await this.API.fetchTableDefinition(datasource.tableId) + } catch (error: any) { + this.store.update(state => ({ + ...state, + error, + })) + return null + } + } + + getSchema(_datasource: UITable | null, definition: Table | null) { + return definition?.schema + } + async getData() { const { datasource, limit, sortColumn, sortOrder, sortType, paginate } = this.options diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 337c090c665..d880b3a5490 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -1,9 +1,9 @@ -import { SortOrder, UIView, ViewV2Type } from "@budibase/types" +import { SortOrder, UIView, ViewV2, ViewV2Type } from "@budibase/types" import DataFetch from "./DataFetch.js" import { get } from "svelte/store" import { isCalculationField } from "packages/shared-core/src/helpers/views.js" -export default class ViewV2Fetch extends DataFetch { +export default class ViewV2Fetch extends DataFetch { determineFeatureFlags() { return { supportsSearch: true, @@ -12,11 +12,11 @@ export default class ViewV2Fetch extends DataFetch { } } - getSchema(_datasource: UIView | null, definition: UIView | null) { + getSchema(_datasource: UIView, definition: ViewV2) { return definition?.schema } - async getDefinition(datasource: UIView | null): Promise { + async getDefinition(datasource: UIView | null): Promise { if (!datasource?.id) { return null } From c7255362b0b8b1a72f785602526515d16669c62a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 13:36:38 +0100 Subject: [PATCH 08/63] Cleanup --- packages/frontend-core/src/fetch/DataFetch.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 389ddd2f17a..2dc42c24255 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -9,7 +9,6 @@ import { SortOrder, SortType, TableSchema, - UIDatasource, UIFetchAPI, UIRow, UISearchFilter, @@ -46,8 +45,8 @@ interface DataFetchDerivedStore extends DataFetchStore { * For other types of datasource, this class is overridden and extended. */ export default abstract class DataFetch< - TDatasource extends UIDatasource | null, - TDefinition extends { primaryDisplay?: string } + TDatasource extends {}, + TDefinition extends {} > { API: UIFetchAPI features: { @@ -337,7 +336,7 @@ export default abstract class DataFetch< * @return {object} the definition */ abstract getDefinition( - datasource: UIDatasource | null + datasource: TDatasource | null ): Promise /** @@ -347,7 +346,7 @@ export default abstract class DataFetch< * @return {object} the schema */ abstract getSchema( - datasource: UIDatasource | null, + datasource: TDatasource | null, definition: TDefinition | null ): any From 54d5047b34932b4beab6f582d3a2b7059d6d488f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 13:38:38 +0100 Subject: [PATCH 09/63] Convert UserFetch --- .../src/fetch/{UserFetch.js => UserFetch.ts} | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) rename packages/frontend-core/src/fetch/{UserFetch.js => UserFetch.ts} (77%) diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.ts similarity index 77% rename from packages/frontend-core/src/fetch/UserFetch.js rename to packages/frontend-core/src/fetch/UserFetch.ts index 36f61542b58..730d96ebd5c 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -2,9 +2,10 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" import { utils } from "@budibase/shared-core" +import { Table, UIFetchAPI } from "@budibase/types" -export default class UserFetch extends DataFetch { - constructor(opts) { +export default class UserFetch extends DataFetch<{ tableId: string }, {}> { + constructor(opts: { API: UIFetchAPI; datasource: Table; options?: {} }) { super({ ...opts, datasource: { @@ -27,12 +28,16 @@ export default class UserFetch extends DataFetch { } } + getSchema(_datasource: any, definition: Table | null) { + return definition?.schema + } + async getData() { const { limit, paginate } = this.options const { cursor, query } = get(this.store) // Convert old format to new one - we now allow use of the lucene format - const { appId, paginated, ...rest } = query || {} + const { appId, paginated, ...rest } = query || ({} as any) // TODO const finalQuery = utils.isSupportedUserSearch(rest) ? query : { string: { email: null } } From 65fa3e04346985c2d9f4293fb0e007d0c79f41e0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 15:36:27 +0100 Subject: [PATCH 10/63] Use APIClient --- packages/frontend-core/src/fetch/DataFetch.ts | 16 ++++++---------- packages/frontend-core/src/fetch/TableFetch.ts | 4 ++-- packages/frontend-core/src/fetch/ViewV2Fetch.ts | 8 +++++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 2dc42c24255..ad709c9c77c 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -5,19 +5,19 @@ import { convertJSONSchemaToTableSchema } from "../utils/json" import { FieldType, LegacyFilter, + Row, SearchFilters, SortOrder, SortType, TableSchema, - UIFetchAPI, - UIRow, UISearchFilter, } from "@budibase/types" +import { APIClient } from "../api/types" const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils interface DataFetchStore { - rows: UIRow[] + rows: Row[] info: null schema: TableSchema | null loading: boolean @@ -48,7 +48,7 @@ export default abstract class DataFetch< TDatasource extends {}, TDefinition extends {} > { - API: UIFetchAPI + API: APIClient features: { supportsSearch: boolean supportsSort: boolean @@ -78,11 +78,7 @@ export default abstract class DataFetch< * Constructs a new DataFetch instance. * @param opts the fetch options */ - constructor(opts: { - API: UIFetchAPI - datasource: TDatasource - options?: {} - }) { + constructor(opts: { API: APIClient; datasource: TDatasource; options?: {} }) { // Feature flags this.features = { supportsSearch: false, @@ -323,7 +319,7 @@ export default abstract class DataFetch< } abstract getData(): Promise<{ - rows: UIRow[] + rows: Row[] info?: any hasNextPage: boolean cursor?: any diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 3c4a1b7abc8..08dc111b289 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -39,10 +39,10 @@ export default class TableFetch extends DataFetch { // Search table try { const res = await this.API.searchTable(tableId, { - query, + query: query ?? undefined, limit, sort: sortColumn, - sortOrder: sortOrder?.toLowerCase() ?? SortOrder.ASCENDING, + sortOrder: sortOrder ?? SortOrder.ASCENDING, sortType, paginate, bookmark: cursor, diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index d880b3a5490..91d2385d3cf 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -1,7 +1,7 @@ import { SortOrder, UIView, ViewV2, ViewV2Type } from "@budibase/types" import DataFetch from "./DataFetch.js" import { get } from "svelte/store" -import { isCalculationField } from "packages/shared-core/src/helpers/views.js" +import { helpers } from "@budibase/shared-core" export default class ViewV2Fetch extends DataFetch { determineFeatureFlags() { @@ -47,7 +47,9 @@ export default class ViewV2Fetch extends DataFetch { // If this is a calculation view and we have no calculations, return nothing if ( definition?.type === ViewV2Type.CALCULATION && - !Object.values(definition.schema || {}).some(isCalculationField) + !Object.values(definition.schema || {}).some( + helpers.views.isCalculationField + ) ) { return { rows: [], @@ -72,7 +74,7 @@ export default class ViewV2Fetch extends DataFetch { limit, bookmark: cursor, sort: sortColumn, - sortOrder: sortOrder?.toLowerCase(), + sortOrder: sortOrder, sortType, }) return { From 1f51489368acccbbeda3deb6ed219f4a60a3c723 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 15:48:04 +0100 Subject: [PATCH 11/63] Type views --- packages/frontend-core/src/api/viewsV2.ts | 13 ++++++--- .../frontend-core/src/fetch/ViewV2Fetch.ts | 28 +++++++++++++++---- .../server/src/api/controllers/row/views.ts | 11 ++++++-- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/packages/frontend-core/src/api/viewsV2.ts b/packages/frontend-core/src/api/viewsV2.ts index 5018448e8c7..4a867e8f6ac 100644 --- a/packages/frontend-core/src/api/viewsV2.ts +++ b/packages/frontend-core/src/api/viewsV2.ts @@ -1,6 +1,7 @@ import { CreateViewRequest, CreateViewResponse, + PaginatedSearchRowResponse, SearchRowResponse, SearchViewRowRequest, UpdateViewRequest, @@ -13,10 +14,14 @@ export interface ViewV2Endpoints { fetchDefinition: (viewId: string) => Promise create: (view: CreateViewRequest) => Promise update: (view: UpdateViewRequest) => Promise - fetch: ( + fetch: ( viewId: string, - opts: SearchViewRowRequest - ) => Promise + opts: T + ) => Promise< + T extends { paginate: true } + ? PaginatedSearchRowResponse + : SearchRowResponse + > delete: (viewId: string) => Promise } @@ -59,7 +64,7 @@ export const buildViewV2Endpoints = (API: BaseAPIClient): ViewV2Endpoints => ({ * @param viewId the id of the view * @param opts the search options */ - fetch: async (viewId, opts) => { + fetch: async (viewId, opts: SearchViewRowRequest) => { return await API.post({ url: `/api/v2/views/${encodeURIComponent(viewId)}/search`, body: opts, diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 91d2385d3cf..d3b607a171d 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -68,7 +68,7 @@ export default class ViewV2Fetch extends DataFetch { } try { - const res = await this.API.viewV2.fetch(datasource.id, { + const request = { ...(query ? { query } : {}), paginate, limit, @@ -76,11 +76,27 @@ export default class ViewV2Fetch extends DataFetch { sort: sortColumn, sortOrder: sortOrder, sortType, - }) - return { - rows: res?.rows || [], - hasNextPage: res?.hasNextPage || false, - cursor: res?.bookmark || null, + } + if (paginate) { + const res = await this.API.viewV2.fetch(datasource.id, { + ...request, + paginate, + }) + return { + rows: res?.rows || [], + hasNextPage: res?.hasNextPage || false, + cursor: res?.bookmark || null, + } + } else { + const res = await this.API.viewV2.fetch(datasource.id, { + ...request, + paginate, + }) + return { + rows: res?.rows || [], + hasNextPage: false, + cursor: null, + } } } catch (error) { return { diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 0655a3b38f0..dcf8680348c 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -1,16 +1,16 @@ import { UserCtx, ViewV2, - SearchRowResponse, SearchViewRowRequest, RequiredKeys, RowSearchParams, + PaginatedSearchRowResponse, } from "@budibase/types" import sdk from "../../../sdk" import { context } from "@budibase/backend-core" export async function searchView( - ctx: UserCtx + ctx: UserCtx ) { const { viewId } = ctx.params @@ -49,7 +49,12 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) result.rows.forEach(r => (r._viewId = view.id)) - ctx.body = result + + ctx.body = { + rows: result.rows, + bookmark: result.bookmark, + hasNextPage: result.hasNextPage, + } } function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { From f0d60c606329558308bd9024865b5046e6f12f05 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 15:55:50 +0100 Subject: [PATCH 12/63] Fix user types --- packages/frontend-core/src/fetch/UserFetch.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index 730d96ebd5c..be73793768c 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -2,10 +2,11 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" import { utils } from "@budibase/shared-core" -import { Table, UIFetchAPI } from "@budibase/types" +import { BasicOperator, SearchUsersRequest, Table } from "@budibase/types" +import { APIClient } from "../api/types.js" export default class UserFetch extends DataFetch<{ tableId: string }, {}> { - constructor(opts: { API: UIFetchAPI; datasource: Table; options?: {} }) { + constructor(opts: { API: APIClient; datasource: Table; options?: {} }) { super({ ...opts, datasource: { @@ -40,12 +41,12 @@ export default class UserFetch extends DataFetch<{ tableId: string }, {}> { const { appId, paginated, ...rest } = query || ({} as any) // TODO const finalQuery = utils.isSupportedUserSearch(rest) ? query - : { string: { email: null } } + : { [BasicOperator.EMPTY]: { email: true } } // TODO: check try { const opts = { - bookmark: cursor, - query: finalQuery, + bookmark: cursor ?? undefined, + query: finalQuery ?? undefined, appId: appId, paginate: paginated || paginate, limit, From 69ad15f79cb1b3dc482bf1557c35868d401f090c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 16:25:25 +0100 Subject: [PATCH 13/63] Fix types --- packages/frontend-core/src/api/views.ts | 2 +- packages/frontend-core/src/fetch/DataFetch.ts | 2 +- packages/frontend-core/src/fetch/TableFetch.ts | 2 +- packages/frontend-core/src/fetch/UserFetch.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/frontend-core/src/api/views.ts b/packages/frontend-core/src/api/views.ts index 3f8ac8aa41c..083a3890e80 100644 --- a/packages/frontend-core/src/api/views.ts +++ b/packages/frontend-core/src/api/views.ts @@ -3,7 +3,7 @@ import { BaseAPIClient } from "./types" export interface ViewEndpoints { // Missing request or response types - fetchViewData: (name: string, opts: any) => Promise + fetchViewData: (name: string, opts?: any) => Promise exportView: (name: string, format: string) => Promise saveView: (view: any) => Promise deleteView: (name: string) => Promise diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index ad709c9c77c..73f4e8bd11c 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -321,7 +321,7 @@ export default abstract class DataFetch< abstract getData(): Promise<{ rows: Row[] info?: any - hasNextPage: boolean + hasNextPage?: boolean cursor?: any error?: any }> diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 08dc111b289..3ea60638823 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -11,7 +11,7 @@ export default class TableFetch extends DataFetch { } } - async getDefinition(datasource: UITable | null) { + async getDefinition(datasource: UITable) { if (!datasource?.tableId) { return null } diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index be73793768c..571825cd3b4 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -2,7 +2,7 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" import { utils } from "@budibase/shared-core" -import { BasicOperator, SearchUsersRequest, Table } from "@budibase/types" +import { BasicOperator, Table } from "@budibase/types" import { APIClient } from "../api/types.js" export default class UserFetch extends DataFetch<{ tableId: string }, {}> { From b420fda5249943dbe52eb2c6485cdd09bec71046 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 16:25:37 +0100 Subject: [PATCH 14/63] Convert ViewFetch --- .../src/fetch/{ViewFetch.js => ViewFetch.ts} | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) rename packages/frontend-core/src/fetch/{ViewFetch.js => ViewFetch.ts} (50%) diff --git a/packages/frontend-core/src/fetch/ViewFetch.js b/packages/frontend-core/src/fetch/ViewFetch.ts similarity index 50% rename from packages/frontend-core/src/fetch/ViewFetch.js rename to packages/frontend-core/src/fetch/ViewFetch.ts index eb89f9b67a8..65fad905646 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.js +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -1,7 +1,25 @@ +import { Table, View } from "@budibase/types" import DataFetch from "./DataFetch.js" -export default class ViewFetch extends DataFetch { - getSchema(datasource, definition) { +type ViewV1 = View & { name: string } + +export default class ViewFetch extends DataFetch { + async getDefinition(datasource: ViewV1) { + if (!datasource?.tableId) { + return null + } + try { + return await this.API.fetchTableDefinition(datasource.tableId) + } catch (error: any) { + this.store.update(state => ({ + ...state, + error, + })) + return null + } + } + + getSchema(datasource: ViewV1, definition: Table) { return definition?.views?.[datasource.name]?.schema } From 2b863cca61ac9096fcf132e8af6c36e46ddac163 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 2 Jan 2025 16:27:43 +0100 Subject: [PATCH 15/63] Fix imports --- packages/client/src/utils/schema.js | 6 +++--- packages/frontend-core/src/fetch/CustomFetch.js | 2 +- packages/frontend-core/src/fetch/FieldFetch.js | 2 +- packages/frontend-core/src/fetch/GroupUserFetch.js | 2 +- packages/frontend-core/src/fetch/NestedProviderFetch.js | 2 +- packages/frontend-core/src/fetch/QueryFetch.js | 2 +- packages/frontend-core/src/fetch/RelationshipFetch.js | 2 +- packages/frontend-core/src/fetch/TableFetch.ts | 2 +- packages/frontend-core/src/fetch/UserFetch.ts | 2 +- packages/frontend-core/src/fetch/ViewFetch.ts | 2 +- packages/frontend-core/src/fetch/ViewV2Fetch.ts | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index ec1cef53cee..9e8b9f3d4c8 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -1,12 +1,12 @@ import { API } from "api" -import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch.js" -import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch.js" +import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch" +import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch" import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js" import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch.js" import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js" import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js" import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js" -import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js" +import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch" import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch" /** diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js index fc62d790e2a..39d3bd6f4c6 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.js +++ b/packages/frontend-core/src/fetch/CustomFetch.js @@ -1,4 +1,4 @@ -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" export default class CustomFetch extends DataFetch { // Gets the correct Budibase type for a JS value diff --git a/packages/frontend-core/src/fetch/FieldFetch.js b/packages/frontend-core/src/fetch/FieldFetch.js index 9402a45a832..7c9d715761b 100644 --- a/packages/frontend-core/src/fetch/FieldFetch.js +++ b/packages/frontend-core/src/fetch/FieldFetch.js @@ -1,4 +1,4 @@ -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" export default class FieldFetch extends DataFetch { async getDefinition(datasource) { diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.js b/packages/frontend-core/src/fetch/GroupUserFetch.js index bd2cf264c51..e40b565728d 100644 --- a/packages/frontend-core/src/fetch/GroupUserFetch.js +++ b/packages/frontend-core/src/fetch/GroupUserFetch.js @@ -1,5 +1,5 @@ import { get } from "svelte/store" -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" import { TableNames } from "../constants" export default class GroupUserFetch extends DataFetch { diff --git a/packages/frontend-core/src/fetch/NestedProviderFetch.js b/packages/frontend-core/src/fetch/NestedProviderFetch.js index 0a08b00cb43..06c74cb6c45 100644 --- a/packages/frontend-core/src/fetch/NestedProviderFetch.js +++ b/packages/frontend-core/src/fetch/NestedProviderFetch.js @@ -1,4 +1,4 @@ -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" export default class NestedProviderFetch extends DataFetch { async getDefinition(datasource) { diff --git a/packages/frontend-core/src/fetch/QueryFetch.js b/packages/frontend-core/src/fetch/QueryFetch.js index 9fac9704d3b..4f7954c0687 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.js +++ b/packages/frontend-core/src/fetch/QueryFetch.js @@ -1,4 +1,4 @@ -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" import { Helpers } from "@budibase/bbui" import { get } from "svelte/store" diff --git a/packages/frontend-core/src/fetch/RelationshipFetch.js b/packages/frontend-core/src/fetch/RelationshipFetch.js index 0dec535724c..61f02075583 100644 --- a/packages/frontend-core/src/fetch/RelationshipFetch.js +++ b/packages/frontend-core/src/fetch/RelationshipFetch.js @@ -1,4 +1,4 @@ -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" export default class RelationshipFetch extends DataFetch { async getData() { diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 3ea60638823..58ad554d6ae 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -1,5 +1,5 @@ import { get } from "svelte/store" -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" import { SortOrder, Table, UITable } from "@budibase/types" export default class TableFetch extends DataFetch { diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index 571825cd3b4..a0f95afe3e1 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -1,5 +1,5 @@ import { get } from "svelte/store" -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" import { TableNames } from "../constants" import { utils } from "@budibase/shared-core" import { BasicOperator, Table } from "@budibase/types" diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts index 65fad905646..cbdd1d425ba 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.ts +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -1,5 +1,5 @@ import { Table, View } from "@budibase/types" -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" type ViewV1 = View & { name: string } diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index d3b607a171d..7973dbf298e 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -1,5 +1,5 @@ import { SortOrder, UIView, ViewV2, ViewV2Type } from "@budibase/types" -import DataFetch from "./DataFetch.js" +import DataFetch from "./DataFetch" import { get } from "svelte/store" import { helpers } from "@budibase/shared-core" From 543660dc2e9448730eb89e71402d5f753acf16b3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 3 Jan 2025 12:34:36 +0100 Subject: [PATCH 16/63] Convert fieldFetch --- .../fetch/{FieldFetch.js => FieldFetch.ts} | 30 +++++++++++++++---- .../frontend-core/src/fetch/JSONArrayFetch.js | 2 +- .../src/fetch/QueryArrayFetch.js | 2 +- packages/frontend-core/src/fetch/index.ts | 2 +- 4 files changed, 28 insertions(+), 8 deletions(-) rename packages/frontend-core/src/fetch/{FieldFetch.js => FieldFetch.ts} (52%) diff --git a/packages/frontend-core/src/fetch/FieldFetch.js b/packages/frontend-core/src/fetch/FieldFetch.ts similarity index 52% rename from packages/frontend-core/src/fetch/FieldFetch.js rename to packages/frontend-core/src/fetch/FieldFetch.ts index 7c9d715761b..7f0dfdf3318 100644 --- a/packages/frontend-core/src/fetch/FieldFetch.js +++ b/packages/frontend-core/src/fetch/FieldFetch.ts @@ -1,9 +1,29 @@ +import { Row, TableSchema } from "@budibase/types" import DataFetch from "./DataFetch" -export default class FieldFetch extends DataFetch { - async getDefinition(datasource) { +interface FieldDatasource { + fieldType: "attachment" | "array" + value: string[] | Row[] +} + +function isArrayOfStrings(value: string[] | Row[]): value is string[] { + return Array.isArray(value) && !!value[0] && typeof value[0] !== "object" +} + +export default class FieldFetch extends DataFetch< + FieldDatasource, + { schema?: Record } +> { + getSchema( + _datasource: FieldDatasource, + definition: { schema?: TableSchema } + ) { + return definition?.schema + } + + async getDefinition(datasource: FieldDatasource) { // Field sources have their schema statically defined - let schema + let schema: Record | undefined if (datasource.fieldType === "attachment") { schema = { url: { @@ -28,8 +48,8 @@ export default class FieldFetch extends DataFetch { // These sources will be available directly from context const data = datasource?.value || [] - let rows - if (Array.isArray(data) && data[0] && typeof data[0] !== "object") { + let rows: Row[] + if (isArrayOfStrings(data)) { rows = data.map(value => ({ value })) } else { rows = data diff --git a/packages/frontend-core/src/fetch/JSONArrayFetch.js b/packages/frontend-core/src/fetch/JSONArrayFetch.js index ab2af3e2c77..f7de74a4b87 100644 --- a/packages/frontend-core/src/fetch/JSONArrayFetch.js +++ b/packages/frontend-core/src/fetch/JSONArrayFetch.js @@ -1,4 +1,4 @@ -import FieldFetch from "./FieldFetch.js" +import FieldFetch from "./FieldFetch" import { getJSONArrayDatasourceSchema } from "../utils/json" export default class JSONArrayFetch extends FieldFetch { diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.js b/packages/frontend-core/src/fetch/QueryArrayFetch.js index 0b36b640a6d..222697b78fa 100644 --- a/packages/frontend-core/src/fetch/QueryArrayFetch.js +++ b/packages/frontend-core/src/fetch/QueryArrayFetch.js @@ -1,4 +1,4 @@ -import FieldFetch from "./FieldFetch.js" +import FieldFetch from "./FieldFetch" import { getJSONArrayDatasourceSchema, generateQueryArraySchemas, diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index a08748a77ea..502b2d3162a 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -4,7 +4,7 @@ import ViewV2Fetch from "./ViewV2Fetch.js" import QueryFetch from "./QueryFetch.js" import RelationshipFetch from "./RelationshipFetch.js" import NestedProviderFetch from "./NestedProviderFetch.js" -import FieldFetch from "./FieldFetch.js" +import FieldFetch from "./FieldFetch" import JSONArrayFetch from "./JSONArrayFetch.js" import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch.js" From 97b0883c6b8b1da5ee8912fe7a861129e932b2f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 3 Jan 2025 12:34:43 +0100 Subject: [PATCH 17/63] Convert fieldFetch --- packages/client/src/utils/schema.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index 9e8b9f3d4c8..2b5dae0acf6 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -4,7 +4,7 @@ import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch" import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js" import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch.js" import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js" -import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js" +import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch" import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js" import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch" import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch" From 550cdd7268d102117dcba98ec210383b6e65c5b4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 3 Jan 2025 12:48:35 +0100 Subject: [PATCH 18/63] Fix field selector on datasource picker --- .../controls/DataSourceSelect/DataSourceSelect.svelte | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index e7a30e68ddf..b23ef5348de 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -43,7 +43,6 @@ export let showDataProviders = true const dispatch = createEventDispatcher() - const arrayTypes = ["attachment", "array"] let anchorRight, dropdownRight let drawer @@ -116,8 +115,11 @@ } }) $: fields = bindings - .filter(x => arrayTypes.includes(x.fieldSchema?.type)) - .filter(x => x.fieldSchema?.tableId != null) + .filter( + x => + x.fieldSchema?.type === "attachment" || + (x.fieldSchema?.type === "array" && x.tableId) + ) .map(binding => { const { providerId, readableBinding, runtimeBinding } = binding const { name, type, tableId } = binding.fieldSchema From 1d661c6290cc54c5b5fd27669fee530375a5a64b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 3 Jan 2025 15:37:04 +0100 Subject: [PATCH 19/63] Fix sort issue (with broken ts) --- packages/frontend-core/src/fetch/TableFetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 58ad554d6ae..344ce30e546 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -42,7 +42,7 @@ export default class TableFetch extends DataFetch { query: query ?? undefined, limit, sort: sortColumn, - sortOrder: sortOrder ?? SortOrder.ASCENDING, + sortOrder: sortOrder?.toLowerCase() ?? SortOrder.ASCENDING, sortType, paginate, bookmark: cursor, From af22eb30a60da04b39dc44834974a3bfcc06ac68 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 10:14:18 +0100 Subject: [PATCH 20/63] Type relationship fetch --- packages/client/src/utils/schema.js | 2 +- .../src/fetch/RelationshipFetch.js | 20 -------- .../src/fetch/RelationshipFetch.ts | 50 +++++++++++++++++++ packages/frontend-core/src/fetch/index.ts | 2 +- 4 files changed, 52 insertions(+), 22 deletions(-) delete mode 100644 packages/frontend-core/src/fetch/RelationshipFetch.js create mode 100644 packages/frontend-core/src/fetch/RelationshipFetch.ts diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index 2b5dae0acf6..87859f1ef8d 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -2,7 +2,7 @@ import { API } from "api" import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch" import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch" import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js" -import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch.js" +import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch" import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js" import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch" import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js" diff --git a/packages/frontend-core/src/fetch/RelationshipFetch.js b/packages/frontend-core/src/fetch/RelationshipFetch.js deleted file mode 100644 index 61f02075583..00000000000 --- a/packages/frontend-core/src/fetch/RelationshipFetch.js +++ /dev/null @@ -1,20 +0,0 @@ -import DataFetch from "./DataFetch" - -export default class RelationshipFetch extends DataFetch { - async getData() { - const { datasource } = this.options - if (!datasource?.rowId || !datasource?.rowTableId) { - return { rows: [] } - } - try { - const res = await this.API.fetchRelationshipData( - datasource.rowTableId, - datasource.rowId, - datasource.fieldName - ) - return { rows: res } - } catch (error) { - return { rows: [] } - } - } -} diff --git a/packages/frontend-core/src/fetch/RelationshipFetch.ts b/packages/frontend-core/src/fetch/RelationshipFetch.ts new file mode 100644 index 00000000000..ab50285b4b0 --- /dev/null +++ b/packages/frontend-core/src/fetch/RelationshipFetch.ts @@ -0,0 +1,50 @@ +import { Table } from "@budibase/types" +import DataFetch from "./DataFetch" + +interface RelationshipDatasource { + tableId: string + rowId: string + rowTableId: string + fieldName: string +} + +export default class RelationshipFetch extends DataFetch< + RelationshipDatasource, + Table +> { + getSchema(_datasource: any, definition: any) { + return definition?.schema + } + + async getDefinition(datasource: RelationshipDatasource) { + if (!datasource?.tableId) { + return null + } + try { + return await this.API.fetchTableDefinition(datasource.tableId) + } catch (error: any) { + this.store.update(state => ({ + ...state, + error, + })) + return null + } + } + + async getData() { + const { datasource } = this.options + if (!datasource?.rowId || !datasource?.rowTableId) { + return { rows: [] } + } + try { + const res = await this.API.fetchRelationshipData( + datasource.rowTableId, + datasource.rowId, + datasource.fieldName + ) + return { rows: res } + } catch (error) { + return { rows: [] } + } + } +} diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 502b2d3162a..0fbda7a4141 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -2,7 +2,7 @@ import TableFetch from "./TableFetch.js" import ViewFetch from "./ViewFetch.js" import ViewV2Fetch from "./ViewV2Fetch.js" import QueryFetch from "./QueryFetch.js" -import RelationshipFetch from "./RelationshipFetch.js" +import RelationshipFetch from "./RelationshipFetch" import NestedProviderFetch from "./NestedProviderFetch.js" import FieldFetch from "./FieldFetch" import JSONArrayFetch from "./JSONArrayFetch.js" From 8d748338739adc9cb66bf80497a0e8723ac4a32c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 11:02:09 +0100 Subject: [PATCH 21/63] Type groupUserFetch --- packages/frontend-core/src/fetch/DataFetch.ts | 35 ++++++++++++------- .../{GroupUserFetch.js => GroupUserFetch.ts} | 25 +++++++++++-- packages/frontend-core/src/fetch/index.ts | 2 +- 3 files changed, 45 insertions(+), 17 deletions(-) rename packages/frontend-core/src/fetch/{GroupUserFetch.js => GroupUserFetch.ts} (67%) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 73f4e8bd11c..ef05f7b8efb 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -16,22 +16,23 @@ import { APIClient } from "../api/types" const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils -interface DataFetchStore { +interface DataFetchStore { rows: Row[] info: null schema: TableSchema | null loading: boolean loaded: boolean - query: SearchFilters | null + query: TQuery pageNumber: number cursor: null cursors: any[] resetKey: number error: null - definition?: T | null + definition?: TDefinition | null } -interface DataFetchDerivedStore extends DataFetchStore { +interface DataFetchDerivedStore + extends DataFetchStore { hasNextPage: boolean hasPrevPage: boolean supportsSearch: boolean @@ -39,6 +40,13 @@ interface DataFetchDerivedStore extends DataFetchStore { supportsPagination: boolean } +interface DataFetchParams { + API: APIClient + datasource: TDatasource + query: TQuery + options?: {} +} + /** * Parent class which handles the implementation of fetching data from an * internal table or datasource plus. @@ -46,7 +54,8 @@ interface DataFetchDerivedStore extends DataFetchStore { */ export default abstract class DataFetch< TDatasource extends {}, - TDefinition extends {} + TDefinition extends {}, + TQuery extends {} = SearchFilters > { API: APIClient features: { @@ -59,7 +68,7 @@ export default abstract class DataFetch< limit: number // Search config filter: UISearchFilter | LegacyFilter[] | null - query: SearchFilters | null + query: TQuery // Sorting config sortColumn: string | null sortOrder: SortOrder @@ -71,14 +80,14 @@ export default abstract class DataFetch< clientSideSorting: boolean clientSideLimiting: boolean } - store: Writable> - derivedStore: Readable> + store: Writable> + derivedStore: Readable> /** * Constructs a new DataFetch instance. * @param opts the fetch options */ - constructor(opts: { API: APIClient; datasource: TDatasource; options?: {} }) { + constructor(opts: DataFetchParams) { // Feature flags this.features = { supportsSearch: false, @@ -93,7 +102,7 @@ export default abstract class DataFetch< // Search config filter: null, - query: null, + query: opts.query, // Sorting config sortColumn: null, @@ -116,7 +125,7 @@ export default abstract class DataFetch< schema: null, loading: false, loaded: false, - query: null, + query: opts.query, pageNumber: 0, cursor: null, cursors: [], @@ -247,7 +256,7 @@ export default abstract class DataFetch< // Build the query let query = this.options.query if (!query) { - query = buildQuery(filter ?? undefined) + query = buildQuery(filter ?? undefined) as TQuery } // Update store @@ -485,7 +494,7 @@ export default abstract class DataFetch< * @param state the current store state * @return {boolean} whether there is a next page of data or not */ - hasNextPage(state: DataFetchStore): boolean { + hasNextPage(state: DataFetchStore): boolean { return state.cursors[state.pageNumber + 1] != null } diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.js b/packages/frontend-core/src/fetch/GroupUserFetch.ts similarity index 67% rename from packages/frontend-core/src/fetch/GroupUserFetch.js rename to packages/frontend-core/src/fetch/GroupUserFetch.ts index e40b565728d..77b5e1de438 100644 --- a/packages/frontend-core/src/fetch/GroupUserFetch.js +++ b/packages/frontend-core/src/fetch/GroupUserFetch.ts @@ -1,9 +1,23 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch" import { TableNames } from "../constants" +import { APIClient } from "../api/types" -export default class GroupUserFetch extends DataFetch { - constructor(opts) { +interface GroupUserQuery { + groupId: string + emailSearch: string +} + +export default class GroupUserFetch extends DataFetch< + any, + any, + GroupUserQuery +> { + constructor(opts: { + API: APIClient + datasource: any + query: GroupUserQuery + }) { super({ ...opts, datasource: { @@ -12,6 +26,10 @@ export default class GroupUserFetch extends DataFetch { }) } + getSchema(_datasource: any, definition: any) { + return definition?.schema + } + determineFeatureFlags() { return { supportsSearch: true, @@ -28,11 +46,12 @@ export default class GroupUserFetch extends DataFetch { async getData() { const { query, cursor } = get(this.store) + try { const res = await this.API.getGroupUsers({ id: query.groupId, emailSearch: query.emailSearch, - bookmark: cursor, + bookmark: cursor ?? undefined, }) return { diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 0fbda7a4141..ef471aa8e43 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -7,7 +7,7 @@ import NestedProviderFetch from "./NestedProviderFetch.js" import FieldFetch from "./FieldFetch" import JSONArrayFetch from "./JSONArrayFetch.js" import UserFetch from "./UserFetch.js" -import GroupUserFetch from "./GroupUserFetch.js" +import GroupUserFetch from "./GroupUserFetch" import CustomFetch from "./CustomFetch.js" import QueryArrayFetch from "./QueryArrayFetch.js" import { Table, UIDatasource, UIFetchAPI } from "@budibase/types" From c52dd568723ae2c1933a1bb40eb91f5782421f39 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 11:17:42 +0100 Subject: [PATCH 22/63] Fix userFetch query --- packages/frontend-core/src/fetch/UserFetch.ts | 36 ++++++++++++++----- packages/shared-core/src/utils.ts | 4 ++- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index a0f95afe3e1..e276af35929 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -2,11 +2,30 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch" import { TableNames } from "../constants" import { utils } from "@budibase/shared-core" -import { BasicOperator, Table } from "@budibase/types" +import { + BasicOperator, + SearchFilters, + SearchUsersRequest, + Table, +} from "@budibase/types" import { APIClient } from "../api/types.js" -export default class UserFetch extends DataFetch<{ tableId: string }, {}> { - constructor(opts: { API: APIClient; datasource: Table; options?: {} }) { +interface UserFetchQuery { + appId: string + paginated: boolean +} + +export default class UserFetch extends DataFetch< + { tableId: string }, + {}, + UserFetchQuery +> { + constructor(opts: { + API: APIClient + datasource: Table + options?: {} + query: UserFetchQuery + }) { super({ ...opts, datasource: { @@ -38,13 +57,14 @@ export default class UserFetch extends DataFetch<{ tableId: string }, {}> { const { cursor, query } = get(this.store) // Convert old format to new one - we now allow use of the lucene format - const { appId, paginated, ...rest } = query || ({} as any) // TODO - const finalQuery = utils.isSupportedUserSearch(rest) - ? query - : { [BasicOperator.EMPTY]: { email: true } } // TODO: check + const { appId, paginated, ...rest } = query || {} + + const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest) + ? rest + : { [BasicOperator.EMPTY]: { email: true } } try { - const opts = { + const opts: SearchUsersRequest = { bookmark: cursor ?? undefined, query: finalQuery ?? undefined, appId: appId, diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index e2c40a88493..fac8fa61eef 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -109,7 +109,9 @@ export function trimOtherProps(object: any, allowedProps: string[]) { return result } -export function isSupportedUserSearch(query: SearchFilters) { +export function isSupportedUserSearch( + query: SearchFilters +): query is SearchFilters { const allowed = [ { op: BasicOperator.STRING, key: "email" }, { op: BasicOperator.EQUAL, key: "_id" }, From dedf2e58594841fc62cc10897f5208614fec479c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 11:22:22 +0100 Subject: [PATCH 23/63] Fix types --- packages/frontend-core/src/fetch/index.ts | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index ef471aa8e43..4d5b2f147a8 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -10,7 +10,8 @@ import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch" import CustomFetch from "./CustomFetch.js" import QueryArrayFetch from "./QueryArrayFetch.js" -import { Table, UIDatasource, UIFetchAPI } from "@budibase/types" +import { Table, UIDatasource } from "@budibase/types" +import { APIClient } from "../api/types.js" const DataFetchMap = { table: TableFetch, @@ -30,15 +31,7 @@ const DataFetchMap = { } // Constructs a new fetch model for a certain datasource -export const fetchData = ({ - API, - datasource, - options, -}: { - API: UIFetchAPI - datasource: UIDatasource - options: {} -}) => { +export const fetchData = ({ API, datasource, options }: any) => { const Fetch = DataFetchMap[datasource?.type as keyof typeof DataFetchMap] || TableFetch return new Fetch({ API, datasource, ...options }) @@ -50,14 +43,14 @@ const createEmptyFetchInstance = ({ API, datasource, }: { - API: UIFetchAPI - datasource: UIDatasource + API: APIClient + datasource: any }) => { const handler = DataFetchMap[datasource?.type as keyof typeof DataFetchMap] if (!handler) { return null } - return new handler({ API }) + return new handler({ API, datasource: null as any, query: null as any }) } // Fetches the definition of any type of datasource @@ -65,7 +58,7 @@ export const getDatasourceDefinition = async ({ API, datasource, }: { - API: UIFetchAPI + API: APIClient datasource: UIDatasource }) => { const instance = createEmptyFetchInstance({ API, datasource }) @@ -78,7 +71,7 @@ export const getDatasourceSchema = ({ datasource, definition, }: { - API: UIFetchAPI + API: APIClient datasource: UIDatasource definition: Table }) => { From 0eddc1a00eb1819800471eabed148adbefbebb96 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 11:47:10 +0100 Subject: [PATCH 24/63] Type QueryFetch --- packages/client/src/utils/schema.js | 2 +- packages/frontend-core/src/fetch/DataFetch.ts | 11 ++++++-- .../fetch/{QueryFetch.js => QueryFetch.ts} | 28 +++++++++++++++---- packages/frontend-core/src/fetch/index.ts | 2 +- 4 files changed, 34 insertions(+), 9 deletions(-) rename packages/frontend-core/src/fetch/{QueryFetch.js => QueryFetch.ts} (82%) diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index 87859f1ef8d..3a1b5acaaad 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -1,7 +1,7 @@ import { API } from "api" import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch" import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch" -import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch.js" +import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch" import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch" import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js" import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch" diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index ef05f7b8efb..60cb96cf8c1 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -40,7 +40,10 @@ interface DataFetchDerivedStore supportsPagination: boolean } -interface DataFetchParams { +export interface DataFetchParams< + TDatasource, + TQuery = SearchFilters | undefined +> { API: APIClient datasource: TDatasource query: TQuery @@ -411,7 +414,11 @@ export default abstract class DataFetch< * Determine the feature flag for this datasource definition * @param definition */ - determineFeatureFlags(_definition: TDefinition | null) { + determineFeatureFlags(_definition: TDefinition | null): { + supportsPagination: boolean + supportsSearch?: boolean + supportsSort?: boolean + } { return { supportsSearch: false, supportsSort: false, diff --git a/packages/frontend-core/src/fetch/QueryFetch.js b/packages/frontend-core/src/fetch/QueryFetch.ts similarity index 82% rename from packages/frontend-core/src/fetch/QueryFetch.js rename to packages/frontend-core/src/fetch/QueryFetch.ts index 4f7954c0687..c671781fcd1 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.js +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -1,9 +1,21 @@ import DataFetch from "./DataFetch" import { Helpers } from "@budibase/bbui" +import { Query } from "@budibase/types" import { get } from "svelte/store" -export default class QueryFetch extends DataFetch { - determineFeatureFlags(definition) { +interface QueryDatasource { + _id: string + fields: any + queryParams: any + parameters: any +} + +export default class QueryFetch extends DataFetch { + getSchema(_datasource: any, definition: any) { + return definition?.schema + } + + determineFeatureFlags(definition: Query) { const supportsPagination = !!definition?.fields?.pagination?.type && !!definition?.fields?.pagination?.location && @@ -11,7 +23,7 @@ export default class QueryFetch extends DataFetch { return { supportsPagination } } - async getDefinition(datasource) { + async getDefinition(datasource: QueryDatasource) { if (!datasource?._id) { return null } @@ -48,9 +60,15 @@ export default class QueryFetch extends DataFetch { } // Add pagination to query if supported - let queryPayload = { parameters } + const queryPayload: { + parameters: any + pagination?: { + page: number | null + limit: number + } + } = { parameters } if (paginate && supportsPagination) { - const requestCursor = type === "page" ? parseInt(cursor || 1) : cursor + const requestCursor = type === "page" ? parseInt(cursor || "1") : cursor queryPayload.pagination = { page: requestCursor, limit } } diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 4d5b2f147a8..fc029324bac 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -1,7 +1,7 @@ import TableFetch from "./TableFetch.js" import ViewFetch from "./ViewFetch.js" import ViewV2Fetch from "./ViewV2Fetch.js" -import QueryFetch from "./QueryFetch.js" +import QueryFetch from "./QueryFetch" import RelationshipFetch from "./RelationshipFetch" import NestedProviderFetch from "./NestedProviderFetch.js" import FieldFetch from "./FieldFetch" From 8c0e7a12d681c3c95aaba9f61e3b2cad166ad2b4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 11:54:42 +0100 Subject: [PATCH 25/63] Remove anys --- packages/server/src/api/controllers/query/index.ts | 4 ++-- packages/types/src/documents/app/query.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 7084d065fac..b522008cad5 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -355,7 +355,7 @@ async function execute( ExecuteQueryRequest, ExecuteV2QueryResponse | ExecuteV1QueryResponse >, - opts: any = { rowsOnly: false, isAutomation: false } + opts = { rowsOnly: false, isAutomation: false } ) { const db = context.getAppDB() @@ -416,7 +416,7 @@ export async function executeV1( export async function executeV2( ctx: UserCtx ) { - return execute(ctx, { rowsOnly: false }) + return execute(ctx, { rowsOnly: false, isAutomation: false }) } export async function executeV2AsAutomation( diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index a545ca144ef..43e7563b31f 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -1,4 +1,5 @@ import { Document } from "../document" +import { Row } from "./row" export interface QuerySchema { name?: string @@ -29,7 +30,7 @@ export interface QueryParameter { } export interface QueryResponse { - rows: any[] + rows: Row[] keys: string[] info: any extra: any From f4ed2176c96bd62cad45afcdbc68cd40eac29a2a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 12:06:37 +0100 Subject: [PATCH 26/63] Fix types --- packages/frontend-core/src/fetch/QueryFetch.ts | 2 +- packages/types/src/api/web/app/query.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts index c671781fcd1..d85b5ffbcd4 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.ts +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -83,7 +83,7 @@ export default class QueryFetch extends DataFetch { if (paginate && supportsPagination) { if (type === "page") { // For "page number" pagination, increment the existing page number - nextCursor = queryPayload.pagination.page + 1 + nextCursor = queryPayload.pagination!.page! + 1 hasNextPage = data?.length === limit && limit > 0 } else { // For "cursor" pagination, the cursor should be in the response diff --git a/packages/types/src/api/web/app/query.ts b/packages/types/src/api/web/app/query.ts index 302f0d03e5d..75cc37e1a99 100644 --- a/packages/types/src/api/web/app/query.ts +++ b/packages/types/src/api/web/app/query.ts @@ -40,6 +40,10 @@ export interface ExecuteQueryRequest { export type ExecuteV1QueryResponse = Record[] export interface ExecuteV2QueryResponse { data: Record[] + pagination?: { + page: number + cursor: string + } } export interface DeleteQueryResponse { From 89485aa9d11aca31163e97e971a9579959b5e3e8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 12:39:40 +0100 Subject: [PATCH 27/63] Extend types --- packages/frontend-core/src/fetch/FieldFetch.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/FieldFetch.ts b/packages/frontend-core/src/fetch/FieldFetch.ts index 7f0dfdf3318..5e89d6e8810 100644 --- a/packages/frontend-core/src/fetch/FieldFetch.ts +++ b/packages/frontend-core/src/fetch/FieldFetch.ts @@ -1,7 +1,8 @@ import { Row, TableSchema } from "@budibase/types" import DataFetch from "./DataFetch" -interface FieldDatasource { +export interface FieldDatasource { + tableId: string fieldType: "attachment" | "array" value: string[] | Row[] } From 364e997fc2f338d051cfd32c36f3a066c4323375 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 12:41:37 +0100 Subject: [PATCH 28/63] Type jsonArrayFetch --- packages/client/src/utils/schema.js | 2 +- packages/frontend-core/src/fetch/FieldFetch.ts | 12 +++++++++--- .../fetch/{JSONArrayFetch.js => JSONArrayFetch.ts} | 9 ++++++--- packages/frontend-core/src/fetch/index.ts | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) rename packages/frontend-core/src/fetch/{JSONArrayFetch.js => JSONArrayFetch.ts} (63%) diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index 3a1b5acaaad..f9cd7dbc60d 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -5,7 +5,7 @@ import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch" import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch" import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js" import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch" -import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js" +import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch" import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch" import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch" diff --git a/packages/frontend-core/src/fetch/FieldFetch.ts b/packages/frontend-core/src/fetch/FieldFetch.ts index 5e89d6e8810..6809dee00eb 100644 --- a/packages/frontend-core/src/fetch/FieldFetch.ts +++ b/packages/frontend-core/src/fetch/FieldFetch.ts @@ -7,13 +7,17 @@ export interface FieldDatasource { value: string[] | Row[] } +export interface FieldDefinition { + schema?: Record | null +} + function isArrayOfStrings(value: string[] | Row[]): value is string[] { return Array.isArray(value) && !!value[0] && typeof value[0] !== "object" } export default class FieldFetch extends DataFetch< FieldDatasource, - { schema?: Record } + FieldDefinition > { getSchema( _datasource: FieldDatasource, @@ -22,9 +26,11 @@ export default class FieldFetch extends DataFetch< return definition?.schema } - async getDefinition(datasource: FieldDatasource) { + async getDefinition( + datasource: FieldDatasource + ): Promise { // Field sources have their schema statically defined - let schema: Record | undefined + let schema if (datasource.fieldType === "attachment") { schema = { url: { diff --git a/packages/frontend-core/src/fetch/JSONArrayFetch.js b/packages/frontend-core/src/fetch/JSONArrayFetch.ts similarity index 63% rename from packages/frontend-core/src/fetch/JSONArrayFetch.js rename to packages/frontend-core/src/fetch/JSONArrayFetch.ts index f7de74a4b87..a254bc3ae48 100644 --- a/packages/frontend-core/src/fetch/JSONArrayFetch.js +++ b/packages/frontend-core/src/fetch/JSONArrayFetch.ts @@ -1,13 +1,16 @@ -import FieldFetch from "./FieldFetch" +import FieldFetch, { FieldDatasource } from "./FieldFetch" import { getJSONArrayDatasourceSchema } from "../utils/json" export default class JSONArrayFetch extends FieldFetch { - async getDefinition(datasource) { + async getDefinition(datasource: FieldDatasource) { // JSON arrays need their table definitions fetched. // We can then extract their schema as a subset of the table schema. try { const table = await this.API.fetchTableDefinition(datasource.tableId) - const schema = getJSONArrayDatasourceSchema(table?.schema, datasource) + const schema: Record | null = getJSONArrayDatasourceSchema( + table?.schema, + datasource + ) return { schema } } catch (error) { return null diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index fc029324bac..3976adc3d90 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -5,7 +5,7 @@ import QueryFetch from "./QueryFetch" import RelationshipFetch from "./RelationshipFetch" import NestedProviderFetch from "./NestedProviderFetch.js" import FieldFetch from "./FieldFetch" -import JSONArrayFetch from "./JSONArrayFetch.js" +import JSONArrayFetch from "./JSONArrayFetch" import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch" import CustomFetch from "./CustomFetch.js" From 690a442e61106ad250cd7838772ec6072f229722 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 13:08:17 +0100 Subject: [PATCH 29/63] convert QueryArrayFetch --- .../fetch/{QueryArrayFetch.js => QueryArrayFetch.ts} | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) rename packages/frontend-core/src/fetch/{QueryArrayFetch.js => QueryArrayFetch.ts} (68%) diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.js b/packages/frontend-core/src/fetch/QueryArrayFetch.ts similarity index 68% rename from packages/frontend-core/src/fetch/QueryArrayFetch.js rename to packages/frontend-core/src/fetch/QueryArrayFetch.ts index 222697b78fa..15f86ae7fc0 100644 --- a/packages/frontend-core/src/fetch/QueryArrayFetch.js +++ b/packages/frontend-core/src/fetch/QueryArrayFetch.ts @@ -1,11 +1,11 @@ -import FieldFetch from "./FieldFetch" +import FieldFetch, { FieldDatasource } from "./FieldFetch" import { getJSONArrayDatasourceSchema, generateQueryArraySchemas, } from "../utils/json" export default class QueryArrayFetch extends FieldFetch { - async getDefinition(datasource) { + async getDefinition(datasource: FieldDatasource) { if (!datasource?.tableId) { return null } @@ -17,7 +17,11 @@ export default class QueryArrayFetch extends FieldFetch { table?.schema, table?.nestedSchemaFields ) - return { schema: getJSONArrayDatasourceSchema(schema, datasource) } + const result: { + schema: Record | null + } = { schema: getJSONArrayDatasourceSchema(schema, datasource) } + + return result } catch (error) { return null } From f76ec8d2c9602960e84fe8858dcfa5156c3595c1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 13:32:44 +0100 Subject: [PATCH 30/63] Fix types --- packages/server/src/threads/definitions.ts | 5 ++++- packages/types/src/documents/app/query.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index 85e546280d1..44a76a60a38 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -3,7 +3,10 @@ import { Datasource, Row, Query } from "@budibase/types" export type WorkerCallback = (error: any, response?: any) => void export interface QueryEvent - extends Omit { + extends Omit< + Query, + "datasourceId" | "name" | "parameters" | "readable" | "nestedSchemaFields" + > { appId?: string datasource: Datasource pagination?: any diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index 43e7563b31f..477a7787b91 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -14,6 +14,7 @@ export interface Query extends Document { fields: RestQueryFields | any transformer: string | null schema: Record + nestedSchemaFields: Record> readable: boolean queryVerb: string // flag to state whether the default bindings are empty strings (old behaviour) or null From 91300c54e903bf4f925a778011bb756092873286 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 13:34:42 +0100 Subject: [PATCH 31/63] Type nestedProvider --- packages/client/src/utils/schema.js | 2 +- .../{NestedProviderFetch.js => NestedProviderFetch.ts} | 8 ++++++-- packages/frontend-core/src/fetch/index.ts | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) rename packages/frontend-core/src/fetch/{NestedProviderFetch.js => NestedProviderFetch.ts} (69%) diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index f9cd7dbc60d..ffab142cf3b 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -3,7 +3,7 @@ import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch" import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch" import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch" import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch" -import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch.js" +import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch" import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch" import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch" import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch" diff --git a/packages/frontend-core/src/fetch/NestedProviderFetch.js b/packages/frontend-core/src/fetch/NestedProviderFetch.ts similarity index 69% rename from packages/frontend-core/src/fetch/NestedProviderFetch.js rename to packages/frontend-core/src/fetch/NestedProviderFetch.ts index 06c74cb6c45..a442e7ec073 100644 --- a/packages/frontend-core/src/fetch/NestedProviderFetch.js +++ b/packages/frontend-core/src/fetch/NestedProviderFetch.ts @@ -1,7 +1,11 @@ import DataFetch from "./DataFetch" -export default class NestedProviderFetch extends DataFetch { - async getDefinition(datasource) { +export default class NestedProviderFetch extends DataFetch { + getSchema(_datasource: any, definition: any) { + return definition?.schema + } + + async getDefinition(datasource: any) { // Nested providers should already have exposed their own schema return { schema: datasource?.value?.schema, diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 3976adc3d90..dd03e715ed1 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -3,7 +3,7 @@ import ViewFetch from "./ViewFetch.js" import ViewV2Fetch from "./ViewV2Fetch.js" import QueryFetch from "./QueryFetch" import RelationshipFetch from "./RelationshipFetch" -import NestedProviderFetch from "./NestedProviderFetch.js" +import NestedProviderFetch from "./NestedProviderFetch" import FieldFetch from "./FieldFetch" import JSONArrayFetch from "./JSONArrayFetch" import UserFetch from "./UserFetch.js" From bf02515ff0c546440ce9a9c16e0f5e7cfe304459 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 13:42:51 +0100 Subject: [PATCH 32/63] Fix types --- .../frontend-core/src/components/grid/stores/rows.ts | 7 ++++--- packages/frontend-core/src/fetch/DataFetch.ts | 11 +++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/rows.ts b/packages/frontend-core/src/components/grid/stores/rows.ts index d6b80df8854..b9c9b3fe1eb 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.ts +++ b/packages/frontend-core/src/components/grid/stores/rows.ts @@ -10,9 +10,10 @@ import { import { tick } from "svelte" import { Helpers } from "@budibase/bbui" import { sleep } from "../../../utils/utils" -import { FieldType, Row, UIFetchAPI, UIRow } from "@budibase/types" +import { FieldType, Row, UIRow } from "@budibase/types" import { getRelatedTableValues } from "../../../utils" import { Store as StoreContext } from "." +import DataFetch from "../../../fetch/DataFetch" interface IndexedUIRow extends UIRow { __idx: number @@ -20,7 +21,7 @@ interface IndexedUIRow extends UIRow { interface RowStore { rows: Writable - fetch: Writable + fetch: Writable | null> loaded: Writable refreshing: Writable loading: Writable @@ -225,7 +226,7 @@ export const createActions = (context: StoreContext): RowActionStore => { }) // Subscribe to changes of this fetch model - unsubscribe = newFetch.subscribe(async ($fetch: UIFetchAPI) => { + unsubscribe = newFetch.subscribe(async $fetch => { if ($fetch.error) { // Present a helpful error to the user let message = "An unknown error occurred" diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 60cb96cf8c1..68cff150401 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -26,8 +26,11 @@ interface DataFetchStore { pageNumber: number cursor: null cursors: any[] - resetKey: number - error: null + resetKey: string + error: { + message: string + status: number + } | null definition?: TDefinition | null } @@ -132,7 +135,7 @@ export default abstract class DataFetch< pageNumber: 0, cursor: null, cursors: [], - resetKey: Math.random(), + resetKey: Math.random().toString(), error: null, }) @@ -284,7 +287,7 @@ export default abstract class DataFetch< info: page.info, cursors: paginate && page.hasNextPage ? [null, page.cursor] : [null], error: page.error, - resetKey: Math.random(), + resetKey: Math.random().toString(), })) } From 30cf6ff2ad2332cca6d54a9375c1ebcf6b842ef4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 13:46:00 +0100 Subject: [PATCH 33/63] Type customFetch --- .../fetch/{CustomFetch.js => CustomFetch.ts} | 22 +++++++++++-------- packages/frontend-core/src/fetch/index.ts | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) rename packages/frontend-core/src/fetch/{CustomFetch.js => CustomFetch.ts} (89%) diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.ts similarity index 89% rename from packages/frontend-core/src/fetch/CustomFetch.js rename to packages/frontend-core/src/fetch/CustomFetch.ts index 39d3bd6f4c6..64739ad6f1e 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.js +++ b/packages/frontend-core/src/fetch/CustomFetch.ts @@ -1,8 +1,12 @@ import DataFetch from "./DataFetch" -export default class CustomFetch extends DataFetch { +export default class CustomFetch extends DataFetch { + getSchema(_datasource: any, definition: any) { + return definition?.schema + } + // Gets the correct Budibase type for a JS value - getType(value) { + getType(value: any) { if (value == null) { return "string" } @@ -22,7 +26,7 @@ export default class CustomFetch extends DataFetch { } // Parses the custom data into an array format - parseCustomData(data) { + parseCustomData(data: any) { if (!data) { return [] } @@ -55,7 +59,7 @@ export default class CustomFetch extends DataFetch { } // Enriches the custom data to ensure the structure and format is usable - enrichCustomData(data) { + enrichCustomData(data: any[]) { if (!data?.length) { return [] } @@ -72,7 +76,7 @@ export default class CustomFetch extends DataFetch { // Try parsing strings if (typeof value === "string") { const split = value.split(",").map(x => x.trim()) - let obj = {} + let obj: Record = {} for (let i = 0; i < split.length; i++) { const suffix = i === 0 ? "" : ` ${i + 1}` const key = `Value${suffix}` @@ -87,13 +91,13 @@ export default class CustomFetch extends DataFetch { } // Extracts and parses the custom data from the datasource definition - getCustomData(datasource) { + getCustomData(datasource: { data: any }) { return this.enrichCustomData(this.parseCustomData(datasource?.data)) } - async getDefinition(datasource) { + async getDefinition(datasource: any) { // Try and work out the schema from the array provided - let schema = {} + let schema: any = {} const data = this.getCustomData(datasource) if (!data?.length) { return { schema } @@ -107,7 +111,7 @@ export default class CustomFetch extends DataFetch { } if (!schema[key]) { let type = this.getType(datum[key]) - let constraints = {} + let constraints: any = {} // Determine whether we should render text columns as options instead if (type === "string") { diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index dd03e715ed1..3afeec76845 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -8,7 +8,7 @@ import FieldFetch from "./FieldFetch" import JSONArrayFetch from "./JSONArrayFetch" import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch" -import CustomFetch from "./CustomFetch.js" +import CustomFetch from "./CustomFetch" import QueryArrayFetch from "./QueryArrayFetch.js" import { Table, UIDatasource } from "@budibase/types" import { APIClient } from "../api/types.js" From 090429fc56b242f931b61275377ad77e09ed7e0b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 14:01:55 +0100 Subject: [PATCH 34/63] Fix sort order enum type --- packages/frontend-core/src/fetch/DataFetch.ts | 5 +++++ packages/frontend-core/src/fetch/TableFetch.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 68cff150401..0dcda4e150f 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -253,9 +253,14 @@ export default abstract class DataFetch< ) { this.options.sortType = SortType.NUMBER } + // If no sort order, default to ascending if (!this.options.sortOrder) { this.options.sortOrder = SortOrder.ASCENDING + } else { + // Ensure sortOrder matches the enum + this.options.sortOrder = + this.options.sortOrder.toLowerCase() as SortOrder } } diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 344ce30e546..58ad554d6ae 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -42,7 +42,7 @@ export default class TableFetch extends DataFetch { query: query ?? undefined, limit, sort: sortColumn, - sortOrder: sortOrder?.toLowerCase() ?? SortOrder.ASCENDING, + sortOrder: sortOrder ?? SortOrder.ASCENDING, sortType, paginate, bookmark: cursor, From dcf52b5dc96670ad0cfc31750f059110cf42b231 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 15:33:05 +0100 Subject: [PATCH 35/63] Fix typings --- packages/frontend-core/src/components/grid/stores/config.ts | 2 +- .../frontend-core/src/components/grid/stores/datasource.ts | 6 +++--- packages/frontend-core/src/fetch/index.ts | 4 ++-- packages/types/src/ui/stores/grid/view.ts | 3 +-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/config.ts b/packages/frontend-core/src/components/grid/stores/config.ts index e334b58495e..2ddaf1b65c9 100644 --- a/packages/frontend-core/src/components/grid/stores/config.ts +++ b/packages/frontend-core/src/components/grid/stores/config.ts @@ -69,7 +69,7 @@ export const deriveStores = (context: StoreContext): ConfigDerivedStore => { } // Disable features for non DS+ - if (!["table", "viewV2"].includes(type)) { + if (type && !["table", "viewV2"].includes(type)) { config.canAddRows = false config.canEditRows = false config.canDeleteRows = false diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index 74101701edc..fe8ff60fd30 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -74,7 +74,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { let schema: Record = getDatasourceSchema({ API, datasource: get(datasource), - definition: $definition, + definition: $definition ?? undefined, }) if (!schema) { return null @@ -136,7 +136,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) { return false } - return ["table", "viewV2", "link"].includes(type) + return !!type && ["table", "viewV2", "link"].includes(type) } ) @@ -186,7 +186,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { API, datasource: get(datasource), }) - definition.set(def) + definition.set((def as any) ?? null) } // Saves the datasource definition diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 3afeec76845..4f4eafe8e86 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -10,7 +10,7 @@ import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch" import CustomFetch from "./CustomFetch" import QueryArrayFetch from "./QueryArrayFetch.js" -import { Table, UIDatasource } from "@budibase/types" +import { TableSchema, UIDatasource } from "@budibase/types" import { APIClient } from "../api/types.js" const DataFetchMap = { @@ -73,7 +73,7 @@ export const getDatasourceSchema = ({ }: { API: APIClient datasource: UIDatasource - definition: Table + definition?: { schema?: TableSchema } }) => { const instance = createEmptyFetchInstance({ API, datasource }) return instance?.getSchema(datasource, definition) diff --git a/packages/types/src/ui/stores/grid/view.ts b/packages/types/src/ui/stores/grid/view.ts index 270faaa1606..f81cc34aaf5 100644 --- a/packages/types/src/ui/stores/grid/view.ts +++ b/packages/types/src/ui/stores/grid/view.ts @@ -1,7 +1,6 @@ import { ViewV2 } from "@budibase/types" import { UIFieldSchema } from "./table" -export interface UIView extends Omit { - type: string +export interface UIView extends ViewV2 { schema: Record } From ed2e35dea06285cdf7677418cb676a3add90189f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 7 Jan 2025 16:02:34 +0100 Subject: [PATCH 36/63] Dry --- .../src/components/grid/stores/datasource.ts | 4 ++-- packages/frontend-core/src/fetch/CustomFetch.ts | 4 ---- packages/frontend-core/src/fetch/DataFetch.ts | 12 +++++++----- packages/frontend-core/src/fetch/FieldFetch.ts | 9 +-------- packages/frontend-core/src/fetch/GroupUserFetch.ts | 4 ---- .../frontend-core/src/fetch/NestedProviderFetch.ts | 4 ---- packages/frontend-core/src/fetch/QueryFetch.ts | 4 ---- .../frontend-core/src/fetch/RelationshipFetch.ts | 4 ---- packages/frontend-core/src/fetch/TableFetch.ts | 4 ---- packages/frontend-core/src/fetch/UserFetch.ts | 4 ---- packages/frontend-core/src/fetch/ViewV2Fetch.ts | 4 ---- 11 files changed, 10 insertions(+), 47 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index fe8ff60fd30..4c20e9493f3 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -71,7 +71,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { } = context const schema = derived(definition, $definition => { - let schema: Record = getDatasourceSchema({ + const schema: Record | null | undefined = getDatasourceSchema({ API, datasource: get(datasource), definition: $definition ?? undefined, @@ -82,7 +82,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { // Ensure schema is configured as objects. // Certain datasources like queries use primitives. - Object.keys(schema || {}).forEach(key => { + Object.keys(schema).forEach(key => { if (typeof schema[key] !== "object") { schema[key] = { name: key, type: schema[key] } } diff --git a/packages/frontend-core/src/fetch/CustomFetch.ts b/packages/frontend-core/src/fetch/CustomFetch.ts index 64739ad6f1e..9db9a935a5f 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.ts +++ b/packages/frontend-core/src/fetch/CustomFetch.ts @@ -1,10 +1,6 @@ import DataFetch from "./DataFetch" export default class CustomFetch extends DataFetch { - getSchema(_datasource: any, definition: any) { - return definition?.schema - } - // Gets the correct Budibase type for a JS value getType(value: any) { if (value == null) { diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 0dcda4e150f..f34f6ddeb78 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -60,7 +60,10 @@ export interface DataFetchParams< */ export default abstract class DataFetch< TDatasource extends {}, - TDefinition extends {}, + TDefinition extends { + schema?: Record | null + primaryDisplay?: string + }, TQuery extends {} = SearchFilters > { API: APIClient @@ -361,10 +364,9 @@ export default abstract class DataFetch< * @param definition the datasource definition * @return {object} the schema */ - abstract getSchema( - datasource: TDatasource | null, - definition: TDefinition | null - ): any + getSchema(_datasource: TDatasource | null, definition: TDefinition | null) { + return definition?.schema + } /** * Enriches a datasource schema with nested fields and ensures the structure diff --git a/packages/frontend-core/src/fetch/FieldFetch.ts b/packages/frontend-core/src/fetch/FieldFetch.ts index 6809dee00eb..636fb63f3d6 100644 --- a/packages/frontend-core/src/fetch/FieldFetch.ts +++ b/packages/frontend-core/src/fetch/FieldFetch.ts @@ -1,4 +1,4 @@ -import { Row, TableSchema } from "@budibase/types" +import { Row } from "@budibase/types" import DataFetch from "./DataFetch" export interface FieldDatasource { @@ -19,13 +19,6 @@ export default class FieldFetch extends DataFetch< FieldDatasource, FieldDefinition > { - getSchema( - _datasource: FieldDatasource, - definition: { schema?: TableSchema } - ) { - return definition?.schema - } - async getDefinition( datasource: FieldDatasource ): Promise { diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.ts b/packages/frontend-core/src/fetch/GroupUserFetch.ts index 77b5e1de438..0c5ac7486de 100644 --- a/packages/frontend-core/src/fetch/GroupUserFetch.ts +++ b/packages/frontend-core/src/fetch/GroupUserFetch.ts @@ -26,10 +26,6 @@ export default class GroupUserFetch extends DataFetch< }) } - getSchema(_datasource: any, definition: any) { - return definition?.schema - } - determineFeatureFlags() { return { supportsSearch: true, diff --git a/packages/frontend-core/src/fetch/NestedProviderFetch.ts b/packages/frontend-core/src/fetch/NestedProviderFetch.ts index a442e7ec073..71eb5177dbf 100644 --- a/packages/frontend-core/src/fetch/NestedProviderFetch.ts +++ b/packages/frontend-core/src/fetch/NestedProviderFetch.ts @@ -1,10 +1,6 @@ import DataFetch from "./DataFetch" export default class NestedProviderFetch extends DataFetch { - getSchema(_datasource: any, definition: any) { - return definition?.schema - } - async getDefinition(datasource: any) { // Nested providers should already have exposed their own schema return { diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts index d85b5ffbcd4..dec7cd21839 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.ts +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -11,10 +11,6 @@ interface QueryDatasource { } export default class QueryFetch extends DataFetch { - getSchema(_datasource: any, definition: any) { - return definition?.schema - } - determineFeatureFlags(definition: Query) { const supportsPagination = !!definition?.fields?.pagination?.type && diff --git a/packages/frontend-core/src/fetch/RelationshipFetch.ts b/packages/frontend-core/src/fetch/RelationshipFetch.ts index ab50285b4b0..7b6e93fbcc1 100644 --- a/packages/frontend-core/src/fetch/RelationshipFetch.ts +++ b/packages/frontend-core/src/fetch/RelationshipFetch.ts @@ -12,10 +12,6 @@ export default class RelationshipFetch extends DataFetch< RelationshipDatasource, Table > { - getSchema(_datasource: any, definition: any) { - return definition?.schema - } - async getDefinition(datasource: RelationshipDatasource) { if (!datasource?.tableId) { return null diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 58ad554d6ae..48a3413716e 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -26,10 +26,6 @@ export default class TableFetch extends DataFetch { } } - getSchema(_datasource: UITable | null, definition: Table | null) { - return definition?.schema - } - async getData() { const { datasource, limit, sortColumn, sortOrder, sortType, paginate } = this.options diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index e276af35929..43efc7767f8 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -48,10 +48,6 @@ export default class UserFetch extends DataFetch< } } - getSchema(_datasource: any, definition: Table | null) { - return definition?.schema - } - async getData() { const { limit, paginate } = this.options const { cursor, query } = get(this.store) diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 7973dbf298e..d541a692b67 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -12,10 +12,6 @@ export default class ViewV2Fetch extends DataFetch { } } - getSchema(_datasource: UIView, definition: ViewV2) { - return definition?.schema - } - async getDefinition(datasource: UIView | null): Promise { if (!datasource?.id) { return null From a1ac0ac0b08c8595b78d031ca9299722fe1ef5a9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 12:22:17 +0100 Subject: [PATCH 37/63] Fix type --- packages/types/src/documents/app/query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts index 477a7787b91..d287a5b2c31 100644 --- a/packages/types/src/documents/app/query.ts +++ b/packages/types/src/documents/app/query.ts @@ -14,7 +14,7 @@ export interface Query extends Document { fields: RestQueryFields | any transformer: string | null schema: Record - nestedSchemaFields: Record> + nestedSchemaFields?: Record> readable: boolean queryVerb: string // flag to state whether the default bindings are empty strings (old behaviour) or null From 022df7cda41710e276f98aa12a1bfd240c8b1dba Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 12:28:36 +0100 Subject: [PATCH 38/63] Lint --- packages/frontend-core/src/fetch/ViewFetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts index cbdd1d425ba..fe891b9e66f 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.ts +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -34,7 +34,7 @@ export default class ViewFetch extends DataFetch { }) return { rows: res || [] } } catch (error) { - console.error(error) + console.error(error, { datasource }) return { rows: [] } } } From d465f7e0574297e8bb01a41b6cd757986fc82a41 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 12:42:33 +0100 Subject: [PATCH 39/63] Type anys --- packages/frontend-core/src/api/views.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/api/views.ts b/packages/frontend-core/src/api/views.ts index 083a3890e80..83f7e97df07 100644 --- a/packages/frontend-core/src/api/views.ts +++ b/packages/frontend-core/src/api/views.ts @@ -3,7 +3,15 @@ import { BaseAPIClient } from "./types" export interface ViewEndpoints { // Missing request or response types - fetchViewData: (name: string, opts?: any) => Promise + fetchViewData: ( + name: string, + opts: { + calculation?: string + field?: string + groupBy?: string + tableId: string + } + ) => Promise exportView: (name: string, format: string) => Promise saveView: (view: any) => Promise deleteView: (name: string) => Promise @@ -20,7 +28,7 @@ export const buildViewEndpoints = (API: BaseAPIClient): ViewEndpoints => ({ fetchViewData: async (name, { field, groupBy, calculation }) => { const params = new URLSearchParams() if (calculation) { - params.set("field", field) + params.set("field", field!) params.set("calculation", calculation) } if (groupBy) { From 7d7c27fa928640d075ce9233932b2761e6d740d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 12:49:17 +0100 Subject: [PATCH 40/63] Simplify determineFeatureFlags --- packages/frontend-core/src/fetch/DataFetch.ts | 9 ++++----- packages/frontend-core/src/fetch/GroupUserFetch.ts | 2 +- packages/frontend-core/src/fetch/QueryFetch.ts | 3 ++- packages/frontend-core/src/fetch/TableFetch.ts | 2 +- packages/frontend-core/src/fetch/UserFetch.ts | 2 +- packages/frontend-core/src/fetch/ViewV2Fetch.ts | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index f34f6ddeb78..a7ed7237bab 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -216,7 +216,7 @@ export default abstract class DataFetch< const definition = await this.getDefinition(datasource) // Determine feature flags - const features = this.determineFeatureFlags(definition) + const features = await this.determineFeatureFlags() this.features = { supportsSearch: !!features?.supportsSearch, supportsSort: !!features?.supportsSort, @@ -421,14 +421,13 @@ export default abstract class DataFetch< } /** - * Determine the feature flag for this datasource definition - * @param definition + * Determine the feature flag for this datasource */ - determineFeatureFlags(_definition: TDefinition | null): { + async determineFeatureFlags(): Promise<{ supportsPagination: boolean supportsSearch?: boolean supportsSort?: boolean - } { + }> { return { supportsSearch: false, supportsSort: false, diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.ts b/packages/frontend-core/src/fetch/GroupUserFetch.ts index 0c5ac7486de..2f5cacd1a2b 100644 --- a/packages/frontend-core/src/fetch/GroupUserFetch.ts +++ b/packages/frontend-core/src/fetch/GroupUserFetch.ts @@ -26,7 +26,7 @@ export default class GroupUserFetch extends DataFetch< }) } - determineFeatureFlags() { + async determineFeatureFlags() { return { supportsSearch: true, supportsSort: false, diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts index dec7cd21839..a6ddcd8f016 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.ts +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -11,7 +11,8 @@ interface QueryDatasource { } export default class QueryFetch extends DataFetch { - determineFeatureFlags(definition: Query) { + async determineFeatureFlags() { + const definition = await this.getDefinition(this.options.datasource) const supportsPagination = !!definition?.fields?.pagination?.type && !!definition?.fields?.pagination?.location && diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 48a3413716e..433de69b59f 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -3,7 +3,7 @@ import DataFetch from "./DataFetch" import { SortOrder, Table, UITable } from "@budibase/types" export default class TableFetch extends DataFetch { - determineFeatureFlags() { + async determineFeatureFlags() { return { supportsSearch: true, supportsSort: true, diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index 43efc7767f8..199dacde009 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -34,7 +34,7 @@ export default class UserFetch extends DataFetch< }) } - determineFeatureFlags() { + async determineFeatureFlags() { return { supportsSearch: true, supportsSort: false, diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index d541a692b67..74ad08f2f48 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -4,7 +4,7 @@ import { get } from "svelte/store" import { helpers } from "@budibase/shared-core" export default class ViewV2Fetch extends DataFetch { - determineFeatureFlags() { + async determineFeatureFlags() { return { supportsSearch: true, supportsSort: true, From f053438a649694233f7849e94df0bdd22ab60859 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 12:55:26 +0100 Subject: [PATCH 41/63] Clean classes --- packages/frontend-core/src/fetch/DataFetch.ts | 18 +++++++----------- .../frontend-core/src/fetch/ViewV2Fetch.ts | 5 +---- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index a7ed7237bab..3e2b0d7dcfb 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -223,13 +223,13 @@ export default abstract class DataFetch< supportsPagination: paginate && !!features?.supportsPagination, } - // Fetch and enrich schema - let schema = this.getSchema(datasource, definition) ?? null - schema = this.enrichSchema(schema) - if (!schema) { + if (!definition?.schema) { return } + // Fetch and enrich schema + const schema = this.enrichSchema(definition.schema) + // If an invalid sort column is specified, delete it if (this.options.sortColumn && !schema[this.options.sortColumn]) { this.options.sortColumn = null @@ -374,11 +374,7 @@ export default abstract class DataFetch< * @param schema the datasource schema * @return {object} the enriched datasource schema */ - enrichSchema(schema: TableSchema | null): TableSchema | null { - if (schema == null) { - return null - } - + private enrichSchema(schema: TableSchema): TableSchema { // Check for any JSON fields so we can add any top level properties let jsonAdditions: Record = {} for (const fieldKey of Object.keys(schema)) { @@ -510,7 +506,7 @@ export default abstract class DataFetch< * @param state the current store state * @return {boolean} whether there is a next page of data or not */ - hasNextPage(state: DataFetchStore): boolean { + private hasNextPage(state: DataFetchStore): boolean { return state.cursors[state.pageNumber + 1] != null } @@ -520,7 +516,7 @@ export default abstract class DataFetch< * @param state the current store state * @return {boolean} whether there is a previous page of data or not */ - hasPrevPage(state: { pageNumber: number }): boolean { + private hasPrevPage(state: { pageNumber: number }): boolean { return state.pageNumber > 0 } diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 74ad08f2f48..197b5b4ae54 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -28,10 +28,7 @@ export default class ViewV2Fetch extends DataFetch { } } - getDefaultSortColumn( - _definition: { primaryDisplay?: string } | null, - _schema: Record - ) { + getDefaultSortColumn() { return null } From 5d63fe251f050e406692b3e302e8be7abcb58cbb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 12:57:25 +0100 Subject: [PATCH 42/63] Simplify --- packages/frontend-core/src/fetch/DataFetch.ts | 5 ++--- packages/frontend-core/src/fetch/ViewFetch.ts | 3 ++- packages/frontend-core/src/fetch/index.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 3e2b0d7dcfb..b5eb774e45f 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -360,12 +360,11 @@ export default abstract class DataFetch< /** * Gets the schema definition for a datasource. - * @param datasource the datasource * @param definition the datasource definition * @return {object} the schema */ - getSchema(_datasource: TDatasource | null, definition: TDefinition | null) { - return definition?.schema + getSchema(definition: TDefinition | null): Record | undefined { + return definition?.schema ?? undefined } /** diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts index fe891b9e66f..2238d226ab7 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.ts +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -19,7 +19,8 @@ export default class ViewFetch extends DataFetch { } } - getSchema(datasource: ViewV1, definition: Table) { + getSchema(definition: Table) { + const { datasource } = this.options return definition?.views?.[datasource.name]?.schema } diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 4f4eafe8e86..52233bd3fb9 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -76,5 +76,5 @@ export const getDatasourceSchema = ({ definition?: { schema?: TableSchema } }) => { const instance = createEmptyFetchInstance({ API, datasource }) - return instance?.getSchema(datasource, definition) + return instance?.getSchema(definition) } From 3741b7144e8b635ce11578fdf597c8f1e758a786 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:05:27 +0100 Subject: [PATCH 43/63] Clean code --- packages/frontend-core/src/fetch/UserFetch.ts | 7 ++----- packages/frontend-core/src/fetch/ViewV2Fetch.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index 199dacde009..b865c32d632 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -23,7 +23,6 @@ export default class UserFetch extends DataFetch< constructor(opts: { API: APIClient datasource: Table - options?: {} query: UserFetchQuery }) { super({ @@ -43,9 +42,7 @@ export default class UserFetch extends DataFetch< } async getDefinition() { - return { - schema: {}, - } + return { schema: {} } } async getData() { @@ -53,7 +50,7 @@ export default class UserFetch extends DataFetch< const { cursor, query } = get(this.store) // Convert old format to new one - we now allow use of the lucene format - const { appId, paginated, ...rest } = query || {} + const { appId, paginated, ...rest } = query const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest) ? rest diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 197b5b4ae54..3bb04d5bc4a 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -62,7 +62,7 @@ export default class ViewV2Fetch extends DataFetch { try { const request = { - ...(query ? { query } : {}), + query, paginate, limit, bookmark: cursor, From 265b22f2b8a93d2f8a00358da206299609baadea Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:23:18 +0100 Subject: [PATCH 44/63] Type query --- packages/bbui/src/helpers.d.ts | 1 + packages/frontend-core/src/fetch/QueryFetch.ts | 16 +++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 packages/bbui/src/helpers.d.ts diff --git a/packages/bbui/src/helpers.d.ts b/packages/bbui/src/helpers.d.ts new file mode 100644 index 00000000000..98c6060590a --- /dev/null +++ b/packages/bbui/src/helpers.d.ts @@ -0,0 +1 @@ +export const cloneDeep: (obj: T) => T diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts index a6ddcd8f016..f8506fefecd 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.ts +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -5,9 +5,15 @@ import { get } from "svelte/store" interface QueryDatasource { _id: string - fields: any - queryParams: any - parameters: any + fields: Record & { + pagination?: { + type: string + location: string + pageParam: string + } + } + queryParams: Record + parameters: { name: string; default: string }[] } export default class QueryFetch extends DataFetch { @@ -49,8 +55,8 @@ export default class QueryFetch extends DataFetch { const type = definition?.fields?.pagination?.type // Set the default query params - let parameters = Helpers.cloneDeep(datasource?.queryParams || {}) - for (let param of datasource?.parameters || {}) { + const parameters = Helpers.cloneDeep(datasource.queryParams) + for (const param of datasource?.parameters || []) { if (!parameters[param.name]) { parameters[param.name] = param.default } From 0112087af15e5b3a903adee612ca33f08fe1d184 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:37:28 +0100 Subject: [PATCH 45/63] Improve typing --- packages/frontend-core/src/fetch/QueryFetch.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts index f8506fefecd..0825d396605 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.ts +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -1,6 +1,6 @@ import DataFetch from "./DataFetch" import { Helpers } from "@budibase/bbui" -import { Query } from "@budibase/types" +import { ExecuteQueryRequest, Query } from "@budibase/types" import { get } from "svelte/store" interface QueryDatasource { @@ -12,7 +12,7 @@ interface QueryDatasource { pageParam: string } } - queryParams: Record + queryParams?: Record parameters: { name: string; default: string }[] } @@ -55,7 +55,7 @@ export default class QueryFetch extends DataFetch { const type = definition?.fields?.pagination?.type // Set the default query params - const parameters = Helpers.cloneDeep(datasource.queryParams) + const parameters = Helpers.cloneDeep(datasource.queryParams || {}) for (const param of datasource?.parameters || []) { if (!parameters[param.name]) { parameters[param.name] = param.default @@ -63,13 +63,7 @@ export default class QueryFetch extends DataFetch { } // Add pagination to query if supported - const queryPayload: { - parameters: any - pagination?: { - page: number | null - limit: number - } - } = { parameters } + const queryPayload: ExecuteQueryRequest = { parameters } if (paginate && supportsPagination) { const requestCursor = type === "page" ? parseInt(cursor || "1") : cursor queryPayload.pagination = { page: requestCursor, limit } From af0312e5fe501e1b0f739b1c391877023f388186 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:48:45 +0100 Subject: [PATCH 46/63] Proper type QueryArrayFetch --- packages/frontend-core/src/fetch/QueryArrayFetch.ts | 10 +++++----- packages/frontend-core/src/utils/json.d.ts | 13 +++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 packages/frontend-core/src/utils/json.d.ts diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.ts b/packages/frontend-core/src/fetch/QueryArrayFetch.ts index 15f86ae7fc0..ce9177e5546 100644 --- a/packages/frontend-core/src/fetch/QueryArrayFetch.ts +++ b/packages/frontend-core/src/fetch/QueryArrayFetch.ts @@ -14,12 +14,12 @@ export default class QueryArrayFetch extends FieldFetch { try { const table = await this.API.fetchQueryDefinition(datasource.tableId) const schema = generateQueryArraySchemas( - table?.schema, - table?.nestedSchemaFields + table.schema, + table.nestedSchemaFields ) - const result: { - schema: Record | null - } = { schema: getJSONArrayDatasourceSchema(schema, datasource) } + const result = { + schema: getJSONArrayDatasourceSchema(schema, datasource), + } return result } catch (error) { diff --git a/packages/frontend-core/src/utils/json.d.ts b/packages/frontend-core/src/utils/json.d.ts new file mode 100644 index 00000000000..ebda694c153 --- /dev/null +++ b/packages/frontend-core/src/utils/json.d.ts @@ -0,0 +1,13 @@ +import { QuerySchema } from "@budibase/types" + +type Schema = Record + +export const getJSONArrayDatasourceSchema: ( + tableSchema: Schema, + datasource: any +) => Record + +export const generateQueryArraySchemas: ( + schema: Schema, + nestedSchemaFields?: Record +) => Schema From fc4336a9f30d52197f1ff40304be455bf346e8f4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:54:21 +0100 Subject: [PATCH 47/63] Add typings --- packages/frontend-core/src/utils/json.d.ts | 10 +++++++++- packages/types/src/documents/app/table/schema.ts | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/utils/json.d.ts b/packages/frontend-core/src/utils/json.d.ts index ebda694c153..4f26f4b264b 100644 --- a/packages/frontend-core/src/utils/json.d.ts +++ b/packages/frontend-core/src/utils/json.d.ts @@ -1,4 +1,4 @@ -import { QuerySchema } from "@budibase/types" +import { JsonFieldMetadata, QuerySchema } from "@budibase/types" type Schema = Record @@ -11,3 +11,11 @@ export const generateQueryArraySchemas: ( schema: Schema, nestedSchemaFields?: Record ) => Schema + +export const convertJSONSchemaToTableSchema: ( + jsonSchema: JsonFieldMetadata, + options: { + squashObjects?: boolean + prefixKeys?: string + } +) => Record diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 771192e2f5b..58af430f7e4 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -227,6 +227,7 @@ interface OtherFieldMetadata extends BaseFieldSchema { | FieldType.OPTIONS | FieldType.BOOLEAN | FieldType.BIGINT + | FieldType.JSON > } From 88760d473e960668e57537597824652bbd946d31 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:54:37 +0100 Subject: [PATCH 48/63] Proper type nestedProvider --- .../src/fetch/NestedProviderFetch.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/fetch/NestedProviderFetch.ts b/packages/frontend-core/src/fetch/NestedProviderFetch.ts index 71eb5177dbf..4bcdd697a20 100644 --- a/packages/frontend-core/src/fetch/NestedProviderFetch.ts +++ b/packages/frontend-core/src/fetch/NestedProviderFetch.ts @@ -1,7 +1,23 @@ +import { Row, TableSchema } from "@budibase/types" import DataFetch from "./DataFetch" -export default class NestedProviderFetch extends DataFetch { - async getDefinition(datasource: any) { +interface NestedProviderDatasource { + value?: { + schema: TableSchema + primaryDisplay: string + rows: Row[] + } +} + +interface NestedProviderDefinition { + schema?: TableSchema + primaryDisplay?: string +} +export default class NestedProviderFetch extends DataFetch< + NestedProviderDatasource, + NestedProviderDefinition +> { + async getDefinition(datasource: NestedProviderDatasource) { // Nested providers should already have exposed their own schema return { schema: datasource?.value?.schema, From fae3c6b3eb76119a88448be10c177b603dbdb85c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 13:58:20 +0100 Subject: [PATCH 49/63] Type GroupUserFetch --- packages/frontend-core/src/constants.ts | 4 ++-- packages/frontend-core/src/fetch/GroupUserFetch.ts | 14 +++++++------- packages/frontend-core/src/fetch/JSONArrayFetch.ts | 5 +---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/frontend-core/src/constants.ts b/packages/frontend-core/src/constants.ts index 8a39e8c106b..907d91825f1 100644 --- a/packages/frontend-core/src/constants.ts +++ b/packages/frontend-core/src/constants.ts @@ -32,8 +32,8 @@ export const Cookies = { } // Table names -export const TableNames = { - USERS: "ta_users", +export const enum TableNames { + USERS = "ta_users", } export const BudibaseRoles = { diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.ts b/packages/frontend-core/src/fetch/GroupUserFetch.ts index 2f5cacd1a2b..bc7688330a7 100644 --- a/packages/frontend-core/src/fetch/GroupUserFetch.ts +++ b/packages/frontend-core/src/fetch/GroupUserFetch.ts @@ -8,16 +8,16 @@ interface GroupUserQuery { emailSearch: string } +interface GroupUserDatasource { + tableId: TableNames.USERS +} + export default class GroupUserFetch extends DataFetch< - any, - any, + GroupUserDatasource, + {}, GroupUserQuery > { - constructor(opts: { - API: APIClient - datasource: any - query: GroupUserQuery - }) { + constructor(opts: { API: APIClient; query: GroupUserQuery }) { super({ ...opts, datasource: { diff --git a/packages/frontend-core/src/fetch/JSONArrayFetch.ts b/packages/frontend-core/src/fetch/JSONArrayFetch.ts index a254bc3ae48..f0cbaa87c51 100644 --- a/packages/frontend-core/src/fetch/JSONArrayFetch.ts +++ b/packages/frontend-core/src/fetch/JSONArrayFetch.ts @@ -7,10 +7,7 @@ export default class JSONArrayFetch extends FieldFetch { // We can then extract their schema as a subset of the table schema. try { const table = await this.API.fetchTableDefinition(datasource.tableId) - const schema: Record | null = getJSONArrayDatasourceSchema( - table?.schema, - datasource - ) + const schema = getJSONArrayDatasourceSchema(table?.schema, datasource) return { schema } } catch (error) { return null From 2fb243a7c7881489590c4b31e768cbc7affb856a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 14:09:05 +0100 Subject: [PATCH 50/63] Fix customFetch --- .../frontend-core/src/fetch/CustomFetch.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/fetch/CustomFetch.ts b/packages/frontend-core/src/fetch/CustomFetch.ts index 9db9a935a5f..176d878a54c 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.ts +++ b/packages/frontend-core/src/fetch/CustomFetch.ts @@ -1,6 +1,15 @@ import DataFetch from "./DataFetch" -export default class CustomFetch extends DataFetch { +interface CustomDatasource { + data: any +} + +type CustomDefinition = Record + +export default class CustomFetch extends DataFetch< + CustomDatasource, + CustomDefinition +> { // Gets the correct Budibase type for a JS value getType(value: any) { if (value == null) { @@ -55,7 +64,7 @@ export default class CustomFetch extends DataFetch { } // Enriches the custom data to ensure the structure and format is usable - enrichCustomData(data: any[]) { + enrichCustomData(data: (string | any)[]) { if (!data?.length) { return [] } @@ -72,7 +81,7 @@ export default class CustomFetch extends DataFetch { // Try parsing strings if (typeof value === "string") { const split = value.split(",").map(x => x.trim()) - let obj: Record = {} + const obj: Record = {} for (let i = 0; i < split.length; i++) { const suffix = i === 0 ? "" : ` ${i + 1}` const key = `Value${suffix}` @@ -87,27 +96,27 @@ export default class CustomFetch extends DataFetch { } // Extracts and parses the custom data from the datasource definition - getCustomData(datasource: { data: any }) { + getCustomData(datasource: CustomDatasource) { return this.enrichCustomData(this.parseCustomData(datasource?.data)) } - async getDefinition(datasource: any) { + async getDefinition(datasource: CustomDatasource) { // Try and work out the schema from the array provided - let schema: any = {} + const schema: CustomDefinition = {} const data = this.getCustomData(datasource) if (!data?.length) { return { schema } } // Go through every object and extract all valid keys - for (let datum of data) { - for (let key of Object.keys(datum)) { + for (const datum of data) { + for (const key of Object.keys(datum)) { if (key === "_id") { continue } if (!schema[key]) { let type = this.getType(datum[key]) - let constraints: any = {} + const constraints: any = {} // Determine whether we should render text columns as options instead if (type === "string") { From 819ca2129e1c9d6f138951314af08ed86ab9b226 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 14:17:10 +0100 Subject: [PATCH 51/63] Clean types --- .../frontend-core/src/fetch/GroupUserFetch.ts | 5 ++--- packages/frontend-core/src/fetch/UserFetch.ts | 16 +++++++--------- packages/frontend-core/src/fetch/ViewV2Fetch.ts | 5 +---- packages/frontend-core/src/fetch/index.ts | 6 +++--- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.ts b/packages/frontend-core/src/fetch/GroupUserFetch.ts index bc7688330a7..a14623bfb06 100644 --- a/packages/frontend-core/src/fetch/GroupUserFetch.ts +++ b/packages/frontend-core/src/fetch/GroupUserFetch.ts @@ -1,7 +1,6 @@ import { get } from "svelte/store" -import DataFetch from "./DataFetch" +import DataFetch, { DataFetchParams } from "./DataFetch" import { TableNames } from "../constants" -import { APIClient } from "../api/types" interface GroupUserQuery { groupId: string @@ -17,7 +16,7 @@ export default class GroupUserFetch extends DataFetch< {}, GroupUserQuery > { - constructor(opts: { API: APIClient; query: GroupUserQuery }) { + constructor(opts: DataFetchParams) { super({ ...opts, datasource: { diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index b865c32d632..8f1ef36cac4 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -1,30 +1,28 @@ import { get } from "svelte/store" -import DataFetch from "./DataFetch" +import DataFetch, { DataFetchParams } from "./DataFetch" import { TableNames } from "../constants" import { utils } from "@budibase/shared-core" import { BasicOperator, SearchFilters, SearchUsersRequest, - Table, } from "@budibase/types" -import { APIClient } from "../api/types.js" interface UserFetchQuery { appId: string paginated: boolean } +interface UserDatasource { + tableId: string +} + export default class UserFetch extends DataFetch< - { tableId: string }, + UserDatasource, {}, UserFetchQuery > { - constructor(opts: { - API: APIClient - datasource: Table - query: UserFetchQuery - }) { + constructor(opts: DataFetchParams) { super({ ...opts, datasource: { diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 3bb04d5bc4a..1be1ba295c4 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -12,10 +12,7 @@ export default class ViewV2Fetch extends DataFetch { } } - async getDefinition(datasource: UIView | null): Promise { - if (!datasource?.id) { - return null - } + async getDefinition(datasource: UIView) { try { const res = await this.API.viewV2.fetchDefinition(datasource.id) return res?.data diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 52233bd3fb9..1577f76034f 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -10,7 +10,7 @@ import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch" import CustomFetch from "./CustomFetch" import QueryArrayFetch from "./QueryArrayFetch.js" -import { TableSchema, UIDatasource } from "@budibase/types" +import { UIDatasource } from "@budibase/types" import { APIClient } from "../api/types.js" const DataFetchMap = { @@ -59,7 +59,7 @@ export const getDatasourceDefinition = async ({ datasource, }: { API: APIClient - datasource: UIDatasource + datasource: any }) => { const instance = createEmptyFetchInstance({ API, datasource }) return await instance?.getDefinition(datasource) @@ -73,7 +73,7 @@ export const getDatasourceSchema = ({ }: { API: APIClient datasource: UIDatasource - definition?: { schema?: TableSchema } + definition?: any }) => { const instance = createEmptyFetchInstance({ API, datasource }) return instance?.getSchema(definition) From 10fca945d27f282c2e765f035f4b1582a60fb3a7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 14:27:13 +0100 Subject: [PATCH 52/63] Cleanups --- .../frontend-core/src/fetch/CustomFetch.ts | 4 ++- packages/frontend-core/src/fetch/DataFetch.ts | 15 ++++++----- .../frontend-core/src/fetch/FieldFetch.ts | 6 ++--- .../frontend-core/src/fetch/JSONArrayFetch.ts | 6 +++-- .../src/fetch/NestedProviderFetch.ts | 4 ++- .../src/fetch/QueryArrayFetch.ts | 6 +++-- .../frontend-core/src/fetch/QueryFetch.ts | 6 +++-- .../src/fetch/RelationshipFetch.ts | 4 ++- .../frontend-core/src/fetch/TableFetch.ts | 4 ++- packages/frontend-core/src/fetch/ViewFetch.ts | 4 ++- .../frontend-core/src/fetch/ViewV2Fetch.ts | 4 ++- packages/frontend-core/src/fetch/index.ts | 27 +++++++++++++------ 12 files changed, 60 insertions(+), 30 deletions(-) diff --git a/packages/frontend-core/src/fetch/CustomFetch.ts b/packages/frontend-core/src/fetch/CustomFetch.ts index 176d878a54c..afd3d18ba9f 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.ts +++ b/packages/frontend-core/src/fetch/CustomFetch.ts @@ -100,7 +100,9 @@ export default class CustomFetch extends DataFetch< return this.enrichCustomData(this.parseCustomData(datasource?.data)) } - async getDefinition(datasource: CustomDatasource) { + async getDefinition() { + const { datasource } = this.options + // Try and work out the schema from the array provided const schema: CustomDefinition = {} const data = this.getCustomData(datasource) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index b5eb774e45f..e5d899d5966 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -11,6 +11,7 @@ import { SortType, TableSchema, UISearchFilter, + ViewSchema, } from "@budibase/types" import { APIClient } from "../api/types" @@ -210,10 +211,10 @@ export default abstract class DataFetch< * Fetches a fresh set of data from the server, resetting pagination */ async getInitialData() { - const { datasource, filter, paginate } = this.options + const { filter, paginate } = this.options // Fetch datasource definition and extract sort properties if configured - const definition = await this.getDefinition(datasource) + const definition = await this.getDefinition() // Determine feature flags const features = await this.determineFeatureFlags() @@ -351,19 +352,19 @@ export default abstract class DataFetch< /** * Gets the definition for this datasource. - * @param datasource + * @return {object} the definition */ - abstract getDefinition( - datasource: TDatasource | null - ): Promise + abstract getDefinition(): Promise /** * Gets the schema definition for a datasource. * @param definition the datasource definition * @return {object} the schema */ - getSchema(definition: TDefinition | null): Record | undefined { + getSchema( + definition: TDefinition | null + ): ViewSchema | Record | undefined { return definition?.schema ?? undefined } diff --git a/packages/frontend-core/src/fetch/FieldFetch.ts b/packages/frontend-core/src/fetch/FieldFetch.ts index 636fb63f3d6..ac1e683c516 100644 --- a/packages/frontend-core/src/fetch/FieldFetch.ts +++ b/packages/frontend-core/src/fetch/FieldFetch.ts @@ -19,9 +19,9 @@ export default class FieldFetch extends DataFetch< FieldDatasource, FieldDefinition > { - async getDefinition( - datasource: FieldDatasource - ): Promise { + async getDefinition(): Promise { + const { datasource } = this.options + // Field sources have their schema statically defined let schema if (datasource.fieldType === "attachment") { diff --git a/packages/frontend-core/src/fetch/JSONArrayFetch.ts b/packages/frontend-core/src/fetch/JSONArrayFetch.ts index f0cbaa87c51..cae9a1e5216 100644 --- a/packages/frontend-core/src/fetch/JSONArrayFetch.ts +++ b/packages/frontend-core/src/fetch/JSONArrayFetch.ts @@ -1,8 +1,10 @@ -import FieldFetch, { FieldDatasource } from "./FieldFetch" +import FieldFetch from "./FieldFetch" import { getJSONArrayDatasourceSchema } from "../utils/json" export default class JSONArrayFetch extends FieldFetch { - async getDefinition(datasource: FieldDatasource) { + async getDefinition() { + const { datasource } = this.options + // JSON arrays need their table definitions fetched. // We can then extract their schema as a subset of the table schema. try { diff --git a/packages/frontend-core/src/fetch/NestedProviderFetch.ts b/packages/frontend-core/src/fetch/NestedProviderFetch.ts index 4bcdd697a20..666340610f0 100644 --- a/packages/frontend-core/src/fetch/NestedProviderFetch.ts +++ b/packages/frontend-core/src/fetch/NestedProviderFetch.ts @@ -17,7 +17,9 @@ export default class NestedProviderFetch extends DataFetch< NestedProviderDatasource, NestedProviderDefinition > { - async getDefinition(datasource: NestedProviderDatasource) { + async getDefinition() { + const { datasource } = this.options + // Nested providers should already have exposed their own schema return { schema: datasource?.value?.schema, diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.ts b/packages/frontend-core/src/fetch/QueryArrayFetch.ts index ce9177e5546..9142000fe64 100644 --- a/packages/frontend-core/src/fetch/QueryArrayFetch.ts +++ b/packages/frontend-core/src/fetch/QueryArrayFetch.ts @@ -1,11 +1,13 @@ -import FieldFetch, { FieldDatasource } from "./FieldFetch" +import FieldFetch from "./FieldFetch" import { getJSONArrayDatasourceSchema, generateQueryArraySchemas, } from "../utils/json" export default class QueryArrayFetch extends FieldFetch { - async getDefinition(datasource: FieldDatasource) { + async getDefinition() { + const { datasource } = this.options + if (!datasource?.tableId) { return null } diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts index 0825d396605..0754edd2673 100644 --- a/packages/frontend-core/src/fetch/QueryFetch.ts +++ b/packages/frontend-core/src/fetch/QueryFetch.ts @@ -18,7 +18,7 @@ interface QueryDatasource { export default class QueryFetch extends DataFetch { async determineFeatureFlags() { - const definition = await this.getDefinition(this.options.datasource) + const definition = await this.getDefinition() const supportsPagination = !!definition?.fields?.pagination?.type && !!definition?.fields?.pagination?.location && @@ -26,7 +26,9 @@ export default class QueryFetch extends DataFetch { return { supportsPagination } } - async getDefinition(datasource: QueryDatasource) { + async getDefinition() { + const { datasource } = this.options + if (!datasource?._id) { return null } diff --git a/packages/frontend-core/src/fetch/RelationshipFetch.ts b/packages/frontend-core/src/fetch/RelationshipFetch.ts index 7b6e93fbcc1..f853a753cd9 100644 --- a/packages/frontend-core/src/fetch/RelationshipFetch.ts +++ b/packages/frontend-core/src/fetch/RelationshipFetch.ts @@ -12,7 +12,9 @@ export default class RelationshipFetch extends DataFetch< RelationshipDatasource, Table > { - async getDefinition(datasource: RelationshipDatasource) { + async getDefinition() { + const { datasource } = this.options + if (!datasource?.tableId) { return null } diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index 433de69b59f..c1152f2869b 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -11,7 +11,9 @@ export default class TableFetch extends DataFetch { } } - async getDefinition(datasource: UITable) { + async getDefinition() { + const { datasource } = this.options + if (!datasource?.tableId) { return null } diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts index 2238d226ab7..b6830e7118a 100644 --- a/packages/frontend-core/src/fetch/ViewFetch.ts +++ b/packages/frontend-core/src/fetch/ViewFetch.ts @@ -4,7 +4,9 @@ import DataFetch from "./DataFetch" type ViewV1 = View & { name: string } export default class ViewFetch extends DataFetch { - async getDefinition(datasource: ViewV1) { + async getDefinition() { + const { datasource } = this.options + if (!datasource?.tableId) { return null } diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts index 1be1ba295c4..cdd3bab6ed4 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts @@ -12,7 +12,9 @@ export default class ViewV2Fetch extends DataFetch { } } - async getDefinition(datasource: UIView) { + async getDefinition() { + const { datasource } = this.options + try { const res = await this.API.viewV2.fetchDefinition(datasource.id) return res?.data diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts index 1577f76034f..4accb0b5ec6 100644 --- a/packages/frontend-core/src/fetch/index.ts +++ b/packages/frontend-core/src/fetch/index.ts @@ -10,7 +10,6 @@ import UserFetch from "./UserFetch.js" import GroupUserFetch from "./GroupUserFetch" import CustomFetch from "./CustomFetch" import QueryArrayFetch from "./QueryArrayFetch.js" -import { UIDatasource } from "@budibase/types" import { APIClient } from "../api/types.js" const DataFetchMap = { @@ -39,12 +38,16 @@ export const fetchData = ({ API, datasource, options }: any) => { // Creates an empty fetch instance with no datasource configured, so no data // will initially be loaded -const createEmptyFetchInstance = ({ +const createEmptyFetchInstance = < + TDatasource extends { + type: keyof typeof DataFetchMap + } +>({ API, datasource, }: { API: APIClient - datasource: any + datasource: TDatasource }) => { const handler = DataFetchMap[datasource?.type as keyof typeof DataFetchMap] if (!handler) { @@ -54,25 +57,33 @@ const createEmptyFetchInstance = ({ } // Fetches the definition of any type of datasource -export const getDatasourceDefinition = async ({ +export const getDatasourceDefinition = async < + TDatasource extends { + type: keyof typeof DataFetchMap + } +>({ API, datasource, }: { API: APIClient - datasource: any + datasource: TDatasource }) => { const instance = createEmptyFetchInstance({ API, datasource }) - return await instance?.getDefinition(datasource) + return await instance?.getDefinition() } // Fetches the schema of any type of datasource -export const getDatasourceSchema = ({ +export const getDatasourceSchema = < + TDatasource extends { + type: keyof typeof DataFetchMap + } +>({ API, datasource, definition, }: { API: APIClient - datasource: UIDatasource + datasource: TDatasource definition?: any }) => { const instance = createEmptyFetchInstance({ API, datasource }) From 95d3238d1e83cb5d840055edbb0214a16b81ff97 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 14:29:28 +0100 Subject: [PATCH 53/63] Fix declarations --- packages/bbui/src/helpers.d.ts | 4 ++- packages/frontend-core/src/utils/json.d.ts | 32 ++++++++++++---------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/bbui/src/helpers.d.ts b/packages/bbui/src/helpers.d.ts index 98c6060590a..79e08657b7b 100644 --- a/packages/bbui/src/helpers.d.ts +++ b/packages/bbui/src/helpers.d.ts @@ -1 +1,3 @@ -export const cloneDeep: (obj: T) => T +declare module "./helpers" { + export const cloneDeep: (obj: T) => T +} diff --git a/packages/frontend-core/src/utils/json.d.ts b/packages/frontend-core/src/utils/json.d.ts index 4f26f4b264b..e9b6ac5703f 100644 --- a/packages/frontend-core/src/utils/json.d.ts +++ b/packages/frontend-core/src/utils/json.d.ts @@ -2,20 +2,22 @@ import { JsonFieldMetadata, QuerySchema } from "@budibase/types" type Schema = Record -export const getJSONArrayDatasourceSchema: ( - tableSchema: Schema, - datasource: any -) => Record +declare module "./json" { + export const getJSONArrayDatasourceSchema: ( + tableSchema: Schema, + datasource: any + ) => Record -export const generateQueryArraySchemas: ( - schema: Schema, - nestedSchemaFields?: Record -) => Schema + export const generateQueryArraySchemas: ( + schema: Schema, + nestedSchemaFields?: Record + ) => Schema -export const convertJSONSchemaToTableSchema: ( - jsonSchema: JsonFieldMetadata, - options: { - squashObjects?: boolean - prefixKeys?: string - } -) => Record + export const convertJSONSchemaToTableSchema: ( + jsonSchema: JsonFieldMetadata, + options: { + squashObjects?: boolean + prefixKeys?: string + } + ) => Record +} From 83bc2e17dbc4bbf30ae234d44a58a431d5851ff8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 14:45:00 +0100 Subject: [PATCH 54/63] Fix types --- .../frontend-core/src/components/grid/stores/datasource.ts | 4 ++-- packages/frontend-core/src/components/grid/stores/rows.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index 4c20e9493f3..7fca6ace49d 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -73,7 +73,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { const schema = derived(definition, $definition => { const schema: Record | null | undefined = getDatasourceSchema({ API, - datasource: get(datasource), + datasource: get(datasource) as any, definition: $definition ?? undefined, }) if (!schema) { @@ -184,7 +184,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { const refreshDefinition = async () => { const def = await getDatasourceDefinition({ API, - datasource: get(datasource), + datasource: get(datasource) as any, }) definition.set((def as any) ?? null) } diff --git a/packages/frontend-core/src/components/grid/stores/rows.ts b/packages/frontend-core/src/components/grid/stores/rows.ts index b9c9b3fe1eb..72502b3dbdc 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.ts +++ b/packages/frontend-core/src/components/grid/stores/rows.ts @@ -254,7 +254,7 @@ export const createActions = (context: StoreContext): RowActionStore => { // Reset state properties when dataset changes if (!$instanceLoaded || resetRows) { - definition.set($fetch.definition) + definition.set($fetch.definition as any) } // Reset scroll state when data changes From d3ba4b103e27b58547b7ccf495c967d0edc8d012 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 15:51:48 +0100 Subject: [PATCH 55/63] Fix return type --- packages/server/src/api/controllers/row/views.ts | 1 + packages/types/src/api/web/pagination.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index dcf8680348c..418aa462c46 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -54,6 +54,7 @@ export async function searchView( rows: result.rows, bookmark: result.bookmark, hasNextPage: result.hasNextPage, + totalRows: result.totalRows, } } diff --git a/packages/types/src/api/web/pagination.ts b/packages/types/src/api/web/pagination.ts index 48588bf6a1d..f87bc978248 100644 --- a/packages/types/src/api/web/pagination.ts +++ b/packages/types/src/api/web/pagination.ts @@ -24,4 +24,5 @@ export interface PaginationRequest extends BasicPaginationRequest { export interface PaginationResponse { bookmark: string | number | undefined hasNextPage?: boolean + totalRows?: number } From 5f82a395174c6afe4930c6f04e8a65139fa4bf71 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 15:57:15 +0100 Subject: [PATCH 56/63] Undo some typings --- packages/frontend-core/src/fetch/DataFetch.ts | 2 +- packages/shared-core/src/filters.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index e5d899d5966..c0be73dd2f0 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -324,7 +324,7 @@ export default abstract class DataFetch< } // If we don't support sorting, do a client-side sort - if (!this.features.supportsSort && clientSideSorting) { + if (!this.features.supportsSort && clientSideSorting && sortType) { rows = sort(rows, sortColumn as any, sortOrder, sortType) } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index a1e8534a95e..b711d4cb614 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -552,7 +552,7 @@ export function search>( */ export function runQuery>( docs: T[], - query: SearchFilters | null + query: SearchFilters ): T[] { if (!docs || !Array.isArray(docs)) { return [] @@ -876,7 +876,7 @@ export function sort>( docs: T[], sort: keyof T, sortOrder: SortOrder, - sortType: SortType | null = SortType.STRING + sortType = SortType.STRING ): T[] { if (!sort || !sortOrder || !sortType) { return docs From a8abe5bc432da6b7644095d9ebb2cf533f3bdb32 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 15:58:55 +0100 Subject: [PATCH 57/63] Remove deprecated fetch --- packages/types/src/ui/stores/grid/fetch.ts | 80 ---------------------- packages/types/src/ui/stores/grid/index.ts | 1 - 2 files changed, 81 deletions(-) delete mode 100644 packages/types/src/ui/stores/grid/fetch.ts diff --git a/packages/types/src/ui/stores/grid/fetch.ts b/packages/types/src/ui/stores/grid/fetch.ts deleted file mode 100644 index 5f42db24b0c..00000000000 --- a/packages/types/src/ui/stores/grid/fetch.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - Row, - SearchFilters, - SortOrder, - SortType, - Table, - UIDatasource, - UILegacyFilter, - UISearchFilter, -} from "@budibase/types" - -interface SearchOptions { - query?: SearchFilters | null | undefined - limit: number - sort: string | null - sortOrder: string | undefined - sortType: SortType | null - paginate: boolean - bookmark: null -} - -interface TableAPI { - fetchTableDefinition(tableId: string): Promise
- searchTable(tableId: string, options: SearchOptions): any -} - -interface ViewV2API { - fetchDefinition: (datasourceId: string) => Promise - fetch: (datasourceId: string, options: SearchOptions) => any -} - -interface UserAPI { - searchUsers: (opts: { - bookmark: null - query: - | SearchFilters - | { - string: { - email: null - } - } - | null - appId: string - paginate: boolean - limit: number - }) => Promise -} - -export interface UIFetchAPI extends TableAPI, UserAPI { - definition: UIDatasource - - getInitialData: () => Promise - loading: any - loaded: boolean - - viewV2: ViewV2API - - resetKey: string | null - error: any - - hasNextPage: boolean - nextPage: () => Promise - - rows: Row[] - - options?: { - datasource?: { - tableId: string - id: string - } - } - update: ({ - sortOrder, - sortColumn, - }: { - sortOrder?: SortOrder - sortColumn?: string - filter?: UILegacyFilter[] | UISearchFilter - }) => any -} diff --git a/packages/types/src/ui/stores/grid/index.ts b/packages/types/src/ui/stores/grid/index.ts index f4191344526..7c3b6d4cb44 100644 --- a/packages/types/src/ui/stores/grid/index.ts +++ b/packages/types/src/ui/stores/grid/index.ts @@ -6,4 +6,3 @@ export * from "./view" export * from "./user" export * from "./filters" export * from "./rows" -export * from "./fetch" From 0ca0ba64336e8a1514ba18e5fbd3f7f887be5127 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 16:04:39 +0100 Subject: [PATCH 58/63] Type nulls --- packages/frontend-core/src/fetch/DataFetch.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index c0be73dd2f0..74450c62542 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -19,14 +19,14 @@ const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils interface DataFetchStore { rows: Row[] - info: null + info: any schema: TableSchema | null loading: boolean loaded: boolean query: TQuery pageNumber: number - cursor: null - cursors: any[] + cursor: string | null + cursors: string[] resetKey: string error: { message: string From a414505eefdcfad7da76f8422f7539effd71bc4e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 16:13:43 +0100 Subject: [PATCH 59/63] Cleanups --- .../frontend-core/src/components/grid/stores/datasource.ts | 4 ++-- packages/frontend-core/src/fetch/TableFetch.ts | 2 +- packages/frontend-core/src/fetch/UserFetch.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index 7fca6ace49d..0b07796fdea 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -71,7 +71,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { } = context const schema = derived(definition, $definition => { - const schema: Record | null | undefined = getDatasourceSchema({ + const schema: Record | undefined = getDatasourceSchema({ API, datasource: get(datasource) as any, definition: $definition ?? undefined, @@ -186,7 +186,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { API, datasource: get(datasource) as any, }) - definition.set((def as any) ?? null) + definition.set(def as any) } // Saves the datasource definition diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts index c1152f2869b..f5927262cbd 100644 --- a/packages/frontend-core/src/fetch/TableFetch.ts +++ b/packages/frontend-core/src/fetch/TableFetch.ts @@ -37,7 +37,7 @@ export default class TableFetch extends DataFetch { // Search table try { const res = await this.API.searchTable(tableId, { - query: query ?? undefined, + query, limit, sort: sortColumn, sortOrder: sortOrder ?? SortOrder.ASCENDING, diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts index 8f1ef36cac4..656cd840fed 100644 --- a/packages/frontend-core/src/fetch/UserFetch.ts +++ b/packages/frontend-core/src/fetch/UserFetch.ts @@ -52,7 +52,7 @@ export default class UserFetch extends DataFetch< const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest) ? rest - : { [BasicOperator.EMPTY]: { email: true } } + : { [BasicOperator.EMPTY]: { email: null } } try { const opts: SearchUsersRequest = { From 0784a9571273abd3bddd74e14776df3f4bf23d18 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 16:33:29 +0100 Subject: [PATCH 60/63] Remove ! usage --- packages/frontend-core/src/api/views.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/api/views.ts b/packages/frontend-core/src/api/views.ts index 83f7e97df07..aa0f797f588 100644 --- a/packages/frontend-core/src/api/views.ts +++ b/packages/frontend-core/src/api/views.ts @@ -28,7 +28,9 @@ export const buildViewEndpoints = (API: BaseAPIClient): ViewEndpoints => ({ fetchViewData: async (name, { field, groupBy, calculation }) => { const params = new URLSearchParams() if (calculation) { - params.set("field", field!) + if (field) { + params.set("field", field) + } params.set("calculation", calculation) } if (groupBy) { From 3a8942f487eb0fdd625d7815aa641977ac8b078c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 17:02:34 +0100 Subject: [PATCH 61/63] Add todos --- .../src/components/grid/stores/datasource.ts | 14 ++++++++------ .../src/components/grid/stores/rows.ts | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.ts b/packages/frontend-core/src/components/grid/stores/datasource.ts index 0b07796fdea..805ace5a8f8 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.ts +++ b/packages/frontend-core/src/components/grid/stores/datasource.ts @@ -1,3 +1,5 @@ +// TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages. + import { derived, get, Readable, Writable } from "svelte/store" import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" import { enrichSchemaWithRelColumns, memo } from "../../../utils" @@ -73,7 +75,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { const schema = derived(definition, $definition => { const schema: Record | undefined = getDatasourceSchema({ API, - datasource: get(datasource) as any, + datasource: get(datasource) as any, // TODO: see line 1 definition: $definition ?? undefined, }) if (!schema) { @@ -130,7 +132,7 @@ export const deriveStores = (context: StoreContext): DerivedDatasourceStore => { ([$datasource, $definition]) => { let type = $datasource?.type if (type === "provider") { - type = ($datasource as any).value?.datasource?.type + type = ($datasource as any).value?.datasource?.type // TODO: see line 1 } // Handle calculation views if (type === "viewV2" && $definition?.type === ViewV2Type.CALCULATION) { @@ -184,9 +186,9 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { const refreshDefinition = async () => { const def = await getDatasourceDefinition({ API, - datasource: get(datasource) as any, + datasource: get(datasource) as any, // TODO: see line 1 }) - definition.set(def as any) + definition.set(def as any) // TODO: see line 1 } // Saves the datasource definition @@ -231,7 +233,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { if ("default" in newDefinition.schema[column]) { delete newDefinition.schema[column].default } - return await saveDefinition(newDefinition as any) + return await saveDefinition(newDefinition as any) // TODO: see line 1 } // Adds a schema mutation for a single field @@ -307,7 +309,7 @@ export const createActions = (context: StoreContext): ActionDatasourceStore => { await saveDefinition({ ...$definition, schema: newSchema, - } as any) + } as any) // TODO: see line 1 resetSchemaMutations() } diff --git a/packages/frontend-core/src/components/grid/stores/rows.ts b/packages/frontend-core/src/components/grid/stores/rows.ts index 72502b3dbdc..d227fc70df1 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.ts +++ b/packages/frontend-core/src/components/grid/stores/rows.ts @@ -254,7 +254,7 @@ export const createActions = (context: StoreContext): RowActionStore => { // Reset state properties when dataset changes if (!$instanceLoaded || resetRows) { - definition.set($fetch.definition as any) + definition.set($fetch.definition as any) // TODO: datasource and defitions are unions of the different implementations. At this point, the datasource does not know what type is being used, and the assignations will cause TS exceptions. Casting it "as any" for now. This should be fixed improving the type usages. } // Reset scroll state when data changes From 23f9e3f3fe02584503a6eaebf58d21a077de22c8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 17:06:19 +0100 Subject: [PATCH 62/63] Add todo --- packages/frontend-core/src/components/grid/stores/rows.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/rows.ts b/packages/frontend-core/src/components/grid/stores/rows.ts index d227fc70df1..07fbf021347 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.ts +++ b/packages/frontend-core/src/components/grid/stores/rows.ts @@ -21,7 +21,7 @@ interface IndexedUIRow extends UIRow { interface RowStore { rows: Writable - fetch: Writable | null> + fetch: Writable | null> // TODO: type this properly, having a union of all the possible options loaded: Writable refreshing: Writable loading: Writable From eb6a2434b6bfe635a5afa2b4a08e146fd0e0ef75 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 8 Jan 2025 17:18:33 +0100 Subject: [PATCH 63/63] Simplify code --- packages/frontend-core/src/fetch/DataFetch.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts index 74450c62542..9312c57637b 100644 --- a/packages/frontend-core/src/fetch/DataFetch.ts +++ b/packages/frontend-core/src/fetch/DataFetch.ts @@ -11,7 +11,6 @@ import { SortType, TableSchema, UISearchFilter, - ViewSchema, } from "@budibase/types" import { APIClient } from "../api/types" @@ -224,12 +223,12 @@ export default abstract class DataFetch< supportsPagination: paginate && !!features?.supportsPagination, } - if (!definition?.schema) { + // Fetch and enrich schema + let schema = this.getSchema(definition) + if (!schema) { return } - - // Fetch and enrich schema - const schema = this.enrichSchema(definition.schema) + schema = this.enrichSchema(schema) // If an invalid sort column is specified, delete it if (this.options.sortColumn && !schema[this.options.sortColumn]) { @@ -362,9 +361,7 @@ export default abstract class DataFetch< * @param definition the datasource definition * @return {object} the schema */ - getSchema( - definition: TDefinition | null - ): ViewSchema | Record | undefined { + getSchema(definition: TDefinition | null): Record | undefined { return definition?.schema ?? undefined } @@ -379,7 +376,7 @@ export default abstract class DataFetch< let jsonAdditions: Record = {} for (const fieldKey of Object.keys(schema)) { const fieldSchema = schema[fieldKey] - if (fieldSchema?.type === FieldType.JSON) { + if (fieldSchema.type === FieldType.JSON) { const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, { squashObjects: true, }) as Record | null // TODO: remove when convertJSONSchemaToTableSchema is typed