diff --git a/frontend/src/widgets/AxisChart/getAxisChartOptions.js b/frontend/src/widgets/AxisChart/getAxisChartOptions.js index e86c3a9d..a33ae08d 100644 --- a/frontend/src/widgets/AxisChart/getAxisChartOptions.js +++ b/frontend/src/widgets/AxisChart/getAxisChartOptions.js @@ -219,8 +219,18 @@ function makeOptions(chartType, labels, datasets, options) { trigger: 'axis', confine: true, appendToBody: false, - valueFormatter: (value) => (isNaN(value) ? value : formatNumber(value)), + formatter: (params) => { + const filteredParams = params + .filter((p) => p.value !== 0 && p.value !== null) + .sort((a, b) => b.value - a.value); + + if (!filteredParams.length) return ''; + return filteredParams + .map((p) => `${p.marker} ${p.seriesName}: ${formatNumber(p.value)}`) + .join(''); + }, }, + } } diff --git a/frontend/src2/charts/chart.ts b/frontend/src2/charts/chart.ts index 0c0a5074..e92a105f 100644 --- a/frontend/src2/charts/chart.ts +++ b/frontend/src2/charts/chart.ts @@ -1,6 +1,7 @@ import { useDebouncedRefHistory, UseRefHistoryReturn } from '@vueuse/core' import { computed, reactive, ref, unref, watch } from 'vue' import { areDeeplyEqual, copy, getUniqueId, waitUntil, wheneverChanges } from '../helpers' +import { GranularityType } from '../helpers/constants' import { createToast } from '../helpers/toasts' import { column, count, query_table } from '../query/helpers' import { getCachedQuery, makeQuery, Query } from '../query/query' @@ -9,10 +10,11 @@ import { AxisChartConfig, DonutChartConfig, NumberChartConfig, - TableChartConfig + TableChartConfig, } from '../types/chart.types' -import { FilterArgs, GranularityType, Measure, Operation } from '../types/query.types' +import { FilterArgs, Measure, Operation } from '../types/query.types' import { WorkbookChart } from '../types/workbook.types' +import { getLinkedQueries } from '../workbook/workbook' const charts = new Map() @@ -51,6 +53,9 @@ function makeChart(workbookChart: WorkbookChart) { updateMeasure, removeMeasure, + getDependentQueries, + getDependentQueryColumns, + history: {} as UseRefHistoryReturn, }) @@ -124,12 +129,12 @@ function makeChart(workbookChart: WorkbookChart) { } function prepareAxisChartQuery(config: AxisChartConfig) { - if (!config.x_axis || !config.x_axis.column_name) { + if (!config.x_axis.dimension || !config.x_axis.dimension.column_name) { console.warn('X-axis is required') chart.dataQuery.reset() return false } - if (config.x_axis.column_name === config.split_by?.column_name) { + if (config.x_axis.dimension.column_name === config.split_by?.column_name) { createToast({ message: 'X-axis and Split by cannot be the same', variant: 'error', @@ -143,14 +148,14 @@ function makeChart(workbookChart: WorkbookChart) { if (config.split_by?.column_name) { chart.dataQuery.addPivotWider({ - rows: [config.x_axis], + rows: [config.x_axis.dimension], columns: [config.split_by], values: values, }) } else { chart.dataQuery.addSummarize({ measures: values, - dimensions: [config.x_axis], + dimensions: [config.x_axis.dimension], }) } @@ -331,6 +336,33 @@ function makeChart(workbookChart: WorkbookChart) { delete chart.doc.calculated_measures[measure.measure_name] } + function getDependentQueries() { + return [chart.doc.query, ...getLinkedQueries(chart.doc.query)] + } + + function getDependentQueryColumns() { + return getDependentQueries() + .map((q) => getCachedQuery(q)) + .filter(Boolean) + .map((q) => { + const query = q! + if (!query.result.executedSQL) { + query.execute() + } + return { + group: query.doc.title, + items: query.result.columnOptions.map((c) => { + const sep = '`' + const value = `${sep}${query.doc.name}${sep}.${sep}${c.value}${sep}` + return { + ...c, + value, + } + }), + } + }) + } + chart.history = useDebouncedRefHistory( // @ts-ignore computed({ diff --git a/frontend/src2/charts/components/BarChartConfigForm.vue b/frontend/src2/charts/components/BarChartConfigForm.vue index 201699ef..3a30b0a3 100644 --- a/frontend/src2/charts/components/BarChartConfigForm.vue +++ b/frontend/src2/charts/components/BarChartConfigForm.vue @@ -1,7 +1,7 @@ @@ -86,11 +109,22 @@ function onSort(newSortOrder: Record) { @sort="onSort" :on-export="drillDownQuery ? drillDownQuery.downloadResults : undefined" > - - - Showing {{ drillDownQuery.result.rows.length }} of - {{ drillDownQuery.result.totalRowCount }} rows - + + + + Showing {{ drillDownQuery.result.rows.length }} of + {{ drillDownQuery.result.totalRowCount }} rows + + + + diff --git a/frontend/src2/charts/components/TableChart.vue b/frontend/src2/charts/components/TableChart.vue index 4086446f..4cc9a35b 100644 --- a/frontend/src2/charts/components/TableChart.vue +++ b/frontend/src2/charts/components/TableChart.vue @@ -15,7 +15,7 @@ const props = defineProps<{ result: QueryResult }>() -const chart = inject('chart')! +const chart = inject('chart', undefined) const tableConfig = computed(() => props.config as TableChartConfig) const sortOrder = computed(() => { diff --git a/frontend/src2/charts/components/XAxisConfig.vue b/frontend/src2/charts/components/XAxisConfig.vue index a2f21183..f0dd259a 100644 --- a/frontend/src2/charts/components/XAxisConfig.vue +++ b/frontend/src2/charts/components/XAxisConfig.vue @@ -1,7 +1,8 @@ @@ -18,11 +30,24 @@ const x_axis = defineModel({ + - - + + diff --git a/frontend/src2/charts/helpers.ts b/frontend/src2/charts/helpers.ts index c716089f..65d1da55 100644 --- a/frontend/src2/charts/helpers.ts +++ b/frontend/src2/charts/helpers.ts @@ -1,6 +1,6 @@ import { graphic } from 'echarts/core' import { copy, ellipsis, formatNumber, getShortNumber, getUniqueId } from '../helpers' -import { FIELDTYPES } from '../helpers/constants' +import { FIELDTYPES, GranularityType } from '../helpers/constants' import { column, getFormattedDate } from '../query/helpers' import useQuery from '../query/query' import { @@ -12,15 +12,14 @@ import { LineChartConfig, Series, SeriesLine, + XAxis, } from '../types/chart.types' import { - ColumnDataType, FilterRule, - GranularityType, Operation, QueryResult, QueryResultColumn, - QueryResultRow, + QueryResultRow } from '../types/query.types' import { getColors, getGradientColors } from './colors' @@ -48,9 +47,11 @@ export function getLineChartOptions(config: LineChartConfig, result: QueryResult const number_columns = _columns.filter((c) => FIELDTYPES.NUMBER.includes(c.type)) const show_legend = number_columns.length > 1 - const xAxis = getXAxis({ column_type: config.x_axis.data_type }) - const xAxisIsDate = FIELDTYPES.DATE.includes(config.x_axis.data_type) - const granularity = xAxisIsDate ? getGranularity(config.x_axis.dimension_name, config) : null + const xAxis = getXAxis(config.x_axis) + const xAxisIsDate = FIELDTYPES.DATE.includes(config.x_axis.dimension.data_type) + const granularity = xAxisIsDate + ? getGranularity(config.x_axis.dimension.dimension_name, config) + : null const leftYAxis = getYAxis() const rightYAxis = getYAxis() @@ -59,15 +60,15 @@ export function getLineChartOptions(config: LineChartConfig, result: QueryResult const sortedRows = xAxisIsDate ? _rows.sort((a, b) => { - const a_date = new Date(a[config.x_axis.dimension_name]) - const b_date = new Date(b[config.x_axis.dimension_name]) + const a_date = new Date(a[config.x_axis.dimension.dimension_name]) + const b_date = new Date(b[config.x_axis.dimension.dimension_name]) return a_date.getTime() - b_date.getTime() }) : _rows const getSeriesData = (column: string) => sortedRows.map((r) => { - const x_value = r[config.x_axis.dimension_name] + const x_value = r[config.x_axis.dimension.dimension_name] const y_value = r[column] return [x_value, y_value] }) @@ -105,7 +106,7 @@ export function getLineChartOptions(config: LineChartConfig, result: QueryResult label: { fontSize: 11, show: show_data_labels, - position: idx === number_columns.length - 1 ? 'top' : 'inside', + position: 'top', formatter: (params: any) => { return getShortNumber(params.value?.[1], 1) }, @@ -140,9 +141,11 @@ export function getBarChartOptions(config: BarChartConfig, result: QueryResult, const number_columns = _columns.filter((c) => FIELDTYPES.NUMBER.includes(c.type)) const show_legend = number_columns.length > 1 - const xAxis = getXAxis({ column_type: config.x_axis.data_type }) - const xAxisIsDate = FIELDTYPES.DATE.includes(config.x_axis.data_type) - const granularity = xAxisIsDate ? getGranularity(config.x_axis.dimension_name, config) : null + const xAxis = getXAxis(config.x_axis) + const xAxisIsDate = FIELDTYPES.DATE.includes(config.x_axis.dimension.data_type) + const granularity = xAxisIsDate + ? getGranularity(config.x_axis.dimension.dimension_name, config) + : null const leftYAxis = getYAxis({ normalized: config.y_axis.normalize }) const rightYAxis = getYAxis({ normalized: config.y_axis.normalize }) @@ -151,14 +154,14 @@ export function getBarChartOptions(config: BarChartConfig, result: QueryResult, const sortedRows = xAxisIsDate ? _rows.sort((a, b) => { - const a_date = new Date(a[config.x_axis.dimension_name]) - const b_date = new Date(b[config.x_axis.dimension_name]) + const a_date = new Date(a[config.x_axis.dimension.dimension_name]) + const b_date = new Date(b[config.x_axis.dimension.dimension_name]) return a_date.getTime() - b_date.getTime() }) : _rows const total_per_x_value = _rows.reduce((acc, row) => { - const x_value = row[config.x_axis.dimension_name] + const x_value = row[config.x_axis.dimension.dimension_name] if (!acc[x_value]) acc[x_value] = 0 number_columns.forEach((m) => (acc[x_value] += row[m.name])) return acc @@ -167,7 +170,7 @@ export function getBarChartOptions(config: BarChartConfig, result: QueryResult, const getSeriesData = (column: string) => sortedRows .map((r) => { - const x_value = r[config.x_axis.dimension_name] + const x_value = r[config.x_axis.dimension.dimension_name] const y_value = r[column] const normalize = config.y_axis.normalize if (!normalize) { @@ -254,11 +257,11 @@ function getSerie(config: AxisChartConfig, number_column: string): Series { ) } -type XAxisCustomizeOptions = { - column_type?: ColumnDataType -} -function getXAxis(options: XAxisCustomizeOptions = {}) { - const xAxisIsDate = options.column_type && FIELDTYPES.DATE.includes(options.column_type) +function getXAxis(x_axis: XAxis) { + const columnType = x_axis.dimension.data_type + const xAxisIsDate = columnType && FIELDTYPES.DATE.includes(columnType) + const rotation = Math.min(Math.max(x_axis.label_rotation || 0, 0), 90) + return { type: xAxisIsDate ? 'time' : 'category', z: 2, @@ -267,6 +270,10 @@ function getXAxis(options: XAxisCustomizeOptions = {}) { splitLine: { show: false }, axisLine: { show: true }, axisTick: { show: false }, + axisLabel: { + show: true, + rotate: rotation, + }, } } @@ -559,6 +566,12 @@ function getTooltip(options: any = {}) { confine: true, appendToBody: false, formatter: (params: Object | Array) => { + if (Array.isArray(params)) { + params = params + .filter((p) => p.value?.[1] !== 0) + .sort((a, b) => b.value?.[1] - a.value?.[1]) + } + if (!Array.isArray(params)) { const p = params as any const value = options.xySwapped ? p.value[0] : p.value[1] @@ -687,6 +700,15 @@ export function getDrillDownQuery( return query } +export function handleOldXAxisConfig(old_x_axis: any): AxisChartConfig['x_axis'] { + if (old_x_axis && old_x_axis.column_name) { + return { + dimension: old_x_axis, + } + } + return old_x_axis +} + export function handleOldYAxisConfig(old_y_axis: any): AxisChartConfig['y_axis'] { if (Array.isArray(old_y_axis)) { return { @@ -709,10 +731,18 @@ export function setDimensionNames(config: any) { return dimension } - const dimensionPaths = ['x_axis', 'split_by', 'date_column', 'label_column'] - dimensionPaths.forEach((path) => { - config[path] = setDimensionName(config[path]) - }) + if (config.x_axis?.dimension) { + config.x_axis.dimension = setDimensionName(config.x_axis.dimension) + } + if (config.split_by) { + config.split_by = setDimensionName(config.split_by) + } + if (config.date_column) { + config.date_column = setDimensionName(config.date_column) + } + if (config.label_column) { + config.label_column = setDimensionName(config.label_column) + } if (config.rows && config.rows.length) { config.rows = config.rows.map(setDimensionName) } diff --git a/frontend/src2/components/Autocomplete.vue b/frontend/src2/components/Autocomplete.vue index 390331bc..640f692c 100644 --- a/frontend/src2/components/Autocomplete.vue +++ b/frontend/src2/components/Autocomplete.vue @@ -96,12 +96,16 @@ v-for="(option, idx) in group.items.slice(0, 50)" :key="option?.value || idx" :value="option" + :disabled="option.disabled" v-slot="{ active, selected }" > - + {{ $props.label }} { { @@ -245,8 +245,8 @@ const colorByValues = computed(() => { { @@ -279,8 +279,8 @@ const colorByValues = computed(() => { :key="idx" > {{ idx + page.startIndex + 1 }} diff --git a/frontend/src2/components/Switch.vue b/frontend/src2/components/Switch.vue index 8e269a0a..f0c229df 100644 --- a/frontend/src2/components/Switch.vue +++ b/frontend/src2/components/Switch.vue @@ -2,9 +2,9 @@ import { Breadcrumbs } from 'frappe-ui' -import { ExternalLink, RefreshCcw } from 'lucide-vue-next' -import { provide } from 'vue' +import { RefreshCcw } from 'lucide-vue-next' +import { provide, ref } from 'vue' import { useRouter } from 'vue-router' -import { waitUntil } from '../helpers' +import { downloadImage, waitUntil } from '../helpers' import useWorkbook from '../workbook/workbook' import useDashboard from './dashboard' -import DashboardFilterSelector from './DashboardFilterSelector.vue' import DashboardItem from './DashboardItem.vue' import useDashboardStore from './dashboards' import VueGridLayout from './VueGridLayout.vue' @@ -29,6 +28,12 @@ const router = useRouter() function openWorkbook() { router.push(`/workbook/${workbook.doc.name}`) } + +const dashboardContainer = ref(null) +async function downloadDashboardImage() { + if (!dashboardContainer.value) return + await downloadImage(dashboardContainer.value, `${dashboard.doc.title}.png`) +} @@ -40,26 +45,34 @@ function openWorkbook() { ]" /> - dashboard.refresh()" label="Refresh"> - - - - - + - + import { Edit3, RefreshCcw, Share2 } from 'lucide-vue-next' -import { computed, provide, ref } from 'vue' +import { computed, inject, provide, ref } from 'vue' import ContentEditable from '../components/ContentEditable.vue' -import { safeJSONParse } from '../helpers' +import { safeJSONParse, wheneverChanges } from '../helpers' import { WorkbookChart, WorkbookDashboard, WorkbookQuery } from '../types/workbook.types' import ChartSelectorDialog from './ChartSelectorDialog.vue' import useDashboard from './dashboard' -import DashboardFilterSelector from './DashboardFilterSelector.vue' import DashboardItem from './DashboardItem.vue' import DashboardShareDialog from './DashboardShareDialog.vue' import VueGridLayout from './VueGridLayout.vue' +import { workbookKey } from '../workbook/workbook' const props = defineProps<{ dashboard: WorkbookDashboard @@ -26,7 +26,6 @@ const selectedCharts = computed(() => { }) const showChartSelectorDialog = ref(false) -const showTextWidgetCreationDialog = ref(false) function onDragOver(event: DragEvent) { if (!event.dataTransfer) return @@ -47,6 +46,15 @@ function onDrop(event: DragEvent) { } const showShareDialog = ref(false) + +const workbook = inject(workbookKey, null) +wheneverChanges( + () => dashboard.editing, + () => { + if (!workbook) return + workbook._pauseAutoSave = dashboard.editing + } +) @@ -59,12 +67,6 @@ const showShareDialog = ref(false) placeholder="Untitled Dashboard" > - Chart + dashboard.addFilter()" + > + Filter + + dashboard.addText()" + > + Text + +import { watchDebounced } from '@vueuse/core' +import { AlertTriangle, Maximize } from 'lucide-vue-next' +import { computed, inject, ref } from 'vue' +import { useRouter } from 'vue-router' +import { getCachedChart } from '../charts/chart' +import ChartRenderer from '../charts/components/ChartRenderer.vue' +import { wheneverChanges } from '../helpers' +import { WorkbookDashboardChart } from '../types/workbook.types' +import { workbookKey } from '../workbook/workbook' +import { Dashboard } from './dashboard' + +const props = defineProps<{ item: WorkbookDashboardChart }>() + +const chart = computed(() => { + if (!props.item.chart) return null + return getCachedChart(props.item.chart) +}) + +const dashboard = inject('dashboard')! +if (props.item.chart && !chart.value?.dataQuery.result.executedSQL) { + dashboard.refreshChart(props.item.chart) +} + +watchDebounced( + () => chart.value?.doc.config.order_by, + () => props.item.chart && dashboard.refreshChart(props.item.chart), + { + deep: true, + debounce: 500, + } +) +const showExpandedChartDialog = ref(false) + +const router = useRouter() +const workbook = inject(workbookKey, null) +wheneverChanges( + () => dashboard.isEditingItem(props.item), + (editing: boolean) => { + if (!workbook) return + if (editing) { + const chartIndex = workbook.doc.charts.findIndex((c) => c.name === props.item.chart) + if (chartIndex !== -1) { + router.push(`/workbook/${workbook.doc.name}/chart/${chartIndex}`) + } + } + } +) + + + + + + + + Chart not found + + + + + + + + + + + + + + + + diff --git a/frontend/src2/dashboard/DashboardFilter.vue b/frontend/src2/dashboard/DashboardFilter.vue new file mode 100644 index 00000000..d2ef18c3 --- /dev/null +++ b/frontend/src2/dashboard/DashboardFilter.vue @@ -0,0 +1,105 @@ + + + + + + + + + + + + {{ label || 'Filter' }} + + + + + + togglePopover()" + > + + + + + + + + diff --git a/frontend/src2/dashboard/DashboardFilterEditor.vue b/frontend/src2/dashboard/DashboardFilterEditor.vue new file mode 100644 index 00000000..f2e32928 --- /dev/null +++ b/frontend/src2/dashboard/DashboardFilterEditor.vue @@ -0,0 +1,163 @@ + + + + + + + + + + + + Linked Charts + + + {{ link.title }} + + + + + + + + + diff --git a/frontend/src2/dashboard/DashboardFilterSelector.vue b/frontend/src2/dashboard/DashboardFilterSelector.vue deleted file mode 100644 index c9b584fd..00000000 --- a/frontend/src2/dashboard/DashboardFilterSelector.vue +++ /dev/null @@ -1,131 +0,0 @@ - - - - - - - - Filter - - - {{ filterGroup.filters.length }} - - - - - - diff --git a/frontend/src2/dashboard/DashboardItem.vue b/frontend/src2/dashboard/DashboardItem.vue index 6cba350d..00775180 100644 --- a/frontend/src2/dashboard/DashboardItem.vue +++ b/frontend/src2/dashboard/DashboardItem.vue @@ -1,12 +1,16 @@ - - + - - - - - - - - Chart not found - - - - - - - - + + + + + + + diff --git a/frontend/src2/dashboard/DashboardItemActions.vue b/frontend/src2/dashboard/DashboardItemActions.vue index fd415c3a..4a6138fc 100644 --- a/frontend/src2/dashboard/DashboardItemActions.vue +++ b/frontend/src2/dashboard/DashboardItemActions.vue @@ -1,46 +1,31 @@ - + diff --git a/frontend/src2/dashboard/DashboardText.vue b/frontend/src2/dashboard/DashboardText.vue new file mode 100644 index 00000000..e684806c --- /dev/null +++ b/frontend/src2/dashboard/DashboardText.vue @@ -0,0 +1,54 @@ + + + + + + + + + Content + + Markdown supported + + + + diff --git a/frontend/src2/dashboard/Filter.vue b/frontend/src2/dashboard/Filter.vue new file mode 100644 index 00000000..d519a5bb --- /dev/null +++ b/frontend/src2/dashboard/Filter.vue @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src2/dashboard/VueGridLayout.vue b/frontend/src2/dashboard/VueGridLayout.vue index 05180aaa..0fa75551 100644 --- a/frontend/src2/dashboard/VueGridLayout.vue +++ b/frontend/src2/dashboard/VueGridLayout.vue @@ -52,8 +52,8 @@ const options = reactive({ isDraggable: computed(() => !props.disabled), isResizable: computed(() => !props.disabled), responsive: true, - verticalCompact: false, - preventCollision: true, + verticalCompact: true, + preventCollision: false, useCssTransforms: true, cols: { lg: props.cols || 12, diff --git a/frontend/src2/dashboard/dashboard.ts b/frontend/src2/dashboard/dashboard.ts index 53ea198a..5732f440 100644 --- a/frontend/src2/dashboard/dashboard.ts +++ b/frontend/src2/dashboard/dashboard.ts @@ -1,9 +1,16 @@ import { reactive } from 'vue' import { getCachedChart } from '../charts/chart' import { getUniqueId, store } from '../helpers' +import { isFilterValid } from '../query/components/filter_utils' +import { column } from '../query/helpers' import { getCachedQuery } from '../query/query' -import { FilterArgs } from '../types/query.types' -import { WorkbookChart, WorkbookDashboard } from '../types/workbook.types' +import { FilterArgs, FilterOperator, FilterRule, FilterValue } from '../types/query.types' +import { + WorkbookChart, + WorkbookDashboard, + WorkbookDashboardFilter, + WorkbookDashboardItem, +} from '../types/workbook.types' const dashboards = new Map() @@ -16,28 +23,26 @@ export default function useDashboard(workbookDashboard: WorkbookDashboard) { return dashboard } +type FilterState = { + operator: FilterOperator + value: FilterValue +} + function makeDashboard(workbookDashboard: WorkbookDashboard) { const dashboard = reactive({ doc: workbookDashboard, editing: false, + editingItemIndex: null as number | null, filters: {} as Record, + filterStates: {} as Record, - activeItemIdx: null as number | null, - setActiveItem(index: number) { - dashboard.activeItemIdx = index - }, - isActiveItem(index: number) { - return dashboard.activeItemIdx == index + isEditingItem(item: WorkbookDashboardItem) { + return dashboard.editing && dashboard.editingItemIndex === dashboard.doc.items.indexOf(item) }, addChart(charts: WorkbookChart[]) { - const maxY = Math.max( - ...dashboard.doc.items - .filter((item) => item.type === 'chart') - .map((chart) => chart.layout.y + chart.layout.h), - 0 - ) + const maxY = dashboard.getMaxY() charts.forEach((chart) => { if ( !dashboard.doc.items.some((item) => item.type === 'chart' && item.chart === chart.name) @@ -57,13 +62,42 @@ function makeDashboard(workbookDashboard: WorkbookDashboard) { }) }, - removeItem(index: number) { - dashboard.doc.items.splice(index, 1) + addText() { + const maxY = dashboard.getMaxY() + dashboard.doc.items.push({ + type: 'text', + text: 'Enter text here', + layout: { + i: getUniqueId(), + x: 0, + y: maxY, + w: 10, + h: 1, + }, + }) + dashboard.editingItemIndex = dashboard.doc.items.length - 1 + }, + + addFilter() { + const maxY = dashboard.getMaxY() + dashboard.doc.items.push({ + type: 'filter', + filter_name: 'Filter', + filter_type: 'String', + links: {}, + layout: { + i: getUniqueId(), + x: 0, + y: maxY, + w: 4, + h: 1, + }, + }) + dashboard.editingItemIndex = dashboard.doc.items.length - 1 }, - applyFilter(query: string, args: FilterArgs) { - if (!dashboard.filters[query]) dashboard.filters[query] = [] - dashboard.filters[query].push(args) + removeItem(index: number) { + dashboard.doc.items.splice(index, 1) }, refresh() { @@ -76,28 +110,122 @@ function makeDashboard(workbookDashboard: WorkbookDashboard) { const chart = getCachedChart(chart_name) if (!chart || !chart.doc.query) return - Object.keys(dashboard.filters).forEach((query) => { - const _query = getCachedQuery(query) - if (!_query) return - _query.dashboardFilters = { + const filtersApplied = dashboard.doc.items.filter( + (item) => item.type === 'filter' && 'links' in item && item.links[chart_name] + ) + + if (!filtersApplied.length) { + chart.refresh(undefined, true) + return + } + + const filtersByQuery = new Map() + + filtersApplied.forEach((item) => { + const filterItem = item as WorkbookDashboardFilter + const linkedColumn = dashboard.getColumnFromFilterLink(filterItem.links[chart_name]) + if (!linkedColumn) return + + const query = getCachedQuery(linkedColumn.query) + if (!query) return + + const filterState = dashboard.filterStates[filterItem.filter_name] || {} + + const filter = { + column: column(linkedColumn.column), + operator: filterState.operator, + value: filterState.value, + } + + if (isFilterValid(filter, filterItem.filter_type)) { + const filters = filtersByQuery.get(linkedColumn.query) || [] + filters.push(filter) + filtersByQuery.set(linkedColumn.query, filters) + } + }) + + filtersByQuery.forEach((filters, query_name) => { + const query = getCachedQuery(query_name)! + query.dashboardFilters = { logical_operator: 'And', - filters: dashboard.filters[query], + filters, } }) chart.refresh(undefined, true) }, + updateFilterState(filter_name: string, operator?: FilterOperator, value?: FilterValue) { + const filter = dashboard.doc.items.find( + (item) => item.type === 'filter' && item.filter_name === filter_name + ) + if (!filter) return + + if (!operator) { + delete dashboard.filterStates[filter_name] + } else { + dashboard.filterStates[filter_name] = { + operator, + value, + } + } + + dashboard.applyFilter(filter_name) + }, + + applyFilter(filter_name: string) { + const item = dashboard.doc.items.find( + (item) => item.type === 'filter' && item.filter_name === filter_name + ) + if (!item) return + + const filterItem = item as WorkbookDashboardFilter + const filteredCharts = Object.keys(filterItem.links) + filteredCharts.forEach((chart_name) => dashboard.refreshChart(chart_name)) + }, + + getColumnFromFilterLink(linkedColumn: string) { + const sep = '`' + // `query`.`column` + const pattern = new RegExp(`^${sep}([^${sep}]+)${sep}\\.${sep}([^${sep}]+)${sep}$`) + const match = linkedColumn.match(pattern) + if (!match) return + + return { + query: match[1], + column: match[2], + } + }, + getShareLink() { return ( dashboard.doc.share_link || `${window.location.origin}/insights/shared/dashboard/${dashboard.doc.name}` ) }, + + getMaxY() { + return Math.max(...dashboard.doc.items.map((item) => item.layout.y + item.layout.h), 0) + }, }) - const key = `insights:dashboard-filters-${workbookDashboard.name}` - dashboard.filters = store(key, () => dashboard.filters) + const defaultFilters = dashboard.doc.items.reduce((acc, item) => { + if (item.type != 'filter') return acc + + const filterItem = item as WorkbookDashboardFilter + if (filterItem.default_operator && filterItem.default_value) { + acc[filterItem.filter_name] = { + operator: filterItem.default_operator, + value: filterItem.default_value, + } + } + return acc + }, {} as typeof dashboard.filterStates) + + Object.assign(dashboard.filterStates, defaultFilters) + + const key2 = `insights:dashboard-filter-states-${workbookDashboard.name}` + dashboard.filterStates = store(key2, () => dashboard.filterStates) return dashboard } diff --git a/frontend/src2/helpers/constants.ts b/frontend/src2/helpers/constants.ts index 28d9e67d..f2de3db8 100644 --- a/frontend/src2/helpers/constants.ts +++ b/frontend/src2/helpers/constants.ts @@ -24,6 +24,9 @@ export const COLUMN_TYPES = [ { label: 'Datetime', value: 'Datetime' }, ] as const +export const FILTER_TYPES = ['String', 'Number', 'Date'] as const +export type FilterType = typeof FILTER_TYPES[number] + export const joinTypes = [ { label: 'Inner', @@ -54,12 +57,14 @@ export const joinTypes = [ export const granularityOptions = [ - // { label: 'Second', value: 'second' }, - // { label: 'Minute', value: 'minute' }, - // { label: 'Hour', value: 'hour' }, + { label: 'Second', value: 'second' }, + { label: 'Minute', value: 'minute' }, + { label: 'Hour', value: 'hour' }, { label: 'Day', value: 'day'}, { label: 'Week', value: 'week'}, { label: 'Month', value: 'month'}, { label: 'Quarter', value: 'quarter'}, { label: 'Year', value: 'year'}, ] as const + +export type GranularityType = typeof granularityOptions[number]['value'] diff --git a/frontend/src2/helpers/index.ts b/frontend/src2/helpers/index.ts index 3ffa40a0..1c9197e3 100644 --- a/frontend/src2/helpers/index.ts +++ b/frontend/src2/helpers/index.ts @@ -1,7 +1,9 @@ import { watchDebounced } from '@vueuse/core' import domtoimage from 'dom-to-image' +import { call } from 'frappe-ui' import { Socket } from 'socket.io-client' import { ComputedRef, inject, onBeforeUnmount, Ref, watch } from 'vue' +import { getFormattedDate } from '../query/helpers' import session from '../session' import { ColumnDataType, @@ -11,8 +13,6 @@ import { } from '../types/query.types' import { FIELDTYPES } from './constants' import { createToast } from './toasts' -import { getFormattedDate } from '../query/helpers' -import { call } from 'frappe-ui' export function getUniqueId(length = 8) { return (+new Date() * Math.random()).toString(36).substring(0, length) @@ -29,6 +29,7 @@ export function titleCase(str: string) { } export function copy(obj: T) { + if (!obj) return obj return JSON.parse(JSON.stringify(obj)) as T } @@ -40,7 +41,7 @@ export function wheneverChanges( let prevValue: any function onChange(value: any) { if (areDeeplyEqual(value, prevValue)) return - prevValue = value + prevValue = copy(value) callback(value) } return watchDebounced(getter, onChange, options) @@ -126,13 +127,13 @@ export function showErrorToast(err: Error, raise = true) { export function downloadImage(element: HTMLElement, filename: string, scale = 1, options = {}) { return domtoimage .toPng(element, { - height: element.offsetHeight * scale, - width: element.offsetWidth * scale, + height: element.scrollHeight * scale, + width: element.scrollWidth * scale, style: { transform: 'scale(' + scale + ')', transformOrigin: 'top left', - width: element.offsetWidth + 'px', - height: element.offsetHeight + 'px', + width: element.scrollWidth + 'px', + height: element.scrollHeight + 'px', }, bgColor: 'white', ...options, @@ -149,7 +150,7 @@ export function downloadImage(element: HTMLElement, filename: string, scale = 1, }) } -export function formatNumber(number: number, precision = 2) { +export function formatNumber(number: number, precision = 0) { if (isNaN(number)) return number precision = precision || guessPrecision(number) const locale = session.user?.country == 'India' ? 'en-IN' : session.user?.locale @@ -160,6 +161,7 @@ export function formatNumber(number: number, precision = 2) { } export function guessPrecision(number: number) { + if (!number || isNaN(number)) return 0 // eg. 1.0 precision = 1, 1.00 precision = 2 const str = number.toString() const decimalIndex = str.indexOf('.') @@ -275,6 +277,22 @@ export function flattenOptions( : (options as DropdownOption[]) } +export function groupOptions( + options: T[], + groupBy: keyof T +): GroupedDropdownOption[] { + return options.reduce((acc, option) => { + const group = option[groupBy] as string + const index = acc.findIndex((g) => g.group === group) + if (index === -1) { + acc.push({ group, items: [option] }) + } else { + acc[index].items.push(option) + } + return acc + }, [] as GroupedDropdownOption[]) +} + export function scrub(text: string, spacer = '_') { return text.replace(/ /g, spacer).toLowerCase() } @@ -394,9 +412,11 @@ export function createHeaders(columns: QueryResultColumn[]) { // i.e first day of each month, then we format the values as 'Oct 2016', 'Nov 2016', 'Dec 2016' for (let headerRow of groupedHeaders) { + const areDates = areValidDates(headerRow.map((header) => header.label)) + if (!areDates) continue + const areFirstOfYear = areFirstDayOfYear(headerRow.map((header) => header.label)) const areFirstOfMonth = areFirstDayOfMonth(headerRow.map((header) => header.label)) - const areDates = areValidDates(headerRow.map((header) => header.label)) for (let header of headerRow) { if (!isValidDate(header.label)) continue @@ -430,7 +450,11 @@ function areValidDates(data: string[]) { } function isValidDate(value: string) { - return !isNaN(new Date(value).getTime()) + // almost all dates will have a valid 4 digit year + if (!value) return false + if (!/\d{4}/.test(value)) return false + const date = new Date(value) + return !isNaN(date.getTime()) && date.getFullYear() >= 1900 && date.getFullYear() <= 2900 } const callCache = new Map() diff --git a/frontend/src2/helpers/resource.ts b/frontend/src2/helpers/resource.ts index eea771e1..fb1cd169 100644 --- a/frontend/src2/helpers/resource.ts +++ b/frontend/src2/helpers/resource.ts @@ -1,6 +1,6 @@ import { useStorage, watchDebounced } from '@vueuse/core' import { call } from 'frappe-ui' -import { computed, reactive, watchEffect } from 'vue' +import { computed, reactive } from 'vue' import { confirmDialog } from '../helpers/confirm_dialog' import { copy, showErrorToast, waitUntil } from './index' import { createToast } from './toasts' @@ -43,7 +43,9 @@ export default function useDocumentResource( doctype, ...removeMetaFields(this.doc), }, - }).catch(showErrorToast) + }) + .catch(showErrorToast) + .finally(() => (this.saving = false)) this.doc = tranformFn({ ...doc }) this.originalDoc = copy(this.doc) this.name = doc.name @@ -78,7 +80,9 @@ export default function useDocumentResource( doctype: this.doctype, name: this.name, fieldname: removeMetaFields(this.doc), - }).catch(showErrorToast) + }) + .catch(showErrorToast) + .finally(() => (this.saving = false)) if (doc) { this.doc = tranformFn({ ...doc }) @@ -106,7 +110,9 @@ export default function useDocumentResource( const doc = await call('frappe.client.get', { doctype: this.doctype, name: this.name, - }).catch(showErrorToast) + }) + .catch(showErrorToast) + .finally(() => (this.loading = false)) if (!doc) return @@ -160,8 +166,13 @@ export default function useDocumentResource( }) // on creation, restore the doc from local storage, if exists if (storage.value && storage.value.doc) { - resource.doc = storage.value.doc - resource.originalDoc = storage.value.originalDoc + const isStale = + new Date(resource.doc.modified).getTime() > new Date(storage.value.doc.modified).getTime() + + if (!isStale) { + resource.doc = storage.value.doc + resource.originalDoc = storage.value.originalDoc + } } // watch for changes in doc and save it to local storage watchDebounced( diff --git a/frontend/src2/query/components/ColumnFilterValueSelector.vue b/frontend/src2/query/components/ColumnFilterValueSelector.vue index 60429a00..40716037 100644 --- a/frontend/src2/query/components/ColumnFilterValueSelector.vue +++ b/frontend/src2/query/components/ColumnFilterValueSelector.vue @@ -3,13 +3,11 @@ import { watchDebounced } from '@vueuse/core' import { LoadingIndicator } from 'frappe-ui' import { CheckSquare, SearchIcon, Square } from 'lucide-vue-next' import { ref } from 'vue' -import { QueryResultColumn } from '../../types/query.types' const props = defineProps<{ - column: QueryResultColumn valuesProvider: (search: string) => Promise }>() -const selectedValues = defineModel({ +const selectedValues = defineModel({ type: Array, default: () => [], }) diff --git a/frontend/src2/query/components/DatePicker.vue b/frontend/src2/query/components/DatePicker.vue index 5f805de7..be7f29e7 100644 --- a/frontend/src2/query/components/DatePicker.vue +++ b/frontend/src2/query/components/DatePicker.vue @@ -38,9 +38,9 @@ class="flex h-[30px] w-[30px] cursor-pointer items-center justify-center text-sm" :class="{ 'font-bold': toValue(date) === toValue(today), - ' rounded-l bg-gray-800 text-white': + ' rounded-l bg-gray-400 !font-medium': fromDateTxt && toValue(date) === toValue(fromDateTxt), - ' rounded-r bg-gray-800 text-white': + ' rounded-r bg-gray-400 !font-medium': toDateTxt && toValue(date) === toValue(toDateTxt), 'bg-gray-100 font-medium text-gray-800': isInRange(date), }" diff --git a/frontend/src2/query/components/ExpressionEditor.vue b/frontend/src2/query/components/ExpressionEditor.vue index 12e5ec28..e2c3806e 100644 --- a/frontend/src2/query/components/ExpressionEditor.vue +++ b/frontend/src2/query/components/ExpressionEditor.vue @@ -3,9 +3,9 @@ import { debounce } from 'frappe-ui' import { onMounted, ref } from 'vue' import Code from '../../components/Code.vue' import { cachedCall } from '../../helpers' -import { ColumnOption } from '../../types/query.types' +import { DropdownOption } from '../../types/query.types' -const props = defineProps<{ columnOptions: ColumnOption[]; placeholder?: string }>() +const props = defineProps<{ columnOptions: DropdownOption[]; placeholder?: string }>() const expression = defineModel({ required: true, }) diff --git a/frontend/src2/query/components/FilterRule.vue b/frontend/src2/query/components/FilterRule.vue index 027a5b57..58bc8da2 100644 --- a/frontend/src2/query/components/FilterRule.vue +++ b/frontend/src2/query/components/FilterRule.vue @@ -1,7 +1,7 @@ + + + + + + + diff --git a/frontend/src2/query/components/RelativeDatePicker.vue b/frontend/src2/query/components/RelativeDatePicker.vue index b0d2a8ab..e8a74d3d 100644 --- a/frontend/src2/query/components/RelativeDatePicker.vue +++ b/frontend/src2/query/components/RelativeDatePicker.vue @@ -1,8 +1,6 @@ - - - + + - - - - - - - - - - - - Done - - - - + + + + + + {{ checkboxLabel }} + + diff --git a/frontend/src2/query/components/RelativeDatePickerControl.vue b/frontend/src2/query/components/RelativeDatePickerControl.vue new file mode 100644 index 00000000..addc7c57 --- /dev/null +++ b/frontend/src2/query/components/RelativeDatePickerControl.vue @@ -0,0 +1,33 @@ + + + + + + + + + + + + Done + + + + + diff --git a/frontend/src2/query/components/filter_utils.ts b/frontend/src2/query/components/filter_utils.ts index d72bf242..cd3c3614 100644 --- a/frontend/src2/query/components/filter_utils.ts +++ b/frontend/src2/query/components/filter_utils.ts @@ -1,22 +1,56 @@ -import { FIELDTYPES } from '../../helpers/constants' -import { ColumnDataType, FilterExpression, FilterRule } from '../../types/query.types' +import { FIELDTYPES, FilterType } from '../../helpers/constants' +import { + ColumnDataType, + FilterExpression, + FilterOperator, + FilterRule, +} from '../../types/query.types' -export function getValueSelectorType(filter: FilterRule, columnType: ColumnDataType) { - if (!filter.column.column_name || !filter.operator) return 'text' // default to text - if (['is_set', 'is_not_set'].includes(filter.operator)) return +export function getOperatorOptions(filterType: FilterType) { + const options = [] as { label: string; value: FilterOperator }[] + if (filterType === 'String') { + options.push({ label: 'is', value: 'in' }) // value selector + options.push({ label: 'is not', value: 'not_in' }) // value selector + options.push({ label: 'contains', value: 'contains' }) // text + options.push({ label: 'does not contain', value: 'not_contains' }) // text + options.push({ label: 'starts with', value: 'starts_with' }) // text + options.push({ label: 'ends with', value: 'ends_with' }) // text + options.push({ label: 'is set', value: 'is_set' }) // no value + options.push({ label: 'is not set', value: 'is_not_set' }) // no value + } + if (filterType === 'Number') { + options.push({ label: 'equals', value: '=' }) + options.push({ label: 'not equals', value: '!=' }) + options.push({ label: 'greater than', value: '>' }) + options.push({ label: 'greater than or equals', value: '>=' }) + options.push({ label: 'less than', value: '<' }) + options.push({ label: 'less than or equals', value: '<=' }) + options.push({ label: 'between', value: 'between' }) + } + if (filterType === 'Date') { + options.push({ label: 'equals', value: '=' }) + options.push({ label: 'not equals', value: '!=' }) + options.push({ label: 'greater than', value: '>' }) + options.push({ label: 'greater than or equals', value: '>=' }) + options.push({ label: 'less than', value: '<' }) + options.push({ label: 'less than or equals', value: '<=' }) + options.push({ label: 'between', value: 'between' }) + options.push({ label: 'within', value: 'within' }) + } + return options +} + +export function getValueSelectorType(operator: FilterOperator, filterType: FilterType) { + if (['is_set', 'is_not_set'].includes(operator)) return - if (FIELDTYPES.TEXT.includes(columnType)) { - return ['in', 'not_in'].includes(filter.operator) ? 'select' : 'text' + if (filterType === 'String') { + return ['in', 'not_in'].includes(operator) ? 'select' : 'text' } - if (FIELDTYPES.NUMBER.includes(columnType)) { - return filter.operator === 'between' ? 'text' : 'number' + if (filterType === 'Number') { + return operator === 'between' ? 'text' : 'number' } - if (FIELDTYPES.DATE.includes(columnType)) { - return filter.operator === 'between' - ? 'date_range' - : filter.operator === 'within' - ? 'relative_date' - : 'date' + if (filterType === 'Date') { + return operator === 'between' ? 'date_range' : operator === 'within' ? 'relative_date' : 'date' } return 'text' } @@ -25,7 +59,14 @@ export function isFilterExpressionValid(filter: FilterExpression) { return filter.expression.expression.trim().length > 0 } -export function isFilterValid(filter: FilterRule, columnType: ColumnDataType) { +export function getFilterType(columnType: ColumnDataType): FilterType { + if (FIELDTYPES.TEXT.includes(columnType)) return 'String' + if (FIELDTYPES.NUMBER.includes(columnType)) return 'Number' + if (FIELDTYPES.DATE.includes(columnType)) return 'Date' + return 'String' +} + +export function isFilterValid(filter: FilterRule, filterType: FilterType) { if (!filter.column.column_name || !filter.operator) { return false } @@ -33,7 +74,7 @@ export function isFilterValid(filter: FilterRule, columnType: ColumnDataType) { return false } - const valueSelectorType = getValueSelectorType(filter, columnType) + const valueSelectorType = getValueSelectorType(filter.operator, filterType) // if selector type is none, no need to validate if (!valueSelectorType) { @@ -45,14 +86,21 @@ export function isFilterValid(filter: FilterRule, columnType: ColumnDataType) { } // for number, validate if it's a number - if (FIELDTYPES.NUMBER.includes(columnType)) { - return !isNaN(filter.value as any) + if (filterType === 'Number') { + if (filter.operator === 'between') { + return ( + Array.isArray(filter.value) && + filter.value.length === 2 && + filter.value.every(isValidNumber) + ) + } + return isValidNumber(filter.value) } // for text, // if it's a select, validate if it's an array of strings // if it's a text, validate if it's a string - if (FIELDTYPES.TEXT.includes(columnType)) { + if (filterType === 'String') { if (valueSelectorType === 'select') { return Boolean( Array.isArray(filter.value) && @@ -67,7 +115,7 @@ export function isFilterValid(filter: FilterRule, columnType: ColumnDataType) { // for date, // if it's a date, validate if it's a date string // if it's a date range, validate if it's an array of 2 date strings - if (FIELDTYPES.DATE.includes(columnType)) { + if (filterType === 'Date') { if (valueSelectorType === 'date' || valueSelectorType === 'relative_date') { return typeof filter.value === 'string' } else if (valueSelectorType === 'date_range') { @@ -81,3 +129,13 @@ export function isFilterValid(filter: FilterRule, columnType: ColumnDataType) { return false } + +export function isValidNumber(value: any) { + const invalidNaNs = [null, undefined, ''] + return ( + !isNaN(value) && + !invalidNaNs.includes(value) && + typeof value !== 'boolean' && + !isNaN(parseFloat(value)) + ) +} diff --git a/frontend/src2/query/helpers.ts b/frontend/src2/query/helpers.ts index c8391162..0845fc6f 100644 --- a/frontend/src2/query/helpers.ts +++ b/frontend/src2/query/helpers.ts @@ -17,7 +17,7 @@ import { } from 'lucide-vue-next' import { h } from 'vue' import { copy } from '../helpers' -import { FIELDTYPES } from '../helpers/constants' +import { FIELDTYPES, GranularityType } from '../helpers/constants' import dayjs from '../helpers/dayjs' import { Cast, @@ -36,7 +36,6 @@ import { FilterGroupArgs, FilterOperator, FilterValue, - GranularityType, Join, JoinArgs, Limit, @@ -138,7 +137,8 @@ export function getFormattedRows(result: QueryResult, operations: Operation[]) { export function getFormattedDate(date: string, granularity: GranularityType) { if (!date) return '' - const dayjsFormat = { + const dayjsFormat: Record = { + second: 'MMMM D, YYYY h:mm:ss A', minute: 'MMMM D, YYYY h:mm A', hour: 'MMMM D, YYYY h:00 A', day: 'MMMM D, YYYY', diff --git a/frontend/src2/query/query.ts b/frontend/src2/query/query.ts index 2306e4f0..ec2b265a 100644 --- a/frontend/src2/query/query.ts +++ b/frontend/src2/query/query.ts @@ -438,24 +438,38 @@ export function makeQuery(workbookQuery: WorkbookQuery) { } function renameColumn(oldName: string, newName: string) { - // first check if there's already a rename operation for the column const existingRenameIdx = query.currentOperations.findIndex( (op) => op.type === 'rename' && op.new_name === oldName ) - if (existingRenameIdx > -1) { - const existingRename = query.currentOperations[existingRenameIdx] as Rename - existingRename.new_name = newName - } - // if not, add a new rename operation - else { + if (existingRenameIdx === -1) { + // No existing rename, add new one addOperation( rename({ column: column(oldName), new_name: newName, }) ) + return + } + + const existingRename = query.currentOperations[existingRenameIdx] as Rename + const originalColumnName = existingRename.column.column_name + + // If renaming back to original name, remove the rename operation + if (originalColumnName === newName) { + removeOperation(existingRenameIdx) + return } + + // Update existing rename operation + query.doc.operations.splice(existingRenameIdx, 1) + addOperation( + rename({ + column: column(originalColumnName), + new_name: newName, + }) + ) } function removeColumn(column_names: string | string[]) { diff --git a/frontend/src2/teams/TeamResourceSelector.vue b/frontend/src2/teams/TeamResourceSelector.vue index fd182fd9..020028ec 100644 --- a/frontend/src2/teams/TeamResourceSelector.vue +++ b/frontend/src2/teams/TeamResourceSelector.vue @@ -6,6 +6,7 @@ import { computed, ref } from 'vue' import useDataSourceStore from '../data_source/data_source' import { DataSourceListItem } from '../data_source/data_source.types' import useTableStore, { DataSourceTable } from '../data_source/tables' +import { toOptions, wheneverChanges } from '../helpers' import ExpressionEditor from '../query/components/ExpressionEditor.vue' import { TeamPermission } from './teams' @@ -197,6 +198,28 @@ function selectTable(dataSource: string, table: string, selected: boolean) { } const expandedTable = ref(null) +const expandedTableColumns = ref([]) +wheneverChanges( + () => [expandedDataSource.value, expandedTable.value], + () => { + if (!expandedDataSource.value || !expandedTable.value) { + return + } + const table = dataSourceTables.value[expandedDataSource.value].find( + (t) => t.name === expandedTable.value + ) + if (!table) { + return + } + tableStore.getTableColumns(expandedDataSource.value, table.table_name).then((columns) => { + expandedTableColumns.value = toOptions(columns, { + label: 'name', + value: 'name', + description: 'type', + }) + }) + } +) function toggleExpandedTable(table: string) { if (expandedTable.value === table) { expandedTable.value = null @@ -312,7 +335,7 @@ function toggleExpandedTable(table: string) { + default_operator?: FilterOperator + default_value?: FilterValue layout: Layout } export type WorkbookDashboardText = { @@ -85,11 +93,10 @@ export type WorkbookDashboardText = { text: string layout: Layout } -export type DashboardFilterColumn = { - query: string - name: string - type: ColumnDataType -} export type ShareAccess = 'view' | 'edit' | undefined -export type WorkbookSharePermission = { email: string; full_name: string; access: ShareAccess } +export type WorkbookSharePermission = { + email: string + full_name: string + access: ShareAccess +} diff --git a/frontend/src2/workbook/workbook.ts b/frontend/src2/workbook/workbook.ts index 62d612e7..36d14933 100644 --- a/frontend/src2/workbook/workbook.ts +++ b/frontend/src2/workbook/workbook.ts @@ -1,17 +1,15 @@ import { watchDebounced } from '@vueuse/core' import { call } from 'frappe-ui' -import { computed, InjectionKey, reactive, toRefs } from 'vue' +import { computed, InjectionKey, reactive, ref, toRefs } from 'vue' import { useRouter } from 'vue-router' import useChart from '../charts/chart' -import { handleOldYAxisConfig, setDimensionNames } from '../charts/helpers' +import { handleOldXAxisConfig, handleOldYAxisConfig, setDimensionNames } from '../charts/helpers' import useDashboard from '../dashboard/dashboard' import { getUniqueId, safeJSONParse, showErrorToast, wheneverChanges } from '../helpers' import { confirmDialog } from '../helpers/confirm_dialog' import useDocumentResource from '../helpers/resource' -import { createToast } from '../helpers/toasts' import useQuery, { getCachedQuery } from '../query/query' import session from '../session' -import { Join, Source } from '../types/query.types' import type { InsightsWorkbook, WorkbookChart, @@ -194,6 +192,7 @@ export default function useWorkbook(name: string) { } let stopAutoSaveWatcher: any + const _pauseAutoSave = ref(false) wheneverChanges( () => workbook.doc.enable_auto_save, () => { @@ -203,9 +202,9 @@ export default function useWorkbook(name: string) { } if (workbook.doc.enable_auto_save && !stopAutoSaveWatcher) { stopAutoSaveWatcher = watchDebounced( - () => workbook.isdirty, - () => workbook.isdirty && workbook.save(), - { immediate: true, debounce: 1000 } + () => workbook.isdirty && !_pauseAutoSave.value, + (shouldSave) => shouldSave && workbook.save(), + { immediate: true, debounce: 2000 } ) } } @@ -217,6 +216,7 @@ export default function useWorkbook(name: string) { isOwner, showSidebar: true, + _pauseAutoSave, isActiveTab, @@ -278,6 +278,10 @@ function getWorkbookResource(name: string) { chart.config.order_by = chart.config.order_by || [] chart.config.limit = chart.config.limit || 100 + if ('x_axis' in chart.config && chart.config.x_axis) { + // @ts-ignore + chart.config.x_axis = handleOldXAxisConfig(chart.config.x_axis) + } if ('y_axis' in chart.config && Array.isArray(chart.config.y_axis)) { // @ts-ignore chart.config.y_axis = handleOldYAxisConfig(chart.config.y_axis) @@ -286,6 +290,7 @@ function getWorkbookResource(name: string) { // @ts-ignore chart.config.label_position = chart.config.label_position || 'left' } + chart.config = setDimensionNames(chart.config) }) return doc @@ -301,7 +306,6 @@ export function newWorkbookName() { return `new-workbook-${unique_id}` } - export function getLinkedQueries(query_name: string): string[] { const query = getCachedQuery(query_name) if (!query) { @@ -309,34 +313,15 @@ export function getLinkedQueries(query_name: string): string[] { return [] } - const querySource = query.doc.operations.find( - (op) => op.type === 'source' && op.table.type === 'query' && op.table.query_name - ) as Source - - const queryJoins = query.doc.operations.filter( - (op) => op.type === 'join' && op.table.type === 'query' && op.table.query_name - ) as Join[] - - const linkedQueries = [] as string[] - if (querySource && querySource.table.type === 'query') { - linkedQueries.push(querySource.table.query_name) - } - if (queryJoins.length) { - queryJoins.forEach((j) => { - if (j.table.type === 'query') { - linkedQueries.push(j.table.query_name) - } - }) - } + const linkedQueries = new Set() - const linkedQueriesByQuery = {} as Record - linkedQueries.forEach((q) => { - linkedQueriesByQuery[q] = getLinkedQueries(q) + query.doc.operations.forEach((op) => { + if ('table' in op && 'type' in op.table && op.table.type === 'query') { + linkedQueries.add(op.table.query_name) + } }) - Object.values(linkedQueriesByQuery).forEach((subLinkedQueries) => { - linkedQueries.concat(subLinkedQueries) - }) + linkedQueries.forEach((q) => getLinkedQueries(q).forEach((q) => linkedQueries.add(q))) - return linkedQueries + return Array.from(linkedQueries) } diff --git a/insights/api/user.py b/insights/api/user.py index 529284be..8d25f244 100644 --- a/insights/api/user.py +++ b/insights/api/user.py @@ -176,7 +176,8 @@ def update_team(team: dict): ) doc.set("team_permissions", []) for permission in team.team_permissions: - if permission["resource_type"] not in [ + permission = frappe._dict(permission) + if permission.resource_type not in [ "Insights Data Source v3", "Insights Table v3", ]: @@ -184,9 +185,11 @@ def update_team(team: dict): doc.append( "team_permissions", { - "resource_type": permission["resource_type"], - "resource_name": permission["resource_name"], - "table_restrictions": permission["table_restrictions"].strip(), + "resource_type": permission.resource_type, + "resource_name": permission.resource_name, + "table_restrictions": permission.table_restrictions.strip() + if permission.table_restrictions + else None, }, ) doc.save() diff --git a/insights/insights/doctype/insights_data_source_v3/ibis/functions.py b/insights/insights/doctype/insights_data_source_v3/ibis/functions.py index 0abd7ce7..b62d4a9b 100644 --- a/insights/insights/doctype/insights_data_source_v3/ibis/functions.py +++ b/insights/insights/doctype/insights_data_source_v3/ibis/functions.py @@ -6,9 +6,16 @@ import ibis.selectors as s from ibis import _ +from insights.insights.query_builders.sql_functions import handle_timespan + # aggregate functions -def count(column: ir.Column = None, where: ir.BooleanValue = None): +def count( + column: ir.Column = None, + where: ir.BooleanValue = None, + group_by=None, + order_by=None, +): """ def count(column=None, where=None) @@ -18,6 +25,7 @@ def count(column=None, where=None) - count() - count(user_id) - count(user_id, status == 'Active') + - count(user_id, group_by=month(date), order_by=asc(date)) """ if column is None: @@ -25,10 +33,15 @@ def count(column=None, where=None) column = query.columns[0] column = getattr(query, column) + if group_by is not None: + return column.count(where=where).over(group_by=group_by, order_by=order_by) + return column.count(where=where) -def count_if(condition: ir.BooleanValue, column: ir.Column = None): +def count_if( + condition: ir.BooleanValue, column: ir.Column = None, group_by=None, order_by=None +): """ def count_if(condition) @@ -36,12 +49,14 @@ def count_if(condition) Examples: - count_if(status == 'Active') + - count_if(status == 'Active', user_id) + - count_if(status == 'Active', user_id, group_by=month(date), order_by=asc(date)) """ - return count(column, where=condition) + return count(column, where=condition, group_by=group_by, order_by=order_by) -def min(column: ir.Column, where: ir.BooleanValue = None): +def min(column: ir.Column, where: ir.BooleanValue = None, group_by=None, order_by=None): """ def min(column, where=None) @@ -49,12 +64,18 @@ def min(column, where=None) Examples: - min(column) - - min(column, status == 'Active') + - min(column, where=status == 'Active') + - min(column, group_by=user_id, order_by=date) + - min(column, group_by=[user_id, month(date)], order_by=asc(date)) """ + + if group_by is not None: + return column.min(where=where).over(group_by=group_by, order_by=order_by) + return column.min(where=where) -def max(column: ir.Column, where: ir.BooleanValue = None): +def max(column: ir.Column, where: ir.BooleanValue = None, group_by=None, order_by=None): """ def max(column, where=None) @@ -63,11 +84,21 @@ def max(column, where=None) Examples: - max(column) - max(column, status == 'Active') + - max(column, group_by=user_id, order_by=date) """ + + if group_by is not None: + return column.max(where=where).over(group_by=group_by, order_by=order_by) + return column.max(where=where) -def sum(column: ir.NumericColumn, where: ir.BooleanValue = None): +def sum( + column: ir.NumericColumn, + where: ir.BooleanValue = None, + group_by=None, + order_by=None, +): """ def sum(column, where=None) @@ -76,11 +107,21 @@ def sum(column, where=None) Examples: - sum(column) - sum(column, status == 'Active') + - sum(column, group_by=user_id, order_by=date) """ + + if group_by is not None: + return column.sum(where=where).over(group_by=group_by, order_by=order_by) + return column.sum(where=where) -def avg(column: ir.NumericColumn, where: ir.BooleanValue = None): +def avg( + column: ir.NumericColumn, + where: ir.BooleanValue = None, + group_by=None, + order_by=None, +): """ def avg(column, where=None) @@ -89,11 +130,21 @@ def avg(column, where=None) Examples: - avg(column) - avg(column, status == 'Active') + - avg(column, group_by=user_id, order_by=date) """ + + if group_by is not None: + return column.mean(where=where).over(group_by=group_by, order_by=order_by) + return column.mean(where=where) -def median(column: ir.NumericColumn, where: ir.BooleanValue = None): +def median( + column: ir.NumericColumn, + where: ir.BooleanValue = None, + group_by=None, + order_by=None, +): """ def median(column, where=None) @@ -102,7 +153,12 @@ def median(column, where=None) Examples: - median(column) - median(column, status == 'Active') + - median(column, group_by=user_id, order_by=date) """ + + if group_by is not None: + return column.median(where=where).over(group_by=group_by, order_by=order_by) + return column.median(where=where) @@ -119,7 +175,9 @@ def group_concat(column, sep=',', where=None) return column.group_concat(sep=sep, where=where) -def distinct_count(column: ir.Column, where: ir.BooleanValue = None): +def distinct_count( + column: ir.Column, where: ir.BooleanValue = None, group_by=None, order_by=None +): """ def distinct_count(column, where=None) @@ -128,7 +186,12 @@ def distinct_count(column, where=None) Examples: - distinct_count(column) - distinct_count(column, status == 'Active') + - distinct_count(column, group_by=user_id, order_by=date) """ + + if group_by is not None: + return column.nunique(where=where).over(group_by=group_by, order_by=order_by) + return column.nunique(where=where) @@ -619,6 +682,47 @@ def date_diff(column, other, unit) return column.delta(other, unit) +def date_add(column: ir.DateValue, value: int, unit: str): + """ + def date_add(column, value, unit) + + Add a value to a date column. The unit can be seconds, minutes, hours, days, weeks, months, or years. + + Examples: + - date_add(order_date, 1, 'days') + - date_add(order_date, 1, 'weeks') + """ + return column + ibis.interval(value, unit) + + +def date_sub(column: ir.DateValue, value: int, unit: str): + """ + def date_sub(column, value, unit) + + Subtract a value from a date column. The unit can be seconds, minutes, hours, days, weeks, months, or years. + + Examples: + - date_sub(order_date, 1, 'days') + - date_sub(order_date, 1, 'weeks') + """ + return column - ibis.interval(value, unit) + + +def within(column: ir.DateValue, timespan: str): + """ + def within(column, timespan) + + Filter rows within a timespan. The timespan can be 'Last [N] [Parts]', 'Current [Parts]', or 'Next [N] [Parts]'. + Parts can be 'day', 'week', 'month', 'quarter', 'year' or 'fiscal year'. + + Examples: + - within(order_date, 'Last 7 days') + - within(order_date, 'Current month') + - within(order_date, 'Next 2 weeks') + """ + return handle_timespan(column, timespan) + + def now(): """ def now() @@ -963,6 +1067,51 @@ def week_start(column) return week_start +def month_start(column: ir.DateValue): + """ + def month_start(column) + + Get the start date of the month for a given date. + + Examples: + - month_start(order_date) + """ + + month_start = column.strftime("%Y-%m-01").cast("date") + return month_start + + +def quarter_start(column: ir.DateValue): + """ + def quarter_start(column) + + Get the start date of the quarter for a given date. + + Examples: + - quarter_start(order_date) + """ + + year = column.year() + quarter = column.quarter() + month = (quarter * 3) - 2 + quarter_start = ibis.date(year, month, 1) + return quarter_start + + +def year_start(column: ir.DateValue): + """ + def year_start(column) + + Get the start date of the year for a given date. + + Examples: + - year_start(order_date) + """ + + year_start = column.strftime("%Y-01-01").cast("date") + return year_start + + def get_retention_data(date_column: ir.DateValue, id_column: ir.Column, unit: str): """ def get_retention_data(date_column, id_column, unit) @@ -983,6 +1132,9 @@ def get_retention_data(date_column, id_column, unit) if isinstance(id_column, str): id_column = getattr(query, id_column) + if date_column.type().is_timestamp(): + date_column = date_column.cast("date") + if not date_column.type().is_date(): frappe.throw(f"Invalid date column. Expected date, got {date_column.type()}") @@ -990,6 +1142,7 @@ def get_retention_data(date_column, id_column, unit) "day": lambda column: column.strftime("%Y-%m-%d").cast("date"), "week": week_start, "month": lambda column: column.strftime("%Y-%m-01").cast("date"), + "quarter": quarter_start, "year": lambda column: column.strftime("%Y-01-01").cast("date"), }[unit] @@ -1017,3 +1170,23 @@ def get_retention_data(date_column, id_column, unit) query = query.mutate(retention=(query.unique_ids / query.cohort_size) * 100) return query + + +def pad_number(number: ir.NumericValue, digits: int): + """ + def pad_number(number, digits) + + Convert an integer into an n digit string. + + Examples: + - pad_number(1, 2) -> '01' + - pad_number(1, 3) -> '001' + """ + string_number = number.cast("string") + zero_literal = ibis.literal("0") + + return if_else( + string_number.length() < digits, + zero_literal.repeat(digits - string_number.length()).concat(string_number), + string_number, + ) diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py index a7d78ee1..46f7a2ec 100644 --- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py +++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py @@ -14,6 +14,7 @@ from ibis.expr.types import Table as IbisQuery from insights.cache_utils import make_digest +from insights.insights.doctype.insights_data_source_v3.data_warehouse import Warehouse from insights.insights.doctype.insights_data_source_v3.insights_data_source_v3 import ( DataSourceConnectionError, ) @@ -24,7 +25,7 @@ from insights.utils import create_execution_log from insights.utils import deep_convert_dict_to_dict as _dict -from .ibis.functions import week_start +from .ibis.functions import quarter_start, week_start from .ibis.utils import get_functions @@ -279,6 +280,9 @@ def make_filter_condition(self, filter_args): else None ) + if filter_operator in ["contains", "not_contains"]: + filter_value = filter_value.replace("%", "") + right_value = right_column or filter_value return operator_fn(left, right_value) @@ -294,8 +298,8 @@ def get_operator(self, operator): "not_in": lambda x, y: ~x.isin(y), "is_set": lambda x, y: (x.notnull()) & (x != ""), "is_not_set": lambda x, y: (x.isnull()) | (x == ""), - "contains": lambda x, y: x.like(y), - "not_contains": lambda x, y: ~x.like(y), + "contains": lambda x, y: x.like(f"%{y}%"), + "not_contains": lambda x, y: ~x.like(f"%{y}%"), "starts_with": lambda x, y: x.like(f"{y}%"), "ends_with": lambda x, y: x.like(f"%{y}"), "between": lambda x, y: x.between(y[0], y[1]), @@ -440,8 +444,7 @@ def apply_code(self, code_args): digest = make_digest(code) results = get_code_results(code, digest) - db = ibis.duckdb.connect() - return db.create_table( + return Warehouse().db.create_table( digest, results, temp=True, @@ -494,13 +497,12 @@ def apply_granularity(self, column, granularity): if granularity == "week": return week_start(column).strftime("%Y-%m-%d").name(column.get_name()) if granularity == "quarter": - year = column.year() - quarter = column.quarter() - month = (quarter * 3) - 2 - quarter_start = ibis.date(year, month, 1) - return quarter_start.strftime("%Y-%m-%d").name(column.get_name()) + return quarter_start(column).strftime("%Y-%m-01").name(column.get_name()) format_str = { + "second": "%Y-%m-%d %H:%M:%S", + "minute": "%Y-%m-%d %H:%M:00", + "hour": "%Y-%m-%d %H:00:00", "day": "%Y-%m-%d", "month": "%Y-%m-01", "year": "%Y-01-01", @@ -542,7 +544,17 @@ def execute_ibis_query(query: IbisQuery, limit=100, cache=True, cache_expiry=360 query = query.head(limit) if limit else query start = time.monotonic() - result: pd.DataFrame = query.execute() + + try: + result: pd.DataFrame = query.execute() + except Exception as e: + if "max_statement_time" in str(e): + frappe.throw( + title="Query Timeout", + msg="Query execution time exceeded the limit. Please try again with a smaller timespan or a more specific filter.", + ) + raise e + time_taken = flt(time.monotonic() - start, 3) create_execution_log(sql, time_taken) diff --git a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py index 5bd22bd4..526532ed 100644 --- a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py +++ b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py @@ -193,6 +193,13 @@ def _get_ibis_backend(self) -> BaseBackend: if self.database_type == "MariaDB": db.raw_sql("SET SESSION time_zone='+00:00'") db.raw_sql("SET collation_connection = 'utf8mb4_unicode_ci'") + MAX_STATEMENT_TIMEOUT = ( + frappe.db.get_single_value( + "Insights Settings", "max_execution_time", cache=True + ) + or 180 + ) + db.raw_sql(f"SET MAX_STATEMENT_TIME={MAX_STATEMENT_TIMEOUT}") frappe.local.insights_db_connections[self.name] = db return db @@ -312,7 +319,7 @@ def before_request(): def after_request(): - for db in getattr(frappe.local, 'insights_db_connections', {}).values(): + for db in getattr(frappe.local, "insights_db_connections", {}).values(): catch_error(db.disconnect) diff --git a/insights/insights/doctype/insights_settings/insights_settings.json b/insights/insights/doctype/insights_settings/insights_settings.json index b7396a71..b02225b6 100644 --- a/insights/insights/doctype/insights_settings/insights_settings.json +++ b/insights/insights/doctype/insights_settings/insights_settings.json @@ -13,6 +13,7 @@ "allowed_origins", "max_records_to_sync", "max_memory_usage", + "max_execution_time", "integrations_section", "telegram_api_token", "query_section", @@ -134,12 +135,17 @@ "fieldname": "apply_user_permissions", "fieldtype": "Check", "label": "Apply User Permissions" + }, + { + "fieldname": "max_execution_time", + "fieldtype": "Int", + "label": "Max Execution Time (Seconds)" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2024-12-03 16:27:21.761873", + "modified": "2025-01-14 13:17:25.798141", "modified_by": "Administrator", "module": "Insights", "name": "Insights Settings", diff --git a/insights/insights/doctype/insights_settings/insights_settings.py b/insights/insights/doctype/insights_settings/insights_settings.py index e0c7dcba..1678c8cc 100644 --- a/insights/insights/doctype/insights_settings/insights_settings.py +++ b/insights/insights/doctype/insights_settings/insights_settings.py @@ -24,6 +24,7 @@ class InsightsSettings(Document): enable_data_store: DF.Check enable_permissions: DF.Check fiscal_year_start: DF.Date | None + max_execution_time: DF.Int max_memory_usage: DF.Int max_records_to_sync: DF.Int onboarding_complete: DF.Check diff --git a/pyproject.toml b/pyproject.toml index eecd5856..5773b11c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "pandas~=2.2.2", "SQLAlchemy==2.0.22", "python-telegram-bot==21.4", - "duckdb==1.0.0", + "duckdb==1.1.3", "ibis-framework==9.5.0", "ibis-framework[duckdb]", "ibis-framework[mysql]",
- Showing {{ drillDownQuery.result.rows.length }} of - {{ drillDownQuery.result.totalRowCount }} rows -
+ Showing {{ drillDownQuery.result.rows.length }} of + {{ drillDownQuery.result.totalRowCount }} rows +
Chart not found
+ {{ label || 'Filter' }} +
{{ link.title }}
Markdown supported