From 36323acffad6e7658fb0ab1c9a73d5a5bceff2e5 Mon Sep 17 00:00:00 2001 From: AkselObdrup Date: Tue, 27 Jul 2021 13:06:16 +0200 Subject: [PATCH 01/13] Added upload/download data as csv buttons --- components/data-points.tsx | 118 +++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 10 deletions(-) diff --git a/components/data-points.tsx b/components/data-points.tsx index b2cc755e..890afb17 100644 --- a/components/data-points.tsx +++ b/components/data-points.tsx @@ -1,4 +1,4 @@ -import { CircularProgress, IconButton } from "@material-ui/core"; +import { CircularProgress, IconButton, Button, Grid, Input } from "@material-ui/core"; import { useEffect, useReducer } from "react"; import { useGlobal } from "../context/global-context"; import { dataPointsReducer, DataPointsState } from "../reducers/data-points-reducer"; @@ -7,6 +7,9 @@ import { EditableTable } from "./editable-table"; import SwapVertIcon from '@material-ui/icons/SwapVert'; import { TitleCard } from './title-card'; import useStyles from "../styles/data-points.style"; +import { useExperiment } from '../context/experiment-context'; + + type DataPointProps = { valueVariables: ValueVariableType[] @@ -131,22 +134,117 @@ export default function DataPoints(props: DataPointProps) { updateFn(rowIndex, ...args) } + function DownloadCSV() { + let csvContent = "data:text/csv;charset=utf-8,"; + state.rows[0].dataPoints.map((item, index) => { + if (index < state.rows[0].dataPoints.length - 1) { + csvContent += item.name + ","; + } + else { + csvContent += item.name + "\r\n"; + } + }); + state.rows.forEach(function (rowArray, rowIndex) { + rowArray.dataPoints.map((item, index) => { + if (state.rows[rowIndex].dataPoints[0].value !== '') { + if (index < rowArray.dataPoints.length - 1) { + csvContent += item.value + ","; + } + else if (rowIndex < state.rows.length - 1 && state.rows[rowIndex + 1].dataPoints[0].value !== '') { + csvContent += item.value + "\r\n"; + } + else { + csvContent += item.value + } + } + }) + }); + var encodedUri = encodeURI(csvContent); + var link = document.createElement("a"); + link.setAttribute("href", encodedUri); + link.setAttribute("download", experiment.id + ".csv"); + document.body.appendChild(link); + link.click(); + } + + function UploadCSV(e) { + const init_data_points = state.rows[state.rows.length - 1].dataPoints[0].value == "" ? state.rows.length - 1 : state.rows.length + const file = e.target.files[0]; + if (!file) { + return; + } + var reader = new FileReader(); + reader.onload = function (e) { + var contents = e.target.result; + var data = String(contents).split(/\r\n|\n/); + if (data[0] !== String(state.rows[0].dataPoints.map((item, index) => item.name))) { + alert("Headers of the CSV are not correct" + "\r\nExpected: " + String(state.rows[0].dataPoints.map((item, index) => item.name)) + + "\r\nBut got: " + data[0]) + return; + } + var dims = (data[0].match(/,/g) || []).length + for (let i = 1; i < data.length; i++) { + if ((data[i].match(/,/g) || []).length == dims) { + if (i > 1 || state.rows[state.rows.length - 1].dataPoints[0].value != "") { + addRow(buildEmptyRow()) + } + var data_array = String(data[i]).split(','); + for (let j = 0; j < data_array.length; j++) { + updateRow(init_data_points - 1 + i, edit, data_array[j], j) + } + } else { + alert("Wrong amount of variables in line " + i + "\r\nExpected: " + dims + "\r\nBut got: " + (data[i].match(/,/g) || []).length) + } + } + }; + reader.readAsText(file) + } + return ( - Data points - global.dispatch({ type: 'setDataPointsNewestFirst', payload: !global.state.dataPointsNewestFirst })}> - - + + + Data points + + + + + + + global.dispatch({ type: 'setDataPointsNewestFirst', payload: !global.state.dataPointsNewestFirst })}> + + + }> {buildCombinedVariables().length === 0 && "Data points will appear here"} {buildCombinedVariables().length > 0 && isLoadingState && - - } + + } {buildCombinedVariables().length > 0 && !isLoadingState && Date: Tue, 27 Jul 2021 13:13:06 +0200 Subject: [PATCH 02/13] Added upload/download data as csv buttons --- components/data-points.tsx | 45 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/components/data-points.tsx b/components/data-points.tsx index 890afb17..856ff853 100644 --- a/components/data-points.tsx +++ b/components/data-points.tsx @@ -23,32 +23,35 @@ type UpdateFnType = (rowIndex: number, ...args: any[]) => void const SCORE = "score" export default function DataPoints(props: DataPointProps) { - const { valueVariables, categoricalVariables, dataPoints, onUpdateDataPoints} = props + const { valueVariables, categoricalVariables, dataPoints, onUpdateDataPoints } = props const classes = useStyles() const [state, dispatch] = useReducer(dataPointsReducer, { rows: [], prevRows: [] }) + const { state: { + experiment + } } = useExperiment() const isLoadingState = state.rows.length === 0 const global = useGlobal() const newestFirst = global.state.dataPointsNewestFirst useEffect(() => { - dispatch({ type: 'setInitialState', payload: buildState()}) + dispatch({ type: 'setInitialState', payload: buildState() }) }, [valueVariables, categoricalVariables]) const buildState = (): DataPointsState => { const combinedVariables: CombinedVariableType[] = buildCombinedVariables() const emptyRow: TableDataRow = buildEmptyRow() const dataPointRows: TableDataRow[] = dataPoints.map((item, i) => { - return { - dataPoints: item.map((point: TableDataPoint, k) => { - return { - ...point, - options: combinedVariables[k] ? combinedVariables[k].options : undefined, - } - }), - isEditMode: false, - isNew: false, - } + return { + dataPoints: item.map((point: TableDataPoint, k) => { + return { + ...point, + options: combinedVariables[k] ? combinedVariables[k].options : undefined, + } + }), + isEditMode: false, + isNew: false, } + } ).concat(emptyRow as any) return { @@ -57,7 +60,7 @@ export default function DataPoints(props: DataPointProps) { } } - const buildCombinedVariables = (): CombinedVariableType[] => { + const buildCombinedVariables = (): CombinedVariableType[] => { return (valueVariables as CombinedVariableType[]).concat(categoricalVariables as CombinedVariableType[]) } @@ -92,14 +95,16 @@ export default function DataPoints(props: DataPointProps) { } function edit(rowIndex: number, editValue: string, itemIndex: number) { - dispatch({ type: 'DATA_POINTS_TABLE_EDITED', payload: { - itemIndex, - rowIndex, - useArrayForValue: SCORE, - value: editValue - }}) + dispatch({ + type: 'DATA_POINTS_TABLE_EDITED', payload: { + itemIndex, + rowIndex, + useArrayForValue: SCORE, + value: editValue + } + }) } - + function deleteRow(rowIndex: number) { dispatch({ type: 'DATA_POINTS_TABLE_ROW_DELETED', payload: rowIndex }) } From 76d5299ce44590724e62c70ec71a6a0aaf7ee246 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Mon, 16 Aug 2021 12:40:41 +0200 Subject: [PATCH 03/13] More generic save to file mechanism --- components/experiment.tsx | 2 +- utility/save-to-local-file.test.ts | 20 ++++++++++++++++++++ utility/save-to-local-file.ts | 10 +++++----- 3 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 utility/save-to-local-file.test.ts diff --git a/components/experiment.tsx b/components/experiment.tsx index b1d4cb5f..e8ac693f 100644 --- a/components/experiment.tsx +++ b/components/experiment.tsx @@ -46,7 +46,7 @@ export default function Experiment(props: ExperimentProps) { }, [experiment]) const onDownload = () => { - saveToLocalFile(experiment, experiment.id) + saveToLocalFile(experiment, experiment.id, 'application/json') } const onSave = async () => { diff --git a/utility/save-to-local-file.test.ts b/utility/save-to-local-file.test.ts new file mode 100644 index 00000000..23a18ed2 --- /dev/null +++ b/utility/save-to-local-file.test.ts @@ -0,0 +1,20 @@ +import { saveToLocalFile } from './save-to-local-file' + +describe('save-to-local-file', () => { + describe('saveToLocalFile', () => { + it('download file', () => { + const objectToSave = {} + const link: any = { + click: jest.fn(), + } + jest.spyOn(document, 'createElement').mockImplementation(() => link) + + saveToLocalFile(objectToSave, 'the-file-name', 'application/json') + + expect(link.className).toEqual('download-helper') + expect(link.download).toEqual('the-file-name') + expect(link.href).toEqual('data:application/json;charset=utf-8;,%7B%7D') + expect(link.click).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/utility/save-to-local-file.ts b/utility/save-to-local-file.ts index 473be9ac..2baabaa8 100644 --- a/utility/save-to-local-file.ts +++ b/utility/save-to-local-file.ts @@ -1,10 +1,10 @@ -export default function saveToLocalFile(payload: object, filename: string): void { - const contentType = 'application/json;charset=utf-8;' +export const saveToLocalFile = function saveToLocalFile(payload: object, filename: string, mimeType: string): void { + const contentType = `${mimeType};charset=utf-8;` const a = document.createElement('a'); + a.className = 'download-helper' a.download = filename; a.href = 'data:' + contentType + ',' + encodeURIComponent(JSON.stringify(payload, null, 2)); a.target = '_blank'; - document.body.appendChild(a); a.click(); - document.body.removeChild(a); -} \ No newline at end of file +} +export default saveToLocalFile \ No newline at end of file From 09a432313052e54996b9ab347b0434d9c20dd4b9 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Mon, 16 Aug 2021 12:40:54 +0200 Subject: [PATCH 04/13] Fix potential character bug --- types/common.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/types/common.ts b/types/common.ts index 8438a4f1..76630117 100644 --- a/types/common.ts +++ b/types/common.ts @@ -37,7 +37,7 @@ export type ValueVariableType = { maxVal: number } -export type VariableType = CategoricalVariableType | ValueVariableType +export type VariableType = CategoricalVariableType | ValueVariableType export type OptimizerConfig = { baseEstimator: string @@ -65,7 +65,7 @@ export type ScoreDataPointType = { export type SpaceType = {type: string, name:string, from?: number, to?: number, categories?: string[]}[] -export type TableDataPointValue = string | number | number[] +export type TableDataPointValue = string | number | number[] export type TableDataPoint = { name: string From 144769dc3a2f21d0d609dbfe6866ccba8267de95 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Mon, 16 Aug 2021 12:53:58 +0200 Subject: [PATCH 05/13] Add save to CSV and Object function --- components/data-points.tsx | 10 +++----- components/experiment.tsx | 4 +-- utility/save-to-local-file.test.ts | 39 ++++++++++++++++++++++++++---- utility/save-to-local-file.ts | 6 +++-- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/components/data-points.tsx b/components/data-points.tsx index 856ff853..5a3e8606 100644 --- a/components/data-points.tsx +++ b/components/data-points.tsx @@ -8,6 +8,7 @@ import SwapVertIcon from '@material-ui/icons/SwapVert'; import { TitleCard } from './title-card'; import useStyles from "../styles/data-points.style"; import { useExperiment } from '../context/experiment-context'; +import { saveCSVToLocalFile } from "../utility/save-to-local-file"; @@ -140,7 +141,7 @@ export default function DataPoints(props: DataPointProps) { } function DownloadCSV() { - let csvContent = "data:text/csv;charset=utf-8,"; + let csvContent = ""; state.rows[0].dataPoints.map((item, index) => { if (index < state.rows[0].dataPoints.length - 1) { csvContent += item.name + ","; @@ -164,12 +165,7 @@ export default function DataPoints(props: DataPointProps) { } }) }); - var encodedUri = encodeURI(csvContent); - var link = document.createElement("a"); - link.setAttribute("href", encodedUri); - link.setAttribute("download", experiment.id + ".csv"); - document.body.appendChild(link); - link.click(); + saveCSVToLocalFile(csvContent, experiment.id + ".csv") } function UploadCSV(e) { diff --git a/components/experiment.tsx b/components/experiment.tsx index e8ac693f..188e03da 100644 --- a/components/experiment.tsx +++ b/components/experiment.tsx @@ -11,7 +11,7 @@ import React, { useState, useEffect } from 'react'; import { ValueVariableType, CategoricalVariableType, OptimizerConfig, DataPointType } from '../types/common'; import LoadingExperiment from './loading-experiment'; import { NextExperiments } from './next-experiment'; -import saveToLocalFile from '../utility/save-to-local-file'; +import { saveObjectToLocalFile } from '../utility/save-to-local-file'; import LoadingButton from './loading-button'; import { theme } from '../theme/theme'; import { Plots } from './plots'; @@ -46,7 +46,7 @@ export default function Experiment(props: ExperimentProps) { }, [experiment]) const onDownload = () => { - saveToLocalFile(experiment, experiment.id, 'application/json') + saveObjectToLocalFile(experiment, experiment.id) } const onSave = async () => { diff --git a/utility/save-to-local-file.test.ts b/utility/save-to-local-file.test.ts index 23a18ed2..86d233f0 100644 --- a/utility/save-to-local-file.test.ts +++ b/utility/save-to-local-file.test.ts @@ -1,20 +1,49 @@ -import { saveToLocalFile } from './save-to-local-file' +import { saveToLocalFile, saveCSVToLocalFile, saveObjectToLocalFile } from './save-to-local-file' describe('save-to-local-file', () => { describe('saveToLocalFile', () => { - it('download file', () => { - const objectToSave = {} + it('Saves file to local disk', () => { + const payload = 'hello' const link: any = { click: jest.fn(), } jest.spyOn(document, 'createElement').mockImplementation(() => link) - saveToLocalFile(objectToSave, 'the-file-name', 'application/json') + saveToLocalFile(payload, 'the-file-name', 'application/json') expect(link.className).toEqual('download-helper') expect(link.download).toEqual('the-file-name') - expect(link.href).toEqual('data:application/json;charset=utf-8;,%7B%7D') + expect(link.href).toEqual('data:application/json;charset=utf-8;,hello') expect(link.click).toHaveBeenCalledTimes(1) }) }) + + describe('saveCSVToLocalFile', () => { + it('Saves CSV', () => { + const payload = 'hello,world' + const link: any = { + click: jest.fn(), + } + jest.spyOn(document, 'createElement').mockImplementation(() => link) + saveCSVToLocalFile(payload, 'file.csv') + expect(link.download).toEqual('file.csv') + expect(link.href).toEqual('data:text/csv;charset=utf-8;,hello%2Cworld') + expect(link.click).toHaveBeenCalledTimes(1) + }) + }) + describe('saveObjectToLocalFile', () => { + it('Saves JSON', () => { + const payload = { + name: 'test' + } + const link: any = { + click: jest.fn(), + } + jest.spyOn(document, 'createElement').mockImplementation(() => link) + saveObjectToLocalFile(payload, 'file.json') + expect(link.download).toEqual('file.json') + expect(link.href).toEqual('data:application/json;charset=utf-8;,%7B%0A%20%20%22name%22%3A%20%22test%22%0A%7D') + expect(link.click).toHaveBeenCalledTimes(1) + }) + }) }) diff --git a/utility/save-to-local-file.ts b/utility/save-to-local-file.ts index 2baabaa8..a347fbb7 100644 --- a/utility/save-to-local-file.ts +++ b/utility/save-to-local-file.ts @@ -1,10 +1,12 @@ -export const saveToLocalFile = function saveToLocalFile(payload: object, filename: string, mimeType: string): void { +export const saveToLocalFile = function saveToLocalFile(payload: string, filename: string, mimeType: string): void { const contentType = `${mimeType};charset=utf-8;` const a = document.createElement('a'); a.className = 'download-helper' a.download = filename; - a.href = 'data:' + contentType + ',' + encodeURIComponent(JSON.stringify(payload, null, 2)); + a.href = 'data:' + contentType + ',' + encodeURIComponent(payload); a.target = '_blank'; a.click(); } +export const saveCSVToLocalFile = (payload, filename) => saveToLocalFile(payload, filename, 'text/csv') +export const saveObjectToLocalFile = (payload, filename) => saveToLocalFile(JSON.stringify(payload, null, 2), filename, 'application/json') export default saveToLocalFile \ No newline at end of file From 8fa621c1e69468917747bf2934b7c5cb812b9b1e Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Tue, 17 Aug 2021 08:39:45 +0200 Subject: [PATCH 06/13] Formatting --- utility/save-to-local-file.test.ts | 41 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/utility/save-to-local-file.test.ts b/utility/save-to-local-file.test.ts index 86d233f0..d579a3d1 100644 --- a/utility/save-to-local-file.test.ts +++ b/utility/save-to-local-file.test.ts @@ -20,30 +20,31 @@ describe('save-to-local-file', () => { describe('saveCSVToLocalFile', () => { it('Saves CSV', () => { - const payload = 'hello,world' - const link: any = { + const payload = 'hello,world' + const link: any = { click: jest.fn(), - } - jest.spyOn(document, 'createElement').mockImplementation(() => link) - saveCSVToLocalFile(payload, 'file.csv') - expect(link.download).toEqual('file.csv') - expect(link.href).toEqual('data:text/csv;charset=utf-8;,hello%2Cworld') - expect(link.click).toHaveBeenCalledTimes(1) - }) + } + jest.spyOn(document, 'createElement').mockImplementation(() => link) + saveCSVToLocalFile(payload, 'file.csv') + expect(link.download).toEqual('file.csv') + expect(link.href).toEqual('data:text/csv;charset=utf-8;,hello%2Cworld') + expect(link.click).toHaveBeenCalledTimes(1) + }) }) + describe('saveObjectToLocalFile', () => { it('Saves JSON', () => { - const payload = { - name: 'test' - } - const link: any = { + const payload = { + name: 'test' + } + const link: any = { click: jest.fn(), - } - jest.spyOn(document, 'createElement').mockImplementation(() => link) - saveObjectToLocalFile(payload, 'file.json') - expect(link.download).toEqual('file.json') - expect(link.href).toEqual('data:application/json;charset=utf-8;,%7B%0A%20%20%22name%22%3A%20%22test%22%0A%7D') - expect(link.click).toHaveBeenCalledTimes(1) - }) + } + jest.spyOn(document, 'createElement').mockImplementation(() => link) + saveObjectToLocalFile(payload, 'file.json') + expect(link.download).toEqual('file.json') + expect(link.href).toEqual('data:application/json;charset=utf-8;,%7B%0A%20%20%22name%22%3A%20%22test%22%0A%7D') + expect(link.click).toHaveBeenCalledTimes(1) + }) }) }) From 4053a86ff6d7bd10c228f6683f702e2f469d7913 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Wed, 8 Sep 2021 09:02:47 +0200 Subject: [PATCH 07/13] Add types to utility method --- utility/save-to-local-file.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utility/save-to-local-file.ts b/utility/save-to-local-file.ts index a347fbb7..138f35dc 100644 --- a/utility/save-to-local-file.ts +++ b/utility/save-to-local-file.ts @@ -7,6 +7,6 @@ export const saveToLocalFile = function saveToLocalFile(payload: string, filenam a.target = '_blank'; a.click(); } -export const saveCSVToLocalFile = (payload, filename) => saveToLocalFile(payload, filename, 'text/csv') -export const saveObjectToLocalFile = (payload, filename) => saveToLocalFile(JSON.stringify(payload, null, 2), filename, 'application/json') -export default saveToLocalFile \ No newline at end of file +export const saveCSVToLocalFile = (payload: string, filename: string) => saveToLocalFile(payload, filename, 'text/csv') +export const saveObjectToLocalFile = (payload: object, filename: string) => saveToLocalFile(JSON.stringify(payload, null, 2), filename, 'application/json') +export default saveToLocalFile From 88ff2a117b2c6eb2c26454c1aa57af61a34473f7 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Wed, 8 Sep 2021 11:33:24 +0200 Subject: [PATCH 08/13] Fix unit test error output --- context/experiment-context.test.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/context/experiment-context.test.tsx b/context/experiment-context.test.tsx index 46cfac53..0207c2ee 100644 --- a/context/experiment-context.test.tsx +++ b/context/experiment-context.test.tsx @@ -3,11 +3,13 @@ import { useExperiment, TestExperimentProvider } from "./experiment-context"; describe("useExperiment", () => { it("fails if called outside provider", async () => { + console.error = jest.fn() function ExperimentTester() { const context = useExperiment() return <>{JSON.stringify(context)} } expect(() => render()).toThrow("useExperiment must be used within an ExperimentProvider") + expect(console.error).toHaveBeenCalled() }) it("provides context when called inside provider", async () => { From 13c547f89aa4e9749f99ad3adc1ddf8304809ce2 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Wed, 8 Sep 2021 15:29:35 +0200 Subject: [PATCH 09/13] Fix looping unit tests --- jest.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/jest.config.js b/jest.config.js index bd57bda2..99641368 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,6 +1,7 @@ module.exports = { setupFilesAfterEnv: ['/jest.setup.ts'], testPathIgnorePatterns: ['/.next/', '/node_modules/'], + watchPathIgnorePatterns: ['/boost-tests-', '/tmp/'], moduleNameMapper: { '\\.(scss|sass|css)$': 'identity-obj-proxy', }, From 854e4a6387b27f0846312a545d957e9a27e939c4 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Wed, 8 Sep 2021 15:30:54 +0200 Subject: [PATCH 10/13] Move csv buttons to separate files --- components/data-points.tsx | 94 +----- components/download-csv-button.tsx | 15 + components/upload-csv-button.tsx | 36 +++ utility/__snapshots__/converters.test.ts.snap | 3 + utility/converters.test.ts | 293 ++++++++++++++---- utility/converters.ts | 45 ++- 6 files changed, 337 insertions(+), 149 deletions(-) create mode 100644 components/download-csv-button.tsx create mode 100644 components/upload-csv-button.tsx create mode 100644 utility/__snapshots__/converters.test.ts.snap diff --git a/components/data-points.tsx b/components/data-points.tsx index 5a3e8606..f3a18c4c 100644 --- a/components/data-points.tsx +++ b/components/data-points.tsx @@ -7,10 +7,8 @@ import { EditableTable } from "./editable-table"; import SwapVertIcon from '@material-ui/icons/SwapVert'; import { TitleCard } from './title-card'; import useStyles from "../styles/data-points.style"; -import { useExperiment } from '../context/experiment-context'; -import { saveCSVToLocalFile } from "../utility/save-to-local-file"; - - +import DownloadCSVButton from "./download-csv-button"; +import UploadCSVButton from "./upload-csv-button"; type DataPointProps = { valueVariables: ValueVariableType[] @@ -27,9 +25,6 @@ export default function DataPoints(props: DataPointProps) { const { valueVariables, categoricalVariables, dataPoints, onUpdateDataPoints } = props const classes = useStyles() const [state, dispatch] = useReducer(dataPointsReducer, { rows: [], prevRows: [] }) - const { state: { - experiment - } } = useExperiment() const isLoadingState = state.rows.length === 0 const global = useGlobal() const newestFirst = global.state.dataPointsNewestFirst @@ -140,67 +135,6 @@ export default function DataPoints(props: DataPointProps) { updateFn(rowIndex, ...args) } - function DownloadCSV() { - let csvContent = ""; - state.rows[0].dataPoints.map((item, index) => { - if (index < state.rows[0].dataPoints.length - 1) { - csvContent += item.name + ","; - } - else { - csvContent += item.name + "\r\n"; - } - }); - state.rows.forEach(function (rowArray, rowIndex) { - rowArray.dataPoints.map((item, index) => { - if (state.rows[rowIndex].dataPoints[0].value !== '') { - if (index < rowArray.dataPoints.length - 1) { - csvContent += item.value + ","; - } - else if (rowIndex < state.rows.length - 1 && state.rows[rowIndex + 1].dataPoints[0].value !== '') { - csvContent += item.value + "\r\n"; - } - else { - csvContent += item.value - } - } - }) - }); - saveCSVToLocalFile(csvContent, experiment.id + ".csv") - } - - function UploadCSV(e) { - const init_data_points = state.rows[state.rows.length - 1].dataPoints[0].value == "" ? state.rows.length - 1 : state.rows.length - const file = e.target.files[0]; - if (!file) { - return; - } - var reader = new FileReader(); - reader.onload = function (e) { - var contents = e.target.result; - var data = String(contents).split(/\r\n|\n/); - if (data[0] !== String(state.rows[0].dataPoints.map((item, index) => item.name))) { - alert("Headers of the CSV are not correct" + "\r\nExpected: " + String(state.rows[0].dataPoints.map((item, index) => item.name)) - + "\r\nBut got: " + data[0]) - return; - } - var dims = (data[0].match(/,/g) || []).length - for (let i = 1; i < data.length; i++) { - if ((data[i].match(/,/g) || []).length == dims) { - if (i > 1 || state.rows[state.rows.length - 1].dataPoints[0].value != "") { - addRow(buildEmptyRow()) - } - var data_array = String(data[i]).split(','); - for (let j = 0; j < data_array.length; j++) { - updateRow(init_data_points - 1 + i, edit, data_array[j], j) - } - } else { - alert("Wrong amount of variables in line " + i + "\r\nExpected: " + dims + "\r\nBut got: " + (data[i].match(/,/g) || []).length) - } - } - }; - reader.readAsText(file) - } - return ( @@ -209,28 +143,8 @@ export default function DataPoints(props: DataPointProps) { Data points - - + + { + const { state: {experiment: {id, dataPoints}} } = useExperiment() + return +} + +export default DownloadCSVButton diff --git a/components/upload-csv-button.tsx b/components/upload-csv-button.tsx new file mode 100644 index 00000000..41e5cd3b --- /dev/null +++ b/components/upload-csv-button.tsx @@ -0,0 +1,36 @@ +import { Button, Input } from '@material-ui/core' +import { useExperiment } from '../context/experiment-context'; +import { csvToDataPoints, dataPointsToCSV } from '../utility/converters'; + +const readFile = (file, dataHandler) => { + var result = "" + if (file) { + const reader = new FileReader() + reader.onload = e => dataHandler(e.target.result as string) + reader.readAsText(file) + } + return result +} + +const UploadCSVButton = () => { + const { dispatch, state: { experiment: { valueVariables, categoricalVariables, dataPoints } } } = useExperiment() + const handleFileUpload = e => readFile(e.target.files[0], data => dispatch({ type: "updateDataPoints", payload: csvToDataPoints(data, valueVariables, categoricalVariables) })) + + return +} + +export default UploadCSVButton diff --git a/utility/__snapshots__/converters.test.ts.snap b/utility/__snapshots__/converters.test.ts.snap new file mode 100644 index 00000000..658a38d2 --- /dev/null +++ b/utility/__snapshots__/converters.test.ts.snap @@ -0,0 +1,3 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`converters csvToDataPoints should fail if header is missing 1`] = `"Headers does not match Sukker,Peber,Hvedemel,Kunde,score !== Sukker,Hvedemel,Kunde,score"`; diff --git a/utility/converters.test.ts b/utility/converters.test.ts index c4ef0fcd..19f76d71 100644 --- a/utility/converters.test.ts +++ b/utility/converters.test.ts @@ -1,63 +1,242 @@ import { initialState } from "../store" -import { ExperimentType } from "../types/common" -import { calculateSpace, calculateData } from "./converters" +import { CategoricalVariableType, DataPointType, ExperimentType, ValueVariableType } from "../types/common" +import { calculateSpace, calculateData, dataPointsToCSV, csvToDataPoints } from "./converters" describe("converters", () => { - const sampleExperiment: ExperimentType = {...initialState.experiment, - id: "123", - info: {...initialState.experiment.info, - name: "Cookies", - description: "Bager haremus' peberkager" - }, - categoricalVariables: [ - {name: "Kunde",description:"",options:["Mus","Ræv"]} - ], - valueVariables: [ - {discrete: true, name: "Sukker", description: "", minVal: 0, maxVal: 1000}, - {discrete: true, name: "Peber", description: "", minVal: 0, maxVal: 1000}, - {discrete: false, name: "Hvedemel", description: "", minVal: 0.0, maxVal: 1000.8}, - {discrete: true, name: "Mælk", description: "", minVal: 1, maxVal: 999}, - ], - optimizerConfig: { - baseEstimator: "GP", - acqFunc:"gp_hedge", - initialPoints: 2, - kappa: 1.96, - xi: 0.012 - }, - dataPoints: [ - [{name: "Sukker", value: 23}, {name: "Peber", value: 982}, {name: "Hvedemel", value: 632}, {name: "Kunde", value: "Mus"}, {name: "score", value: [0.1]}], - [{name: "Sukker", value: 15}, {name: "Peber", value: 123}, {name: "Hvedemel", value: 324}, {name: "Kunde", value: "Ræv"}, {name: "score", value: [0.2]}] + const sampleExperiment: ExperimentType = { + ...initialState.experiment, + id: "123", + info: { + ...initialState.experiment.info, + name: "Cookies", + description: "Bager haremus' peberkager" + }, + categoricalVariables: [ + { name: "Kunde", description: "", options: ["Mus", "Ræv"] } + ], + valueVariables: [ + { discrete: true, name: "Sukker", description: "", minVal: 0, maxVal: 1000 }, + { discrete: true, name: "Peber", description: "", minVal: 0, maxVal: 1000 }, + { discrete: false, name: "Hvedemel", description: "", minVal: 0.0, maxVal: 1000.8 }, + { discrete: true, name: "Mælk", description: "", minVal: 1, maxVal: 999 }, + ], + optimizerConfig: { + baseEstimator: "GP", + acqFunc: "gp_hedge", + initialPoints: 2, + kappa: 1.96, + xi: 0.012 + }, + dataPoints: [ + [{ name: "Sukker", value: 23 }, { name: "Peber", value: 982 }, { name: "Hvedemel", value: 632 }, { name: "Kunde", value: "Mus" }, { name: "score", value: [0.1] }], + [{ name: "Sukker", value: 15 }, { name: "Peber", value: 123 }, { name: "Hvedemel", value: 324 }, { name: "Kunde", value: "Ræv" }, { name: "score", value: [0.2] }] + ] + } + + describe("calculateSpace", () => { + it("should convert space to proper output format", () => { + const space = calculateSpace(sampleExperiment) + expect(space).toContainEqual({ type: "discrete", from: 0, name: "Sukker", to: 1000 }) + expect(space).toContainEqual({ type: "continuous", from: 0, name: "Hvedemel", to: 1000.8 }) + }) + }) + + describe("calculateSpace", () => { + it("should retain the correct order of variables", () => { + const space = calculateSpace(sampleExperiment) + expect(space[0].name).toEqual("Sukker") + expect(space[1].name).toEqual("Peber") + expect(space[2].name).toEqual("Hvedemel") + expect(space[3].name).toEqual("Mælk") + expect(space[4].name).toEqual("Kunde") + }) + }) + + describe("calculateData", () => { + it("should format data in proper output format", () => { + const expectedData = [ + { xi: [23, 982, 632, "Mus"], yi: 0.1 }, + { xi: [15, 123, 324, "Ræv"], yi: 0.2 } + ] + const actualData = calculateData(sampleExperiment.categoricalVariables, sampleExperiment.valueVariables, sampleExperiment.dataPoints) + expect(actualData).toEqual(expectedData) + }) + }) + + describe("dataPointsToCSV", () => { + it("should accept empty data set", () => { + const input = [[]] + const expected = "" + const actual = dataPointsToCSV(input) + expect(actual).toEqual(expected) + }) + + it("should convert known value", () => { + const input = [[ + { + "name": "Sukker", + "value": 28 + }, + { + "name": "Peber", + "value": 982 + }, + { + "name": "Hvedemel", + "value": 632 + }, + { + "name": "Kunde", + "value": "Mus" + }, + { + "name": "score", + "value": [ + 1 ] - } - - describe("calculateSpace", () => { - it("should convert space to proper output format", () => { - const space = calculateSpace(sampleExperiment) - expect(space).toContainEqual({type: "discrete", from: 0, name: "Sukker", to: 1000}) - expect(space).toContainEqual({type: "continuous", from: 0, name: "Hvedemel", to: 1000.8}) - }) - }) - - describe("calculateSpace", () => { - it("should retain the correct order of variables", () => { - const space = calculateSpace(sampleExperiment) - expect(space[0].name).toEqual("Sukker") - expect(space[1].name).toEqual("Peber") - expect(space[2].name).toEqual("Hvedemel") - expect(space[3].name).toEqual("Mælk") - expect(space[4].name).toEqual("Kunde") - }) - }) - - describe("calculateData", () => { - it("should format data in proper output format", () => { - const expectedData = [ - {xi: [23,982,632,"Mus"], yi: 0.1}, - {xi: [15,123,324,"Ræv"], yi: 0.2} - ] - const actualData = calculateData(sampleExperiment.categoricalVariables, sampleExperiment.valueVariables, sampleExperiment.dataPoints) - expect(actualData).toEqual(expectedData) - }) + } + ], + [ + { + "name": "Sukker", + "value": "15" + }, + { + "name": "Peber", + "value": "986" + }, + { + "name": "Hvedemel", + "value": "5" + }, + { + "name": "Kunde", + "value": "Mus" + }, + { + "name": "score", + "value": "2" + } + ]] + const expected = "Sukker;Peber;Hvedemel;Kunde;score\n28;982;632;Mus;1\n15;986;5;Mus;2" + const actual = dataPointsToCSV(input) + expect(actual).toEqual(expected) + }) + }) + + describe("csvToDataPoints", () => { + const categorialVariables: CategoricalVariableType[] = [ + { + "name": "Kunde", + "description": "", + "options": [ + "Mus", + "Ræv" + ] + } + ] + const valueVariables: ValueVariableType[] = [ + { + "name": "Sukker", + "description": "", + "minVal": 0, + "maxVal": 1000, + discrete: false + }, + { + "name": "Peber", + "description": "", + "minVal": 0, + "maxVal": 1000, + discrete: false + }, + { + "name": "Hvedemel", + "description": "", + "minVal": 0, + "maxVal": 1000, + discrete: false + } + ] + + const sampleDataPoints = [[ + { + "name": "Sukker", + "value": 28 + }, + { + "name": "Peber", + "value": 982 + }, + { + "name": "Hvedemel", + "value": 632 + }, + { + "name": "Kunde", + "value": "Mus" + }, + { + "name": "score", + "value": 1 + } + ], + [ + { + "name": "Sukker", + "value": 15 + }, + { + "name": "Peber", + "value": 986 + }, + { + "name": "Hvedemel", + "value": 5 + }, + { + "name": "Kunde", + "value": "Mus" + }, + { + "name": "score", + "value": 2 + } + ]] + + it("should accept empty data string", () => { + const input = "" + const expected = [[]] + const actual = csvToDataPoints(input, [], []) + expect(actual).toEqual(expected) + }) + + it("should convert known value", () => { + const input = "Sukker;Peber;Hvedemel;Kunde;score\n28;982;632;Mus;1\n15;986;5;Mus;2" + const expected = sampleDataPoints + const actual = csvToDataPoints(input, valueVariables, categorialVariables) + expect(actual).toEqual(expected) }) + + it("should accept shuffled columns", () => { + const input = "Sukker;score;Hvedemel;Peber;Kunde\n28;1;632;982;Mus\n15;2;5;986;Mus" + const expected = sampleDataPoints + const actual = csvToDataPoints(input, valueVariables, categorialVariables) + expect(actual).toEqual(expected) + }) + + it("should fail if header is missing", () => { + const input = "Sukker;Hvedemel;Kunde;score\n28;632;Mus;1\n15;5;Mus;2" + expect(() => csvToDataPoints(input, valueVariables, categorialVariables)) + .toThrowErrorMatchingSnapshot() + }) + + it("should not fail if there are extra headers", () => { + const input = "Sukker;Peber;Hvedemel;Halm;Kunde;score\n28;982;632;007;Mus;1\n15;986;5;008;Mus;2" + const expected = sampleDataPoints + const actual = csvToDataPoints(input, valueVariables, categorialVariables) + expect(actual).toEqual(expected) + }) + + }) }) \ No newline at end of file diff --git a/utility/converters.ts b/utility/converters.ts index 11721c38..d8e69c63 100644 --- a/utility/converters.ts +++ b/utility/converters.ts @@ -1,5 +1,5 @@ import { ExperimentData } from "../openapi" -import { CategoricalVariableType, DataPointType, ExperimentType, ScoreDataPointType, SpaceType, ValueVariableType } from "../types/common" +import { CategoricalVariableType, DataPointType, DataPointTypeValue, ExperimentType, ScoreDataPointType, SpaceType, ValueVariableType } from "../types/common" export const calculateSpace = (experiment: ExperimentType): SpaceType => { const numerical: SpaceType = experiment.valueVariables.map(v => { const type = v.discrete ? "discrete" : "continuous" @@ -11,4 +11,45 @@ export const calculateSpace = (experiment: ExperimentType): SpaceType => { const numPat = / [0-9] + / export const calculateData = (categoricalValues: CategoricalVariableType[], numericValues: ValueVariableType[], dataPoints: DataPointType[][]): ExperimentData[] => { return dataPoints.map((run):ExperimentData => ({xi: run.filter(it => it.name !== "score").map(it => numericValues.find(p => p.name === it.name) ? Number(it.value) : it.value), yi: Number((run.filter(it => it.name === "score")[0] as ScoreDataPointType).value[0])})) -} \ No newline at end of file +} + +export const dataPointsToCSV = (dataPoints: DataPointType[][], separator=";", newline="\n"): string => dataPoints +.reduce((prev, curr, idx) => idx === 0 ? [curr.map(item => item.name).join(separator)] : prev, [] as string[]) +.concat(dataPoints + .map(line => line + .map(item => item.value) + .join(separator))) + .filter(s => "" !== s) + .join(newline) + +const convertValue = (valueHeaders:string[], categorialHeaders:string[], name: string, value: any): DataPointTypeValue => { + if (valueHeaders.includes(name)) { + return Number(value) + } else if (categorialHeaders.includes(name)) { + return value as string + } else { + return Array.isArray(value) ? value : Number(value) + } +} + +export const csvToDataPoints = (csv: string, valueVariables:ValueVariableType[], categorialVariables:CategoricalVariableType[], separator=";", newlinePattern=/\r\n|\n/):DataPointType[][] => { + const valueHeaders = valueVariables.map(x => x.name) + const categorialHeaders = categorialVariables.map(x => x.name) + const expectedHeader = valueHeaders.concat(categorialHeaders).concat(['score']) + const lines = csv.split(newlinePattern) + if ("" === csv || lines.length < 2) return [[]] + else { + const header = lines[0].split(separator) + if (header.length >= expectedHeader.length && expectedHeader.every((value, _) => header.includes(value))) { + const data = lines.slice(1) + return data + .map(line => line + .split(separator) + .map((value, idx) => ({name: header[idx], value: convertValue(valueHeaders, categorialHeaders, header[idx], value)} as DataPointType))) + .map(data => data.sort((a, b) => expectedHeader.indexOf(a.name) - expectedHeader.indexOf(b.name))) + .map(data => data.filter(x => expectedHeader.includes(x.name))) + } else { + throw new Error(`Headers does not match ${expectedHeader.join(",")} !== ${header.join(",")}`) + } + } +} From 13c2806cf3faffb6f8060aad8c347d5db1c849da Mon Sep 17 00:00:00 2001 From: Jack Ord <36296721+j-or@users.noreply.github.com> Date: Thu, 9 Sep 2021 14:42:08 +0200 Subject: [PATCH 11/13] Update table on csv upload --- components/data-points.tsx | 38 ++++++++++++++++++-------------- components/upload-csv-button.tsx | 14 +++++++----- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/components/data-points.tsx b/components/data-points.tsx index f3a18c4c..f3086c31 100644 --- a/components/data-points.tsx +++ b/components/data-points.tsx @@ -1,4 +1,4 @@ -import { CircularProgress, IconButton, Button, Grid, Input } from "@material-ui/core"; +import { CircularProgress, IconButton, Grid } from "@material-ui/core"; import { useEffect, useReducer } from "react"; import { useGlobal } from "../context/global-context"; import { dataPointsReducer, DataPointsState } from "../reducers/data-points-reducer"; @@ -30,10 +30,14 @@ export default function DataPoints(props: DataPointProps) { const newestFirst = global.state.dataPointsNewestFirst useEffect(() => { - dispatch({ type: 'setInitialState', payload: buildState() }) + dispatch({ type: 'setInitialState', payload: buildState(dataPoints) }) }, [valueVariables, categoricalVariables]) - const buildState = (): DataPointsState => { + useEffect(() => { + updateDataPoints(state.rows.filter(item => !item.isNew) as TableDataRow[]) + }, [state.rows]) + + const buildState = (dataPoints: DataPointType[][]): DataPointsState => { const combinedVariables: CombinedVariableType[] = buildCombinedVariables() const emptyRow: TableDataRow = buildEmptyRow() const dataPointRows: TableDataRow[] = dataPoints.map((item, i) => { @@ -78,19 +82,15 @@ export default function DataPoints(props: DataPointProps) { } } - useEffect(() => { - updateDataPoints(state.rows.filter(item => !item.isNew) as TableDataRow[]) - }, [state.rows]) - - function toggleEditMode(rowIndex: number) { + const toggleEditMode = (rowIndex: number) => { dispatch({ type: 'DATA_POINTS_TABLE_EDIT_TOGGLED', payload: rowIndex }) } - function cancelEdit(rowIndex: number) { + const cancelEdit = (rowIndex: number) => { dispatch({ type: 'DATA_POINTS_TABLE_EDIT_CANCELLED', payload: rowIndex }) } - function edit(rowIndex: number, editValue: string, itemIndex: number) { + const edit = (rowIndex: number, editValue: string, itemIndex: number) => { dispatch({ type: 'DATA_POINTS_TABLE_EDITED', payload: { itemIndex, @@ -101,17 +101,17 @@ export default function DataPoints(props: DataPointProps) { }) } - function deleteRow(rowIndex: number) { + const deleteRow = (rowIndex: number) => { dispatch({ type: 'DATA_POINTS_TABLE_ROW_DELETED', payload: rowIndex }) } - function addRow(emptyRow: TableDataRow) { + const addRow = (emptyRow: TableDataRow) => { dispatch({ type: 'DATA_POINTS_TABLE_ROW_ADDED', payload: emptyRow }) } - function updateDataPoints(dataRows: TableDataRow[]) { + const updateDataPoints = (dataRows: TableDataRow[]) => { onUpdateDataPoints(dataRows - .map((item, i) => { + .map(item => { return item.dataPoints.map(item => { return { name: item.name, @@ -122,7 +122,7 @@ export default function DataPoints(props: DataPointProps) { ) } - function onEditConfirm(row: TableDataRow, rowIndex: number) { + const onEditConfirm = (row: TableDataRow, rowIndex: number) => { if (row.isNew) { addRow(buildEmptyRow()) } else { @@ -130,11 +130,15 @@ export default function DataPoints(props: DataPointProps) { } } - function updateRow(index: number, updateFn: UpdateFnType, ...args: any[]) { + const updateRow = (index: number, updateFn: UpdateFnType, ...args: any[]) => { const rowIndex = newestFirst ? state.rows.length - 1 - index : index updateFn(rowIndex, ...args) } + const updateTable = (dataPoints: DataPointType[][]) => { + dispatch({ type: 'setInitialState', payload: buildState(dataPoints) }) + } + return ( @@ -144,7 +148,7 @@ export default function DataPoints(props: DataPointProps) { - + updateTable(dataPoints)} /> { var result = "" @@ -11,11 +12,14 @@ const readFile = (file, dataHandler) => { } return result } +interface UploadCSVButtonProps { + onUpload: (dataPoints: DataPointType[][]) => void +} -const UploadCSVButton = () => { - const { dispatch, state: { experiment: { valueVariables, categoricalVariables, dataPoints } } } = useExperiment() - const handleFileUpload = e => readFile(e.target.files[0], data => dispatch({ type: "updateDataPoints", payload: csvToDataPoints(data, valueVariables, categoricalVariables) })) - +const UploadCSVButton = ({ onUpload } : UploadCSVButtonProps) => { + const { state: { experiment: { valueVariables, categoricalVariables } } } = useExperiment() + const handleFileUpload = e => readFile(e.target.files[0], data => onUpload(csvToDataPoints(data, valueVariables, categoricalVariables))) + return + return + saveCSVToLocalFile(dataPointsToCSV(dataPoints), id + ".csv")} + > + + + + } export default DownloadCSVButton diff --git a/components/upload-csv-button.tsx b/components/upload-csv-button.tsx index 85b9c37a..3765cdfd 100644 --- a/components/upload-csv-button.tsx +++ b/components/upload-csv-button.tsx @@ -1,7 +1,8 @@ -import { Button, Input } from '@material-ui/core' +import { IconButton, Input, Tooltip } from '@material-ui/core' import { useExperiment } from '../context/experiment-context'; import { DataPointType } from '../types/common'; import { csvToDataPoints } from '../utility/converters'; +import PublishIcon from '@material-ui/icons/Publish'; const readFile = (file, dataHandler) => { var result = "" @@ -13,28 +14,32 @@ const readFile = (file, dataHandler) => { return result } interface UploadCSVButtonProps { + light?: boolean onUpload: (dataPoints: DataPointType[][]) => void } -const UploadCSVButton = ({ onUpload } : UploadCSVButtonProps) => { +const UploadCSVButton = ({ onUpload, light } : UploadCSVButtonProps) => { const { state: { experiment: { valueVariables, categoricalVariables } } } = useExperiment() const handleFileUpload = e => readFile(e.target.files[0], data => onUpload(csvToDataPoints(data, valueVariables, categoricalVariables))) - - return + + return + + + + + } export default UploadCSVButton diff --git a/styles/data-points.style.tsx b/styles/data-points.style.tsx index f80a7ebc..2b30f465 100644 --- a/styles/data-points.style.tsx +++ b/styles/data-points.style.tsx @@ -1,10 +1,10 @@ import { makeStyles } from "@material-ui/core"; export const useStyles = makeStyles(theme => ({ - orderButton: { + titleButton: { float: 'right', }, - orderIcon: { + titleIcon: { color: 'white', }, })); From 2226b61e7796dbce0f569365d5aeb95e78f6864f Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Thu, 9 Sep 2021 21:47:19 +0200 Subject: [PATCH 13/13] Make score array work on upload --- utility/converters.test.ts | 4 ++-- utility/converters.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utility/converters.test.ts b/utility/converters.test.ts index 19f76d71..fe62ae35 100644 --- a/utility/converters.test.ts +++ b/utility/converters.test.ts @@ -178,7 +178,7 @@ describe("converters", () => { }, { "name": "score", - "value": 1 + "value": [1] } ], [ @@ -200,7 +200,7 @@ describe("converters", () => { }, { "name": "score", - "value": 2 + "value": [2] } ]] diff --git a/utility/converters.ts b/utility/converters.ts index d8e69c63..feb68bac 100644 --- a/utility/converters.ts +++ b/utility/converters.ts @@ -28,7 +28,7 @@ const convertValue = (valueHeaders:string[], categorialHeaders:string[], name: s } else if (categorialHeaders.includes(name)) { return value as string } else { - return Array.isArray(value) ? value : Number(value) + return Array.isArray(value) ? value : [Number(value)] } }