From f2974bf2eec8179bafc159826798d1f4af847260 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sun, 17 May 2020 20:08:25 +0200 Subject: [PATCH 1/3] display version of WR and RiaH in the help menu --- pom.xml | 2 +- .../org/ohdsi/rabbitInAHat/RabbitInAHatMain.java | 14 ++++++++++---- .../org/ohdsi/whiteRabbit/WhiteRabbitMain.java | 9 +++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index d3b9d658..47c88ed6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.ohdsi leporidae pom - 0.9.0 + 0.10.0-SNAPSHOT rabbitinahat whiterabbit diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java index 30c65f5c..07f70f2c 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java @@ -57,6 +57,8 @@ */ public class RabbitInAHatMain implements ResizeListener { + public static String version = "0.10.0-SNAPSHOT"; + public final static String ACTION_SAVE = "Save"; public final static String ACTION_SAVE_AS = "Save As"; public final static String ACTION_OPEN_ETL_SPECS = "Open ETL Specs"; @@ -300,22 +302,26 @@ private JMenuBar createMenuBar() { JMenu helpMenu = new JMenu("Help"); menuBar.add(helpMenu); + addMenuItem(helpMenu, "Rabbit in a Hat v" + version, null).setEnabled(false); addMenuItem(helpMenu, ACTION_HELP, evt -> this.doOpenDocumentation()); return menuBar; } - public void addMenuItem(JMenu menu, String description, ActionListener actionListener) { - addMenuItem(menu, description, actionListener, null); + public JMenuItem addMenuItem(JMenu menu, String description, ActionListener actionListener) { + return addMenuItem(menu, description, actionListener, null); } - public void addMenuItem(JMenu menu, String description, ActionListener actionListener, Integer keyEvent) { + public JMenuItem addMenuItem(JMenu menu, String description, ActionListener actionListener, Integer keyEvent) { JMenuItem menuItem = new JMenuItem(description); - menuItem.addActionListener(actionListener); + if (actionListener != null) { + menuItem.addActionListener(actionListener); + } if (keyEvent != null) { menuItem.setAccelerator(KeyStroke.getKeyStroke(keyEvent, Toolkit.getDefaultToolkit().getMenuShortcutKeyMask())); } menu.add(menuItem); + return menuItem; } @Override diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index 7c5dd946..d99fa44b 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -82,6 +82,8 @@ */ public class WhiteRabbitMain implements ActionListener { + public static String version = "0.10.0-SNAPSHOT"; + public final static String DOCUMENTATION_URL = "http://ohdsi.github.io/WhiteRabbit"; public final static String ACTION_CMD_HELP = "Open documentation"; @@ -507,7 +509,6 @@ public void actionPerformed(ActionEvent e) { } private JPanel createFakeDataPanel() { - // TODO: add sas7bdat as target for fake data. JPanel panel = new JPanel(); panel.setLayout(new GridBagLayout()); @@ -541,7 +542,6 @@ public void actionPerformed(ActionEvent e) { targetPanel.setBorder(BorderFactory.createTitledBorder("Target data location")); targetPanel.add(new JLabel("Data type")); targetType = new JComboBox(new String[] { "Delimited text files", "MySQL", "Oracle", "SQL Server", "PostgreSQL" }); - // targetType = new JComboBox(new String[] { "Delimited text files", "MySQL" }); targetType.setToolTipText("Select the type of source data available"); targetType.addItemListener(new ItemListener() { @@ -1163,6 +1163,11 @@ private void handleError(Exception e) { private JMenuBar createMenuBar() { JMenuBar menuBar = new JMenuBar(); JMenu helpMenu = new JMenu("Help"); + + JMenuItem versionItem = new JMenuItem("White Rabbit v" + version); + versionItem.setEnabled(false); + helpMenu.add(versionItem); + menuBar.add(helpMenu); JMenuItem helpItem = new JMenuItem(ACTION_CMD_HELP); helpItem.addActionListener(this); From 7f112bd3c9295b7329889d1dd0b9822fb9cadb6b Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sun, 17 May 2020 20:47:11 +0200 Subject: [PATCH 2/3] read version from pom.xml --- pom.xml | 8 ++++++++ rabbit-core/pom.xml | 2 +- .../main/java/org/ohdsi/utilities/Version.java | 17 +++++++++++++++++ .../src/main/resources/project.properties | 2 ++ rabbitinahat/pom.xml | 2 +- .../ohdsi/rabbitInAHat/RabbitInAHatMain.java | 7 +++---- whiterabbit/pom.xml | 2 +- .../org/ohdsi/whiteRabbit/WhiteRabbitMain.java | 5 ++--- 8 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 rabbit-core/src/main/java/org/ohdsi/utilities/Version.java create mode 100644 rabbit-core/src/main/resources/project.properties diff --git a/pom.xml b/pom.xml index 47c88ed6..aa25d2c1 100644 --- a/pom.xml +++ b/pom.xml @@ -129,5 +129,13 @@ + + + + + src/main/resources + true + + diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml index c8735e02..ede2ba08 100644 --- a/rabbit-core/pom.xml +++ b/rabbit-core/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.9.0 + 0.10.0-SNAPSHOT 4.0.0 diff --git a/rabbit-core/src/main/java/org/ohdsi/utilities/Version.java b/rabbit-core/src/main/java/org/ohdsi/utilities/Version.java new file mode 100644 index 00000000..79f4be67 --- /dev/null +++ b/rabbit-core/src/main/java/org/ohdsi/utilities/Version.java @@ -0,0 +1,17 @@ +package org.ohdsi.utilities; + +import java.io.IOException; +import java.util.Properties; + +public class Version { + + public static String getVersion(Class c) { + final Properties properties = new Properties(); + try { + properties.load(c.getClassLoader().getResourceAsStream("project.properties")); + } catch (IOException e) { + e.printStackTrace(); + } + return properties.getProperty("version"); + } +} diff --git a/rabbit-core/src/main/resources/project.properties b/rabbit-core/src/main/resources/project.properties new file mode 100644 index 00000000..4155ab2b --- /dev/null +++ b/rabbit-core/src/main/resources/project.properties @@ -0,0 +1,2 @@ +version=${project.version} +artifactId=${project.artifactId} \ No newline at end of file diff --git a/rabbitinahat/pom.xml b/rabbitinahat/pom.xml index c7820d9e..cafdb05f 100644 --- a/rabbitinahat/pom.xml +++ b/rabbitinahat/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.9.0 + 0.10.0-SNAPSHOT 4.0.0 diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java index 07f70f2c..f39d321d 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java @@ -50,6 +50,7 @@ import org.ohdsi.rabbitInAHat.dataModel.MappableItem; import org.ohdsi.rabbitInAHat.dataModel.StemTableFactory; import org.ohdsi.rabbitInAHat.dataModel.Table; +import org.ohdsi.utilities.Version; import org.ohdsi.utilities.collections.Pair; /** @@ -57,8 +58,6 @@ */ public class RabbitInAHatMain implements ResizeListener { - public static String version = "0.10.0-SNAPSHOT"; - public final static String ACTION_SAVE = "Save"; public final static String ACTION_SAVE_AS = "Save As"; public final static String ACTION_OPEN_ETL_SPECS = "Open ETL Specs"; @@ -83,7 +82,7 @@ public class RabbitInAHatMain implements ResizeListener { public final static String ACTION_SET_TARGET_V530 = "CDM v5.3.0"; public final static String ACTION_SET_TARGET_V531 = "CDM v5.3.1"; public final static String ACTION_SET_TARGET_V60 = "CDM v6.0"; - public final static String ACTION_SET_TARGET_V60_O = "CDM v6.0 + Oncology"; + public final static String ACTION_SET_TARGET_V60_O = "CDM v6.0 + Oncology"; public final static String ACTION_ADD_STEM_TABLE = "Add stem table"; public final static String ACTION_REMOVE_STEM_TABLE = "Remove stem table"; public final static String ACTION_SET_TARGET_CUSTOM = "Load Custom..."; @@ -302,7 +301,7 @@ private JMenuBar createMenuBar() { JMenu helpMenu = new JMenu("Help"); menuBar.add(helpMenu); - addMenuItem(helpMenu, "Rabbit in a Hat v" + version, null).setEnabled(false); + addMenuItem(helpMenu, "Rabbit in a Hat v" + Version.getVersion(this.getClass()), null).setEnabled(false); addMenuItem(helpMenu, ACTION_HELP, evt -> this.doOpenDocumentation()); return menuBar; diff --git a/whiterabbit/pom.xml b/whiterabbit/pom.xml index 957092fa..335ae2a2 100644 --- a/whiterabbit/pom.xml +++ b/whiterabbit/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.9.0 + 0.10.0-SNAPSHOT 4.0.0 diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index d99fa44b..21685456 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -73,6 +73,7 @@ import org.ohdsi.databases.RichConnection; import org.ohdsi.utilities.DirectoryUtilities; import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.Version; import org.ohdsi.utilities.files.IniFile; import org.ohdsi.whiteRabbit.fakeDataGenerator.FakeDataGenerator; import org.ohdsi.whiteRabbit.scan.SourceDataScan; @@ -82,8 +83,6 @@ */ public class WhiteRabbitMain implements ActionListener { - public static String version = "0.10.0-SNAPSHOT"; - public final static String DOCUMENTATION_URL = "http://ohdsi.github.io/WhiteRabbit"; public final static String ACTION_CMD_HELP = "Open documentation"; @@ -1164,7 +1163,7 @@ private JMenuBar createMenuBar() { JMenuBar menuBar = new JMenuBar(); JMenu helpMenu = new JMenu("Help"); - JMenuItem versionItem = new JMenuItem("White Rabbit v" + version); + JMenuItem versionItem = new JMenuItem("White Rabbit v" + Version.getVersion(this.getClass())); versionItem.setEnabled(false); helpMenu.add(versionItem); From 22d47e93c13b612462af9a1b7a0075b99bb257c4 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sun, 17 May 2020 22:39:01 +0200 Subject: [PATCH 3/3] write version and settings to scan report sheet --- .../org/ohdsi/whiteRabbit/DbSettings.java | 42 +++++----- .../ohdsi/whiteRabbit/WhiteRabbitMain.java | 30 ++++---- .../fakeDataGenerator/FakeDataGenerator.java | 6 +- .../whiteRabbit/scan/SourceDataScan.java | 76 +++++++++++++------ .../scan/UniformSamplingReservoir.java | 1 - 5 files changed, 91 insertions(+), 64 deletions(-) diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java index aca2d7c2..72637902 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/DbSettings.java @@ -1,14 +1,14 @@ /******************************************************************************* * Copyright 2019 Observational Health Data Sciences and Informatics - * + * * This file is part of WhiteRabbit - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,22 +24,22 @@ import org.ohdsi.databases.DbType; public class DbSettings { - public static int DATABASE = 1; - public static int CSVFILES = 2; - public static int SASFILES = 3; + public enum SourceType { + DATABASE, CSV_FILES, SAS_FILES + } - public int dataType; - public List tables = new ArrayList(); - - // Database settings - public DbType dbType; - public String user; - public String password; - public String database; - public String server; - public String domain; - - // CSV file settings - public char delimiter = ','; - public CSVFormat csvFormat = CSVFormat.RFC4180; + public SourceType sourceType; + public List tables = new ArrayList<>(); + + // Database settings + public DbType dbType; + public String user; + public String password; + public String database; + public String server; + public String domain; + + // CSV file settings + public char delimiter = ','; + public CSVFormat csvFormat = CSVFormat.RFC4180; } diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java index 21685456..7e07e7fd 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/WhiteRabbitMain.java @@ -153,15 +153,15 @@ private void launchCommandLine(String iniFileName) { IniFile iniFile = new IniFile(iniFileName); DbSettings dbSettings = new DbSettings(); if (iniFile.get("DATA_TYPE").equalsIgnoreCase("Delimited text files")) { - dbSettings.dataType = DbSettings.CSVFILES; + dbSettings.sourceType = DbSettings.SourceType.CSV_FILES; if (iniFile.get("DELIMITER").equalsIgnoreCase("tab")) dbSettings.delimiter = '\t'; else dbSettings.delimiter = iniFile.get("DELIMITER").charAt(0); } else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("SAS7bdat")) { - dbSettings.dataType = DbSettings.SASFILES; + dbSettings.sourceType = DbSettings.SourceType.SAS_FILES; } else { - dbSettings.dataType = DbSettings.DATABASE; + dbSettings.sourceType = DbSettings.SourceType.DATABASE; dbSettings.user = iniFile.get("USER_NAME"); dbSettings.password = iniFile.get("PASSWORD"); dbSettings.server = iniFile.get("SERVER_LOCATION"); @@ -217,7 +217,7 @@ else if (iniFile.get("DATA_TYPE").equalsIgnoreCase("BigQuery")) { } } else { for (String table : iniFile.get("TABLES_TO_SCAN").split(",")) { - if (dbSettings.dataType == DbSettings.CSVFILES) + if (dbSettings.sourceType == DbSettings.SourceType.CSV_FILES) table = iniFile.get("WORKING_FOLDER") + "/" + table; dbSettings.tables.add(table); } @@ -732,14 +732,14 @@ private void addAllTables() { private void pickTables() { DbSettings sourceDbSettings = getSourceDbSettings(); if (sourceDbSettings != null) { - if (sourceDbSettings.dataType == DbSettings.CSVFILES || sourceDbSettings.dataType == DbSettings.SASFILES) { + if (sourceDbSettings.sourceType == DbSettings.SourceType.CSV_FILES || sourceDbSettings.sourceType == DbSettings.SourceType.SAS_FILES) { JFileChooser fileChooser = new JFileChooser(new File(folderField.getText())); fileChooser.setMultiSelectionEnabled(true); fileChooser.setFileSelectionMode(JFileChooser.FILES_ONLY); - if (sourceDbSettings.dataType == DbSettings.CSVFILES) { + if (sourceDbSettings.sourceType == DbSettings.SourceType.CSV_FILES) { fileChooser.setFileFilter(new FileNameExtensionFilter("Delimited text files", "csv", "txt")); - } else if (sourceDbSettings.dataType == DbSettings.SASFILES) { + } else if (sourceDbSettings.sourceType == DbSettings.SourceType.SAS_FILES) { fileChooser.setFileFilter(new FileNameExtensionFilter("SAS Data Files", "sas7bdat")); } @@ -753,7 +753,7 @@ private void pickTables() { } } - } else if (sourceDbSettings.dataType == DbSettings.DATABASE) { + } else if (sourceDbSettings.sourceType == DbSettings.SourceType.DATABASE) { RichConnection connection = new RichConnection(sourceDbSettings.server, sourceDbSettings.domain, sourceDbSettings.user, sourceDbSettings.password, sourceDbSettings.dbType); String tableNames = StringUtilities.join(connection.getTableNames(sourceDbSettings.database), "\t"); @@ -778,7 +778,7 @@ private void pickTables() { private DbSettings getSourceDbSettings() { DbSettings dbSettings = new DbSettings(); if (sourceType.getSelectedItem().equals("Delimited text files")) { - dbSettings.dataType = DbSettings.CSVFILES; + dbSettings.sourceType = DbSettings.SourceType.CSV_FILES; if (sourceDelimiterField.getText().length() == 0) { JOptionPane.showMessageDialog(frame, "Delimiter field cannot be empty for source database", "Error connecting to server", JOptionPane.ERROR_MESSAGE); @@ -789,9 +789,9 @@ private DbSettings getSourceDbSettings() { else dbSettings.delimiter = sourceDelimiterField.getText().charAt(0); } else if (sourceType.getSelectedItem().equals("SAS7bdat")) { - dbSettings.dataType = DbSettings.SASFILES; + dbSettings.sourceType = DbSettings.SourceType.SAS_FILES; } else { - dbSettings.dataType = DbSettings.DATABASE; + dbSettings.sourceType = DbSettings.SourceType.DATABASE; dbSettings.user = sourceUserField.getText(); dbSettings.password = sourcePasswordField.getText(); dbSettings.server = sourceServerField.getText(); @@ -837,7 +837,7 @@ else if (sourceType.getSelectedItem().toString().equals("Azure")) { } private void testConnection(DbSettings dbSettings) { - if (dbSettings.dataType == DbSettings.CSVFILES || dbSettings.dataType == DbSettings.SASFILES) { + if (dbSettings.sourceType == DbSettings.SourceType.CSV_FILES || dbSettings.sourceType == DbSettings.SourceType.SAS_FILES) { if (new File(folderField.getText()).exists()) { String message = "Folder " + folderField.getText() + " found"; JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Working folder found", JOptionPane.INFORMATION_MESSAGE); @@ -885,7 +885,7 @@ private void testConnection(DbSettings dbSettings) { private DbSettings getTargetDbSettings() { DbSettings dbSettings = new DbSettings(); if (targetType.getSelectedItem().equals("Delimited text files")) { - dbSettings.dataType = DbSettings.CSVFILES; + dbSettings.sourceType = DbSettings.SourceType.CSV_FILES; switch((String) targetCSVFormat.getSelectedItem()) { case "Default (comma, CRLF)": @@ -908,7 +908,7 @@ private DbSettings getTargetDbSettings() { } } else { - dbSettings.dataType = DbSettings.DATABASE; + dbSettings.sourceType = DbSettings.SourceType.DATABASE; dbSettings.user = targetUserField.getText(); dbSettings.password = targetPasswordField.getText(); dbSettings.server = targetServerField.getText(); @@ -1032,7 +1032,7 @@ public void run() { DbSettings dbSettings = getSourceDbSettings(); if (dbSettings != null) { for (String table : tables) { - if (dbSettings.dataType == DbSettings.CSVFILES || dbSettings.dataType == DbSettings.SASFILES) + if (dbSettings.sourceType == DbSettings.SourceType.CSV_FILES || dbSettings.sourceType == DbSettings.SourceType.SAS_FILES) table = folderField.getText() + "/" + table; dbSettings.tables.add(table); } diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java index e0d353c7..7b953fdf 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/fakeDataGenerator/FakeDataGenerator.java @@ -34,7 +34,7 @@ public class FakeDataGenerator { private RichConnection connection; - private int targetType; + private DbSettings.SourceType targetType; private int maxRowsPerTable = 1000; private static int REGULAR = 0; @@ -43,13 +43,13 @@ public class FakeDataGenerator { public void generateData(DbSettings dbSettings, int maxRowsPerTable, String filename, String folder) { this.maxRowsPerTable = maxRowsPerTable; - this.targetType = dbSettings.dataType; + this.targetType = dbSettings.sourceType; StringUtilities.outputWithTime("Starting creation of fake data"); System.out.println("Loading scan report from " + filename); Database database = Database.generateModelFromScanReport(filename); - if (targetType == DbSettings.DATABASE) { + if (targetType == DbSettings.SourceType.DATABASE) { connection = new RichConnection(dbSettings.server, dbSettings.domain, dbSettings.user, dbSettings.password, dbSettings.dbType); connection.use(dbSettings.database); for (Table table : database.getTables()) { diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java index 03c3d48d..9c4a6595 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/SourceDataScan.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; @@ -37,10 +38,7 @@ import org.ohdsi.databases.RichConnection; import org.ohdsi.databases.RichConnection.QueryResult; import org.ohdsi.rabbitInAHat.dataModel.Table; -import org.ohdsi.utilities.DateUtilities; -import org.ohdsi.utilities.ScanFieldName; -import org.ohdsi.utilities.ScanSheetName; -import org.ohdsi.utilities.StringUtilities; +import org.ohdsi.utilities.*; import org.ohdsi.utilities.collections.CountingSet; import org.ohdsi.utilities.collections.CountingSet.Count; import org.ohdsi.utilities.collections.Pair; @@ -56,18 +54,22 @@ public class SourceDataScan { public static int N_FOR_FREE_TEXT_CHECK = 1000; public static int MIN_AVERAGE_LENGTH_FOR_FREE_TEXT = 100; - private char delimiter = ','; - private int sampleSize; + private SXSSFWorkbook workbook; + private char delimiter = ','; + private int sampleSize; private boolean scanValues = false; private boolean calculateNumericStats = false; - private int numStatsSamplerSize; - private int minCellCount; - private int maxValues; - private DbType dbType; - private String database; + private int numStatsSamplerSize; + private int minCellCount; + private int maxValues; + private DbSettings.SourceType sourceType; + private DbType dbType; + private String database; private Map> tableToFieldInfos; private Map indexedTableNameLookup; + private LocalDateTime startTimeStamp; + public void setSampleSize(int sampleSize) { this.sampleSize = sampleSize; @@ -94,12 +96,17 @@ public void setNumStatsSamplerSize(int numStatsSamplerSize) { } public void process(DbSettings dbSettings, String outputFileName) { + startTimeStamp = LocalDateTime.now(); + sourceType = dbSettings.sourceType; + dbType = dbSettings.dbType; + database = dbSettings.database; + tableToFieldInfos = new HashMap<>(); - if (dbSettings.dataType == DbSettings.CSVFILES) { + if (sourceType == DbSettings.SourceType.CSV_FILES) { if (!scanValues) this.minCellCount = Math.max(minCellCount, MIN_CELL_COUNT_FOR_CSV); processCsvFiles(dbSettings); - } else if (dbSettings.dataType == DbSettings.SASFILES) { + } else if (sourceType == DbSettings.SourceType.SAS_FILES) { processSasFiles(dbSettings); } else { processDatabase(dbSettings); @@ -118,9 +125,6 @@ private void processDatabase(DbSettings dbSettings) { connection.setVerbose(false); connection.use(dbSettings.database); - dbType = dbSettings.dbType; - database = dbSettings.database; - tableToFieldInfos = dbSettings.tables.stream() .collect(Collectors.toMap( Table::new, @@ -162,7 +166,7 @@ private void generateReport(String filename) { System.out.println("Generating scan report"); removeEmptyTables(); - SXSSFWorkbook workbook = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk + workbook = new SXSSFWorkbook(100); // keep 100 rows in memory, exceeding rows will be flushed to disk int i = 0; indexedTableNameLookup = new HashMap<>(); @@ -172,14 +176,15 @@ private void generateReport(String filename) { i++; } - createFieldOverviewSheet(workbook); - - createTableOverviewSheet(workbook); + createFieldOverviewSheet(); + createTableOverviewSheet(); if (scanValues) { - createValueSheet(workbook); + createValueSheet(); } + createMetaSheet(); + try (FileOutputStream out = new FileOutputStream(new File(filename))) { workbook.write(out); out.close(); @@ -189,7 +194,7 @@ private void generateReport(String filename) { } } - private void createFieldOverviewSheet(SXSSFWorkbook workbook) { + private void createFieldOverviewSheet() { Sheet overviewSheet = workbook.createSheet(ScanSheetName.FIELD_OVERVIEW); CellStyle percentageStyle = workbook.createCellStyle(); percentageStyle.setDataFormat(workbook.createDataFormat().getFormat("0.0%")); @@ -269,7 +274,7 @@ private void createFieldOverviewSheet(SXSSFWorkbook workbook) { } } - private void createTableOverviewSheet(SXSSFWorkbook workbook) { + private void createTableOverviewSheet() { Sheet tableOverviewSheet = workbook.createSheet(ScanSheetName.TABLE_OVERVIEW); addRow(tableOverviewSheet, @@ -308,7 +313,7 @@ private void createTableOverviewSheet(SXSSFWorkbook workbook) { } } - private void createValueSheet(SXSSFWorkbook workbook) { + private void createValueSheet() { // Make a copy of the tableNames, such that we can dereference the table at the end of each loop to save memory Table[] tables = tableToFieldInfos.keySet().toArray(new Table[0]); @@ -353,6 +358,29 @@ private void createValueSheet(SXSSFWorkbook workbook) { } } + private void createMetaSheet() { + // All variables to be stored + Sheet metaSheet = workbook.createSheet("_"); + addRow(metaSheet, "Key", "Value"); + addRow(metaSheet, "Version", Version.getVersion(this.getClass())); + addRow(metaSheet, "Scan started at ", startTimeStamp.toString()); + addRow(metaSheet, "Scan finished at", LocalDateTime.now().toString()); + addRow(metaSheet, "MAX_VALUES_IN_MEMORY", SourceDataScan.MAX_VALUES_IN_MEMORY); + addRow(metaSheet, "MIN_CELL_COUNT_FOR_CSV", SourceDataScan.MIN_CELL_COUNT_FOR_CSV); + addRow(metaSheet, "N_FOR_FREE_TEXT_CHECK", SourceDataScan.N_FOR_FREE_TEXT_CHECK); + addRow(metaSheet, "MIN_AVERAGE_LENGTH_FOR_FREE_TEXT", SourceDataScan.MIN_AVERAGE_LENGTH_FOR_FREE_TEXT); + addRow(metaSheet, "sourceType", this.sourceType.toString()); + addRow(metaSheet, "dbType", this.dbType.toString()); +// addRow(metaSheet, "database", this.database); + addRow(metaSheet, "delimiter", this.delimiter); + addRow(metaSheet, "sampleSize", this.sampleSize); + addRow(metaSheet, "scanValues", this.scanValues); + addRow(metaSheet, "minCellCount", this.minCellCount); + addRow(metaSheet, "maxValues", this.maxValues); + addRow(metaSheet, "calculateNumericStats", this.calculateNumericStats); + addRow(metaSheet, "numStatsSamplerSize", this.numStatsSamplerSize); + + } private void removeEmptyTables() { tableToFieldInfos.entrySet() diff --git a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/UniformSamplingReservoir.java b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/UniformSamplingReservoir.java index 48b9f4bb..d7b8a068 100644 --- a/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/UniformSamplingReservoir.java +++ b/whiterabbit/src/main/java/org/ohdsi/whiteRabbit/scan/UniformSamplingReservoir.java @@ -1,7 +1,6 @@ package org.ohdsi.whiteRabbit.scan; import java.math.BigDecimal; -import java.math.BigInteger; import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays;