From 4a01d63411449a8119d2c47c7c137887de8b532c Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Mon, 23 Sep 2019 11:18:54 +0200 Subject: [PATCH 01/13] list views to scan for mssql, oracle, msaccess and teradata --- .../java/org/ohdsi/databases/RichConnection.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java b/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java index a390dc5e..d654f334 100644 --- a/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java +++ b/rabbit-core/src/main/java/org/ohdsi/databases/RichConnection.java @@ -145,15 +145,22 @@ public List getTableNames(String database) { if (dbType == DbType.MYSQL) { query = "SHOW TABLES IN " + database; } else if (dbType == DbType.MSSQL || dbType == DbType.PDW) { - query = "SELECT CONCAT(schemas.name, '.', tables.name) FROM " + database + ".sys.tables INNER JOIN " + database + ".sys.schemas ON tables.schema_id = schemas.schema_id ORDER BY schemas.name, tables.name"; + query = "SELECT CONCAT(schemas.name, '.', tables_views.name) FROM " + + "(SELECT schema_id, name FROM %1$s.sys.tables UNION ALL SELECT schema_id, name FROM %1$s.sys.views) tables_views " + + "INNER JOIN %1$s.sys.schemas ON tables_views.schema_id = schemas.schema_id " + + "ORDER BY schemas.name, tables_views.name"; + query = String.format(query, database); + System.out.println(query); } else if (dbType == DbType.ORACLE) { - query = "SELECT table_name FROM all_tables WHERE owner='" + database.toUpperCase() + "'"; + query = "SELECT table_name FROM " + + "(SELECT table_name, owner FROM all_tables UNION ALL SELECT view_name, owner FROM all_views) tables_views " + + "WHERE owner='" + database.toUpperCase() + "'"; } else if (dbType == DbType.POSTGRESQL || dbType == DbType.REDSHIFT) { query = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + database.toLowerCase() + "' ORDER BY table_name"; } else if (dbType == DbType.MSACCESS) { - query = "SELECT Name FROM sys.MSysObjects WHERE Type=1 AND Flags=0;"; + query = "SELECT Name FROM sys.MSysObjects WHERE (Type=1 OR Type=5) AND Flags=0;"; } else if (dbType == DbType.TERADATA) { - query = "SELECT TableName from dbc.tables WHERE tablekind = 'T' and databasename='" + database + "'"; + query = "SELECT TableName from dbc.tables WHERE tablekind IN ('T','V') and databasename='" + database + "'"; } for (Row row : query(query)) From b7ff76eddeeb95edd078a998581aaa3c2f92bcd4 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 10 Jun 2020 21:46:39 +0200 Subject: [PATCH 02/13] bump snapshot version --- pom.xml | 2 +- rabbit-core/pom.xml | 2 +- rabbitinahat/pom.xml | 2 +- whiterabbit/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 762672c7..6512839e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.ohdsi leporidae pom - 0.10.1 + 0.10.2-SNAPSHOT rabbitinahat whiterabbit diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml index a6e4e8d4..cb84fbbd 100644 --- a/rabbit-core/pom.xml +++ b/rabbit-core/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.10.1 + 0.10.2-SNAPSHOT 4.0.0 diff --git a/rabbitinahat/pom.xml b/rabbitinahat/pom.xml index 1d290a9b..d0bdf203 100644 --- a/rabbitinahat/pom.xml +++ b/rabbitinahat/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.10.1 + 0.10.2-SNAPSHOT 4.0.0 diff --git a/whiterabbit/pom.xml b/whiterabbit/pom.xml index 2752e3ee..fa5baf5f 100644 --- a/whiterabbit/pom.xml +++ b/whiterabbit/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.10.1 + 0.10.2-SNAPSHOT 4.0.0 From c0fdb69f4f4c0d88a945585ca1233b2269c8bfb5 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Fri, 19 Jun 2020 11:18:15 +0200 Subject: [PATCH 03/13] fixes #242 --- .../ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java index 97e2cdc1..f748ed69 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java @@ -335,7 +335,7 @@ private void createLookupFunctions() { writer.println(" } else {"); writer.println(" statement <- paste0(statement, \" AND\")"); writer.println(" }"); - writer.println(" statement <- paste0(statement, \" " + sqlFieldName + " = \", " + createSqlValueCode(rFieldName) + ")"); + writer.println(" statement <- paste0(statement, \" " + sqlFieldName + "\"," + createSqlValueCode(rFieldName) + ")"); writer.println(" }"); writer.println(""); } @@ -454,7 +454,7 @@ private void createGenerateTestSqlFunction() { writer.println(" \"' AS test, CASE WHEN (SELECT COUNT(*) FROM @cdm_database_schema.\","); writer.println(" expect$table,"); writer.println(" \" WHERE \","); - writer.println(" paste(paste(expect$fields, operators, expect$values), collapse = \" AND \"),"); + writer.println(" paste(paste(expect$fields, expect$values), collapse = \" AND \"),"); writer.println(" \") \","); writer.println(" if (expect$type == " + DEFAULT + ") \"= 0\" else if (expect$type == " + NEGATE + ") \"!= 0\" else paste(\"!=\", expect$rowCount),"); @@ -582,10 +582,10 @@ private String convertToRName(String name) { } private String createSqlValueCode(String rFieldName) { - return "if (is.null(" + rFieldName + ")) \"NULL\" " + + return "if (is.null(" + rFieldName + ")) \" IS NULL\" " + "else if (is(" + rFieldName + ", \"subQuery\")) " + - "paste0(\"(\", as.character(" + rFieldName + "), \")\") " + - "else paste0(\"'\", as.character(" + rFieldName + "), \"'\")"; + "paste0(\" = (\", as.character(" + rFieldName + "), \")\") " + + "else paste0(\" = '\", as.character(" + rFieldName + "), \"'\")"; } private String convertToSqlName(String name) { From 445ff9d67b5482da94b53b4c19e6c583ee734d3d Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Tue, 23 Jun 2020 10:01:23 +0200 Subject: [PATCH 04/13] fix bug in writing source test data values --- .../ETLTestFrameWorkGenerator.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java index f748ed69..713ee344 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/ETLTestFrameWorkGenerator.java @@ -31,6 +31,7 @@ import org.ohdsi.rabbitInAHat.dataModel.*; import org.ohdsi.rabbitInAHat.dataModel.ETL.FileFormat; +// TODO: use templating to generate the R code (e.g. Jinja/Apache FreeMarker/Mustache). At least put static R code in separate (.R) files. public class ETLTestFrameWorkGenerator { private static int DEFAULT = 0; @@ -260,7 +261,7 @@ private void createAddFunctions() { writer.println(" frameworkContext$sourceFieldsTested <- c(frameworkContext$sourceFieldsTested, '" + convertFieldToFullName(field) + "')"); writer.println(" }"); writer.println(" fields <- c(fields, \"" + sqlFieldName + "\")"); - writer.println(" values <- c(values, " + createSqlValueCode(rFieldName) + ")"); + writer.println(" values <- c(values, " + createValueCode(rFieldName) + ")"); writer.println(""); } writer.println(" inserts <- list(testId = frameworkContext$testId, testDescription = frameworkContext$testDescription, table = \"" + sqlTableName @@ -581,11 +582,18 @@ private String convertToRName(String name) { return name; } - private String createSqlValueCode(String rFieldName) { - return "if (is.null(" + rFieldName + ")) \" IS NULL\" " + + private String createSqlValueCode(String rFieldName) { + return "if (is.null(" + rFieldName + ")) \" IS NULL\" " + + "else if (is(" + rFieldName + ", \"subQuery\")) " + + "paste0(\" = (\", as.character(" + rFieldName + "), \")\") " + + "else paste0(\" = '\", as.character(" + rFieldName + "), \"'\")"; + } + + private String createValueCode(String rFieldName) { + return "if (is.null(" + rFieldName + ")) \"NULL\" " + "else if (is(" + rFieldName + ", \"subQuery\")) " + - "paste0(\" = (\", as.character(" + rFieldName + "), \")\") " + - "else paste0(\" = '\", as.character(" + rFieldName + "), \"'\")"; + "paste0(\"(\", as.character(" + rFieldName + "), \")\") " + + "else paste0(\"'\", as.character(" + rFieldName + "), \"'\")"; } private String convertToSqlName(String name) { From b6f439441df149da9745a628efd35824b8d5fbbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2020 23:07:38 +0000 Subject: [PATCH 05/13] Bump mysql-connector-java from 5.1.37 to 8.0.16 in /rabbit-core Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 5.1.37 to 8.0.16. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/5.1.37...8.0.16) Signed-off-by: dependabot[bot] --- rabbit-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml index a6e4e8d4..a17acf1f 100644 --- a/rabbit-core/pom.xml +++ b/rabbit-core/pom.xml @@ -30,7 +30,7 @@ mysql mysql-connector-java - 5.1.37 + 8.0.16 dom4j From ba4128bfb8155a02e8071c4b06dfca2cc51ef55a Mon Sep 17 00:00:00 2001 From: Spayralbe Date: Wed, 29 Jul 2020 11:44:56 +0200 Subject: [PATCH 06/13] Expand gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8cfd120f..c3477ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,13 @@ dist/ /.classpath *.iml .idea/ +target/ +*.class # Repo specific ignores /ScanReport.xlsx *swp tmp/* Try* -/examples/ \ No newline at end of file +/examples/ +.DS_Store \ No newline at end of file From 2da703473a0937d2f10827c2961fd2327aaaaeb7 Mon Sep 17 00:00:00 2001 From: Spayralbe Date: Wed, 29 Jul 2020 11:47:47 +0200 Subject: [PATCH 07/13] Make FilterDialog a singleton --- .../org/ohdsi/rabbitInAHat/FilterDialog.java | 35 ++++++++++++++++--- .../ohdsi/rabbitInAHat/RabbitInAHatMain.java | 14 ++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java index 259803b6..9c8933e6 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java @@ -8,6 +8,8 @@ import java.awt.event.FocusListener; import java.awt.event.KeyEvent; import java.awt.event.KeyListener; +import java.awt.event.WindowAdapter; +import java.awt.event.WindowEvent; import org.ohdsi.rabbitInAHat.ResizeListener; @@ -16,6 +18,7 @@ import javax.swing.JLabel; import javax.swing.JTextField; import javax.swing.SpringLayout; +import javax.swing.WindowConstants; public class FilterDialog extends JDialog implements ActionListener, ResizeListener { @@ -24,6 +27,7 @@ public class FilterDialog extends JDialog implements ActionListener, ResizeListe * */ private static final long serialVersionUID = 7009265246652874341L; + private static FilterDialog instance; private JTextField sourceSearchField; @@ -34,9 +38,11 @@ public class FilterDialog extends JDialog implements ActionListener, ResizeListe Container contentPane = this.getContentPane(); - public FilterDialog(Window parentWindow){ - - super(parentWindow,"Filter",ModalityType.MODELESS); + public FilterDialog(Window parentWindow) throws ExceptionInInitializerError{ + super(parentWindow,"Filter",ModalityType.MODELESS); + if (instance != null) { + throw new ExceptionInInitializerError("An instance of FilterDialog already exists"); + } this.setResizable(false); this.setLocation(parentWindow.getX()+parentWindow.getWidth()/2, parentWindow.getY()+100); @@ -96,7 +102,28 @@ public FilterDialog(Window parentWindow){ layout.putConstraint(SpringLayout.EAST, contentPane, 5, SpringLayout.EAST, targetClearBtn); this.pack(); - }; + + // Make sure the instance is reset to null when the search dialog window is closed + this.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE); + this.addWindowListener(new WindowAdapter() { + @Override + public void windowClosing(WindowEvent arg0) { + instance = null; + } + }); + + instance = this; + } + + public static boolean alreadyOpened(){ + return (instance != null); + } + + public static void bringToFront(){ + if (alreadyOpened()) { + instance.toFront(); + } + } public void setFilterPanel(MappingPanel aFilterPanel){ if (filterPanel != null) { diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java index 90996559..10ff392c 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/RabbitInAHatMain.java @@ -489,12 +489,14 @@ private void doSetTargetCDM(CDMVersion cdmVersion) { } private void doOpenFilterDialog() { - FilterDialog filter; - filter = new FilterDialog(frame); - - filter.setFilterPanel(tableMappingPanel); - - filter.setVisible(true); + if (FilterDialog.alreadyOpened()){ + FilterDialog.bringToFront(); + } + else { + FilterDialog filter = new FilterDialog(frame); + filter.setFilterPanel(tableMappingPanel); + filter.setVisible(true); + } } private void doMakeMappings() { From 23a0d9e5bd757a0d5500063b7e7898c7a48063ef Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Thu, 30 Jul 2020 11:56:05 +0200 Subject: [PATCH 08/13] set WhiteRabbit memory allocation back to 1200m --- whiterabbit/pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/whiterabbit/pom.xml b/whiterabbit/pom.xml index fa5baf5f..f3494c91 100644 --- a/whiterabbit/pom.xml +++ b/whiterabbit/pom.xml @@ -24,8 +24,7 @@ appassembler-maven-plugin ${project.parent.basedir}/dist - - -Xmx2400m + -Xmx1200m org.ohdsi.whiteRabbit.WhiteRabbitMain From 362a32e2162ae579ac2aa9f513ef069a490eb73a Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Thu, 30 Jul 2020 12:07:11 +0200 Subject: [PATCH 09/13] add instructions for memory issues --- docs/WhiteRabbit.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/WhiteRabbit.md b/docs/WhiteRabbit.md index ddaa6a2c..27d98b16 100644 --- a/docs/WhiteRabbit.md +++ b/docs/WhiteRabbit.md @@ -35,6 +35,13 @@ Once the scan report is created, this report can then be used in the Rabbit-In-a Note: on releases earlier than version 0.8.0, open the respective WhiteRabbit.jar or RabbitInAHat.jar files instead. +### Memory +WhiteRabbit possibly does not start when the memory allocated by the JVM is too big or too small. +By default this is set to 1200m. +To increase the memory (in this example to 2400m), either set the environment variable `EXTRA_JVM_ARGUMENTS=-Xmx2400m` before starting or edit in `bin/WhiteRabbit.bat` the line `%JAVACMD% %JAVA_OPTS% -Xmx2400m...`. +To lower the memory, set one of these variables to e.g. `-Xmx600m`. +If you have a 32-bit Java VM installed and problems persist, consider installing 64-bit Java. + ## Support All source code, descriptions and input/output examples are available on GitHub: From 90c3a2574e56fc1275e6723fcf8c6a33d67964a8 Mon Sep 17 00:00:00 2001 From: Spayralbe Date: Thu, 30 Jul 2020 15:45:48 +0200 Subject: [PATCH 10/13] Minor refactoring --- .../src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java index 9c8933e6..a39f6b14 100644 --- a/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java +++ b/rabbitinahat/src/main/java/org/ohdsi/rabbitInAHat/FilterDialog.java @@ -40,7 +40,7 @@ public class FilterDialog extends JDialog implements ActionListener, ResizeListe public FilterDialog(Window parentWindow) throws ExceptionInInitializerError{ super(parentWindow,"Filter",ModalityType.MODELESS); - if (instance != null) { + if (alreadyOpened()) { throw new ExceptionInInitializerError("An instance of FilterDialog already exists"); } this.setResizable(false); From 1c1c9fde3bf19af849ef3ac97e1fef8e4b4373be Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 5 Aug 2020 15:54:43 +0000 Subject: [PATCH 11/13] fix: rabbit-core/pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHEPOI-32049 - https://snyk.io/vuln/SNYK-JAVA-ORGPOSTGRESQL-173997 --- rabbit-core/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml index a6e4e8d4..a72cabbb 100644 --- a/rabbit-core/pom.xml +++ b/rabbit-core/pom.xml @@ -40,17 +40,17 @@ org.apache.poi poi - 3.9 + 3.17 org.apache.poi poi-ooxml - 3.9 + 3.17 org.apache.poi poi-excelant - 3.9 + 3.17 org.apache.poi @@ -70,7 +70,7 @@ org.postgresql postgresql - 9.4-1204-jdbc41 + 42.2.5 com.cedarsoftware From 6edbd0cca29db8ee4382ec50a617e115aec47022 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 5 Aug 2020 17:59:29 +0200 Subject: [PATCH 12/13] bump version --- pom.xml | 2 +- rabbit-core/pom.xml | 2 +- whiterabbit/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 6512839e..70f033da 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.ohdsi leporidae pom - 0.10.2-SNAPSHOT + 0.10.2 rabbitinahat whiterabbit diff --git a/rabbit-core/pom.xml b/rabbit-core/pom.xml index 5457a6d2..b96bce1f 100644 --- a/rabbit-core/pom.xml +++ b/rabbit-core/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.10.2-SNAPSHOT + 0.10.2 4.0.0 diff --git a/whiterabbit/pom.xml b/whiterabbit/pom.xml index f3494c91..439bcfea 100644 --- a/whiterabbit/pom.xml +++ b/whiterabbit/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.10.2-SNAPSHOT + 0.10.2 4.0.0 From 246f5943dbf6a9fdadbc29f49d169c69b7774fc7 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Sat, 15 Aug 2020 12:32:58 +0200 Subject: [PATCH 13/13] bump riah version --- rabbitinahat/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rabbitinahat/pom.xml b/rabbitinahat/pom.xml index d0bdf203..fe67c65a 100644 --- a/rabbitinahat/pom.xml +++ b/rabbitinahat/pom.xml @@ -5,7 +5,7 @@ leporidae org.ohdsi - 0.10.2-SNAPSHOT + 0.10.2 4.0.0