From 01f5e5a6836123d4c70ad3672dcf320286cc1d1b Mon Sep 17 00:00:00 2001 From: Laurent Hasson Date: Mon, 17 May 2021 15:44:04 -0400 Subject: [PATCH] #606 made RecordProcessor start/end throw exception - updated processors in the framework - update code-gen for inline processors in factories --- .../ExporterCSVObjectProcessor.java | 33 +++++---- .../ExporterJSONObjectProcessor.java | 14 ++-- .../processors/ExporterObjectProcessor.java | 41 +++++++---- src/tilda/db/processors/ObjectProcessor.java | 13 ++-- src/tilda/db/processors/RecordProcessor.java | 2 + src/tilda/generation/Generator.java | 68 +++++++++---------- src/tilda/generation/java8/TildaFactory.java | 11 ++- 7 files changed, 104 insertions(+), 78 deletions(-) diff --git a/src/tilda/db/processors/ExporterCSVObjectProcessor.java b/src/tilda/db/processors/ExporterCSVObjectProcessor.java index 2b3f090be..019ddb4bf 100644 --- a/src/tilda/db/processors/ExporterCSVObjectProcessor.java +++ b/src/tilda/db/processors/ExporterCSVObjectProcessor.java @@ -17,7 +17,7 @@ package tilda.db.processors; import java.io.FileNotFoundException; -import java.io.PrintWriter; +import java.io.Writer; import java.lang.reflect.Method; import org.apache.logging.log4j.LogManager; @@ -29,34 +29,39 @@ public class ExporterCSVObjectProcessor extends ExporterObjec { protected static final Logger LOG = LogManager.getLogger(ExporterCSVObjectProcessor.class.getName()); - public ExporterCSVObjectProcessor(PrintWriter out, long logFreq, String name, Class factoryClass) + public ExporterCSVObjectProcessor(Writer out, String outName, long logFreq, Class factoryClass, boolean header) { - super(out, logFreq, name); + super(out, outName, logFreq); _factoryClass = factoryClass; + _header = header; } - public ExporterCSVObjectProcessor(String outFile, long logFreq, Class factoryClass) + public ExporterCSVObjectProcessor(String outFile, long logFreq, Class factoryClass, boolean header) throws FileNotFoundException { super(outFile, logFreq); _factoryClass = factoryClass; + _header = header; } protected Class _factoryClass; + protected boolean _header; @Override public void start() + throws Exception { super.start(); - try - { - Method M = _factoryClass.getMethod("getCSVHeader"); - _out.println((String) M.invoke(null)); - } - catch (Throwable E) - { - throw new Error("An error occurred.\n", E); - } + if (_header == true) + try + { + Method M = _factoryClass.getMethod("getCSVHeader"); + _out.append((String) M.invoke(null)).append(System.lineSeparator()); + } + catch (Throwable E) + { + throw new Error("An error occurred.\n", E); + } } @Override @@ -64,7 +69,7 @@ public boolean process(int count, T obj) throws Exception { obj.toCSV(_out, ""); - _out.println(); + _out.append(System.lineSeparator()); return super.process(count); } diff --git a/src/tilda/db/processors/ExporterJSONObjectProcessor.java b/src/tilda/db/processors/ExporterJSONObjectProcessor.java index dbf35cbfd..ae3202118 100644 --- a/src/tilda/db/processors/ExporterJSONObjectProcessor.java +++ b/src/tilda/db/processors/ExporterJSONObjectProcessor.java @@ -17,7 +17,7 @@ package tilda.db.processors; import java.io.FileNotFoundException; -import java.io.PrintWriter; +import java.io.Writer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,9 +28,9 @@ public class ExporterJSONObjectProcessor extends ExporterObj { protected static final Logger LOG = LogManager.getLogger(ExporterObjectProcessor.class.getName()); - public ExporterJSONObjectProcessor(PrintWriter out, long logFreq, String name, boolean jsonLines) + public ExporterJSONObjectProcessor(Writer out, String outName, long logFreq, boolean jsonLines) { - super(out, logFreq, name); + super(out, outName, logFreq); _jsonLines = jsonLines; } @@ -44,17 +44,18 @@ public ExporterJSONObjectProcessor(String outFile, long logFreq, boolean jsonLin protected boolean _jsonLines; @Override - public void start() + public void start() throws Exception { super.start(); if (_jsonLines == false) - _out.println("["); + _out.append("[").append(System.lineSeparator()); } @Override public boolean process(int count, T obj) throws Exception { +// LOG.debug("------------> "+count); if (_jsonLines == false) obj.toJSON(_out, "", count == 0 ? " " : " ,", true, true); else @@ -64,9 +65,10 @@ public boolean process(int count, T obj) @Override public void end(boolean hasMore, int maxCount) + throws Exception { if (_jsonLines == false) - _out.println("]"); + _out.append("]").append(System.lineSeparator()); super.end(hasMore, maxCount); } } diff --git a/src/tilda/db/processors/ExporterObjectProcessor.java b/src/tilda/db/processors/ExporterObjectProcessor.java index 71300abde..c78fff309 100644 --- a/src/tilda/db/processors/ExporterObjectProcessor.java +++ b/src/tilda/db/processors/ExporterObjectProcessor.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.PrintWriter; +import java.io.Writer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -29,14 +30,26 @@ public abstract class ExporterObjectProcessor implements ObjectProcessor { protected static final Logger LOG = LogManager.getLogger(ExporterObjectProcessor.class.getName()); - public ExporterObjectProcessor(PrintWriter out, long logFreq, String name) + /** + * + * @param out + * @param outName The name of the Writer + * @param logFreq + */ + public ExporterObjectProcessor(Writer out, String outName, long logFreq) { _out = out; _totalCount = 0; _logFreq = logFreq; - _name = name; + _outName = outName; } + /** + * + * @param outFile + * @param logFreq + * @throws FileNotFoundException + */ public ExporterObjectProcessor(String outFile, long logFreq) throws FileNotFoundException { @@ -44,19 +57,20 @@ public ExporterObjectProcessor(String outFile, long logFreq) _cleanWriter = true; _totalCount = 0; _logFreq = logFreq; - _name = outFile; + _outName = outFile; } - protected PrintWriter _out; - protected boolean _cleanWriter = false; - protected long _totalCount; - protected long _startTs; - protected long _endTs; - protected long _logFreq; - protected String _name; + protected Writer _out; + protected String _outName; + protected boolean _cleanWriter = false; + protected long _totalCount; + protected long _startTs; + protected long _endTs; + protected long _logFreq; public void start() + throws Exception { _startTs = System.nanoTime(); } @@ -68,20 +82,21 @@ public boolean process(int count) if (count % _logFreq == 0) { long durationNano = System.nanoTime() - _startTs; - LOG.info("Saved " + count + " records to CSV in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, count) + " records/min)"); + LOG.info("Saved " + count + " records in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, count) + " records/min)"); } return true; } public void end(boolean hasMore, int maxCount) + throws Exception { if (_cleanWriter == true) _out.close(); _endTs = System.nanoTime(); long durationNano = _endTs - _startTs; LOG.info("\n==========================================================================================================================================\n" - + "== " + _name + "\n" - + "== " + _totalCount + " records to CSV in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, _totalCount) + " records/min)\n" + + "== " + _outName + "\n" + + "== " + _totalCount + " records in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, _totalCount) + " records/min)\n" + "==========================================================================================================================================\n"); } diff --git a/src/tilda/db/processors/ObjectProcessor.java b/src/tilda/db/processors/ObjectProcessor.java index fbb9e1479..185a85fa7 100644 --- a/src/tilda/db/processors/ObjectProcessor.java +++ b/src/tilda/db/processors/ObjectProcessor.java @@ -22,19 +22,21 @@ public interface ObjectProcessor * Called before the first record is processed */ default public void start() + throws Exception { } - + /** * Called for each record - * + * * @param count the count of the object processed, starting at 0 for the first object processed. * @param obj the object processed. * @return true if processing was successful and should continue, or false if processing was unsuccessful and should be aborted. * @throws Exception */ - public boolean process(int count, T obj) throws Exception; - + public boolean process(int count, T obj) + throws Exception; + /** * Called after the last record has been processed successfully * @@ -42,6 +44,7 @@ default public void start() * @param maxCount the max count originally supplied to the query handler */ default void end(boolean hasMore, int maxCount) + throws Exception { - } + } } diff --git a/src/tilda/db/processors/RecordProcessor.java b/src/tilda/db/processors/RecordProcessor.java index 3c70b1d1e..bd4e9db35 100644 --- a/src/tilda/db/processors/RecordProcessor.java +++ b/src/tilda/db/processors/RecordProcessor.java @@ -24,6 +24,7 @@ public interface RecordProcessor * Called before the first record is processed */ default public void start() + throws Exception { } @@ -46,6 +47,7 @@ public boolean process(int count, ResultSet RS) * @param maxCount the max count originally supplied to the query handler */ default void end(boolean hasMore, int maxCount) + throws Exception { } } diff --git a/src/tilda/generation/Generator.java b/src/tilda/generation/Generator.java index 2d846f24a..af1db8660 100644 --- a/src/tilda/generation/Generator.java +++ b/src/tilda/generation/Generator.java @@ -17,6 +17,7 @@ package tilda.generation; import java.io.File; +import java.io.IOException; import java.io.PrintWriter; import java.util.List; @@ -157,25 +158,7 @@ protected static void genTildaBigQuerySchemas(GeneratorSession G, File GenFolder { File f = new File(GenFolder.getAbsolutePath() + File.separator + "bq." + O._Name + ".json"); PrintWriter Out = new PrintWriter(f); - Out.println("["); - boolean First = true; - for (Column col : O._Columns) - if (col != null) - { - if (First == true) - { - First = false; - Out.print(" {"); - } - else - Out.print(" ,{"); - JSONUtil.print(Out, "name", true, col.getName()); - JSONUtil.print(Out, "type", false, col.getType().getBigQueryType()); - JSONUtil.print(Out, "mode", false, col.isCollection() == true ? "REPEATED" : col._Nullable == false ? "REQUIRED" : "NULLABLE"); - JSONUtil.print(Out, "description", false, col._Description); - Out.println(" }"); - } - Out.println("]"); + genTildaBigQuerySchema(O, Out); Out.close(); } @@ -184,28 +167,39 @@ protected static void genTildaBigQuerySchemas(GeneratorSession G, File GenFolder { File f = new File(GenFolder.getAbsolutePath() + File.separator + "bq." + V._Name + ".json"); PrintWriter Out = new PrintWriter(f); - Out.println("["); - boolean First = true; - for (ViewColumn col : V._ViewColumns) - if (col != null) - { - if (First == true) - { - First = false; - Out.print(" {"); - } - else - Out.print(" ,{"); - JSONUtil.print(Out, "name", true, col.getName()); - JSONUtil.print(Out, "type", false, col.getType().getBigQueryType()); - JSONUtil.print(Out, "mode", false, col.isCollection() == true ? "REPEATED" : "NULLABLE"); - Out.println(" }"); - } - Out.println("]"); + genTildaBigQuerySchema(V, Out); Out.close(); } } + + public static void genTildaBigQuerySchema(Base O, PrintWriter Out) + throws IOException + { + Out.println("["); + boolean First = true; + for (String name : O.getColumnNames()) + if (name != null) + { + Column col = O.getColumn(name); + if (col == null) + continue; + if (First == true) + { + First = false; + Out.print(" {"); + } + else + Out.print(" ,{"); + JSONUtil.print(Out, "name", true, col.getName()); + JSONUtil.print(Out, "type", false, col.getType().getBigQueryType()); + JSONUtil.print(Out, "mode", false, col.isCollection() == true ? "REPEATED" : col._Nullable == false ? "REQUIRED" : "NULLABLE"); + JSONUtil.print(Out, "description", false, col._Description); + Out.println(" }"); + } + Out.println("]"); + } + public static void getTableDDL(CodeGenSql CG, PrintWriter Out, Object O, boolean mainDDL, boolean keysDDL) throws Exception { diff --git a/src/tilda/generation/java8/TildaFactory.java b/src/tilda/generation/java8/TildaFactory.java index 41ae009db..8ddd6af58 100644 --- a/src/tilda/generation/java8/TildaFactory.java +++ b/src/tilda/generation/java8/TildaFactory.java @@ -228,8 +228,8 @@ public void genClassStart(PrintWriter Out, GeneratorSession G, Object O) Out.println(" protected Connection _C = null;"); Out.println(" protected tilda.db.processors.ObjectProcessor<" + Helper.getFullAppDataClassName(O) + "> _OP;"); Out.println(" protected ArrayListResults<" + Helper.getFullAppDataClassName(O) + "> _L = null;"); - Out.println(" public void start () { if (_OP != null) _OP.start(); }"); - Out.println(" public void end (boolean hasMore, int maxCount) { if (_OP == null) _L.wrapup(hasMore, maxCount); else _OP.end(hasMore, maxCount); }"); + Out.println(" public void start () throws Exception { if (_OP != null) _OP.start(); }"); + Out.println(" public void end (boolean hasMore, int maxCount) throws Exception { if (_OP == null) _L.wrapup(hasMore, maxCount); else _OP.end(hasMore, maxCount); }"); Out.println(" public boolean process(int count, java.sql.ResultSet RS) throws Exception"); Out.println(" {"); Out.println(" " + Helper.getFullAppDataClassName(O) + " Obj = new " + Helper.getFullAppDataClassName(O) + "();"); @@ -1096,9 +1096,10 @@ protected static void genMethodToJSON(PrintWriter Out, GeneratorSession G, Outpu Out.println(" toJSON" + J._Name + "(out, obj, lead, fullObject, false);"); Out.println(" }"); Out.println(); - Out.println(" public static void toJSON" + J._Name + "(java.io.Writer out, " + Helper.getFullAppDataClassName(J._ParentObject) + " obj, String lead, boolean fullObject, boolean noNullArrays) throws java.io.IOException"); + Out.println(" public static void toJSON" + J._Name + "(java.io.Writer outWriter, " + Helper.getFullAppDataClassName(J._ParentObject) + " obj, String lead, boolean fullObject, boolean noNullArrays) throws java.io.IOException"); Out.println(" {"); Out.println(" long T0 = System.nanoTime();"); + Out.println(" org.apache.commons.io.output.StringBuilderWriter out = new org.apache.commons.io.output.StringBuilderWriter();"); Out.println(" " + Helper.getFullBaseClassName(J._ParentObject) + " Obj = (" + Helper.getFullBaseClassName(J._ParentObject) + ") obj;"); Out.println(" if (fullObject == true)"); Out.println(" {"); @@ -1112,6 +1113,10 @@ protected static void genMethodToJSON(PrintWriter Out, GeneratorSession G, Outpu Helper.JSONExport(Out, C); Out.println(" if (fullObject == true)"); Out.println(" out.write(\" }\\n\");"); + Out.println(); + Out.println(" outWriter.append(out.getBuilder().toString());"); + Out.println(" out.close();"); + Out.println(); Out.println(" PerfTracker.add(TransactionType.TILDA_TOJSON, System.nanoTime() - T0);"); Out.println(" }");