Skip to content

Commit

Permalink
#606 made RecordProcessor start/end throw exception
Browse files Browse the repository at this point in the history 
- updated processors in the framework
- update code-gen for inline processors in factories
  • Loading branch information
@ldhasson
ldhasson committed May 17, 2021
1 parent 56b71f0 commit 01f5e5a
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 78 deletions.
33 changes: 19 additions & 14 deletions src/tilda/db/processors/ExporterCSVObjectProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package tilda.db.processors;

import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.Writer;
import java.lang.reflect.Method;

import org.apache.logging.log4j.LogManager;
Expand All @@ -29,42 +29,47 @@ public class ExporterCSVObjectProcessor<T extends CSVable> extends ExporterObjec
{
protected static final Logger LOG = LogManager.getLogger(ExporterCSVObjectProcessor.class.getName());

public ExporterCSVObjectProcessor(PrintWriter out, long logFreq, String name, Class<?> factoryClass)
public ExporterCSVObjectProcessor(Writer out, String outName, long logFreq, Class<?> factoryClass, boolean header)
{
super(out, logFreq, name);
super(out, outName, logFreq);
_factoryClass = factoryClass;
_header = header;
}

public ExporterCSVObjectProcessor(String outFile, long logFreq, Class<?> factoryClass)
public ExporterCSVObjectProcessor(String outFile, long logFreq, Class<?> factoryClass, boolean header)
throws FileNotFoundException
{
super(outFile, logFreq);
_factoryClass = factoryClass;
_header = header;
}

protected Class<?> _factoryClass;
protected boolean _header;

@Override
public void start()
throws Exception
{
super.start();
try
{
Method M = _factoryClass.getMethod("getCSVHeader");
_out.println((String) M.invoke(null));
}
catch (Throwable E)
{
throw new Error("An error occurred.\n", E);
}
if (_header == true)
try
{
Method M = _factoryClass.getMethod("getCSVHeader");
_out.append((String) M.invoke(null)).append(System.lineSeparator());
}
catch (Throwable E)
{
throw new Error("An error occurred.\n", E);
}
}

@Override
public boolean process(int count, T obj)
throws Exception
{
obj.toCSV(_out, "");
_out.println();
_out.append(System.lineSeparator());
return super.process(count);
}

Expand Down
14 changes: 8 additions & 6 deletions src/tilda/db/processors/ExporterJSONObjectProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package tilda.db.processors;

import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.Writer;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -28,9 +28,9 @@ public class ExporterJSONObjectProcessor<T extends JSONable> extends ExporterObj
{
protected static final Logger LOG = LogManager.getLogger(ExporterObjectProcessor.class.getName());

public ExporterJSONObjectProcessor(PrintWriter out, long logFreq, String name, boolean jsonLines)
public ExporterJSONObjectProcessor(Writer out, String outName, long logFreq, boolean jsonLines)
{
super(out, logFreq, name);
super(out, outName, logFreq);
_jsonLines = jsonLines;
}

Expand All @@ -44,17 +44,18 @@ public ExporterJSONObjectProcessor(String outFile, long logFreq, boolean jsonLin
protected boolean _jsonLines;

@Override
public void start()
public void start() throws Exception
{
super.start();
if (_jsonLines == false)
_out.println("[");
_out.append("[").append(System.lineSeparator());
}

@Override
public boolean process(int count, T obj)
throws Exception
{
// LOG.debug("------------> "+count);
if (_jsonLines == false)
obj.toJSON(_out, "", count == 0 ? " " : " ,", true, true);
else
Expand All @@ -64,9 +65,10 @@ public boolean process(int count, T obj)

@Override
public void end(boolean hasMore, int maxCount)
throws Exception
{
if (_jsonLines == false)
_out.println("]");
_out.append("]").append(System.lineSeparator());
super.end(hasMore, maxCount);
}
}
41 changes: 28 additions & 13 deletions src/tilda/db/processors/ExporterObjectProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.Writer;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -29,34 +30,47 @@ public abstract class ExporterObjectProcessor<T> implements ObjectProcessor<T>
{
protected static final Logger LOG = LogManager.getLogger(ExporterObjectProcessor.class.getName());

public ExporterObjectProcessor(PrintWriter out, long logFreq, String name)
/**
*
* @param out
* @param outName The name of the Writer
* @param logFreq
*/
public ExporterObjectProcessor(Writer out, String outName, long logFreq)
{
_out = out;
_totalCount = 0;
_logFreq = logFreq;
_name = name;
_outName = outName;
}

/**
*
* @param outFile
* @param logFreq
* @throws FileNotFoundException
*/
public ExporterObjectProcessor(String outFile, long logFreq)
throws FileNotFoundException
{
_out = new PrintWriter(new File(outFile));
_cleanWriter = true;
_totalCount = 0;
_logFreq = logFreq;
_name = outFile;
_outName = outFile;
}

protected PrintWriter _out;
protected boolean _cleanWriter = false;
protected long _totalCount;
protected long _startTs;
protected long _endTs;
protected long _logFreq;
protected String _name;
protected Writer _out;
protected String _outName;
protected boolean _cleanWriter = false;
protected long _totalCount;
protected long _startTs;
protected long _endTs;
protected long _logFreq;


public void start()
throws Exception
{
_startTs = System.nanoTime();
}
Expand All @@ -68,20 +82,21 @@ public boolean process(int count)
if (count % _logFreq == 0)
{
long durationNano = System.nanoTime() - _startTs;
LOG.info("Saved " + count + " records to CSV in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, count) + " records/min)");
LOG.info("Saved " + count + " records in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, count) + " records/min)");
}
return true;
}

public void end(boolean hasMore, int maxCount)
throws Exception
{
if (_cleanWriter == true)
_out.close();
_endTs = System.nanoTime();
long durationNano = _endTs - _startTs;
LOG.info("\n==========================================================================================================================================\n"
+ "== " + _name + "\n"
+ "== " + _totalCount + " records to CSV in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, _totalCount) + " records/min)\n"
+ "== " + _outName + "\n"
+ "== " + _totalCount + " records in " + DurationUtil.printDuration(durationNano) + " (" + DurationUtil.printPerformancePerMinute(durationNano, _totalCount) + " records/min)\n"
+ "==========================================================================================================================================\n");
}

Expand Down
13 changes: 8 additions & 5 deletions src/tilda/db/processors/ObjectProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,29 @@ public interface ObjectProcessor<T>
* Called before the first record is processed
*/
default public void start()
throws Exception
{
}

/**
* Called for each record
*
*
* @param count the count of the object processed, starting at 0 for the first object processed.
* @param obj the object processed.
* @return true if processing was successful and should continue, or false if processing was unsuccessful and should be aborted.
* @throws Exception
*/
public boolean process(int count, T obj) throws Exception;

public boolean process(int count, T obj)
throws Exception;

/**
* Called after the last record has been processed successfully
*
* @param hasMore whether there are more records to be processed
* @param maxCount the max count originally supplied to the query handler
*/
default void end(boolean hasMore, int maxCount)
throws Exception
{
}
}
}
2 changes: 2 additions & 0 deletions src/tilda/db/processors/RecordProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public interface RecordProcessor
* Called before the first record is processed
*/
default public void start()
throws Exception
{
}

Expand All @@ -46,6 +47,7 @@ public boolean process(int count, ResultSet RS)
* @param maxCount the max count originally supplied to the query handler
*/
default void end(boolean hasMore, int maxCount)
throws Exception
{
}
}
68 changes: 31 additions & 37 deletions src/tilda/generation/Generator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package tilda.generation;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;

Expand Down Expand Up @@ -157,25 +158,7 @@ protected static void genTildaBigQuerySchemas(GeneratorSession G, File GenFolder
{
File f = new File(GenFolder.getAbsolutePath() + File.separator + "bq." + O._Name + ".json");
PrintWriter Out = new PrintWriter(f);
Out.println("[");
boolean First = true;
for (Column col : O._Columns)
if (col != null)
{
if (First == true)
{
First = false;
Out.print(" {");
}
else
Out.print(" ,{");
JSONUtil.print(Out, "name", true, col.getName());
JSONUtil.print(Out, "type", false, col.getType().getBigQueryType());
JSONUtil.print(Out, "mode", false, col.isCollection() == true ? "REPEATED" : col._Nullable == false ? "REQUIRED" : "NULLABLE");
JSONUtil.print(Out, "description", false, col._Description);
Out.println(" }");
}
Out.println("]");
genTildaBigQuerySchema(O, Out);
Out.close();
}

Expand All @@ -184,28 +167,39 @@ protected static void genTildaBigQuerySchemas(GeneratorSession G, File GenFolder
{
File f = new File(GenFolder.getAbsolutePath() + File.separator + "bq." + V._Name + ".json");
PrintWriter Out = new PrintWriter(f);
Out.println("[");
boolean First = true;
for (ViewColumn col : V._ViewColumns)
if (col != null)
{
if (First == true)
{
First = false;
Out.print(" {");
}
else
Out.print(" ,{");
JSONUtil.print(Out, "name", true, col.getName());
JSONUtil.print(Out, "type", false, col.getType().getBigQueryType());
JSONUtil.print(Out, "mode", false, col.isCollection() == true ? "REPEATED" : "NULLABLE");
Out.println(" }");
}
Out.println("]");
genTildaBigQuerySchema(V, Out);
Out.close();
}
}


public static void genTildaBigQuerySchema(Base O, PrintWriter Out)
throws IOException
{
Out.println("[");
boolean First = true;
for (String name : O.getColumnNames())
if (name != null)
{
Column col = O.getColumn(name);
if (col == null)
continue;
if (First == true)
{
First = false;
Out.print(" {");
}
else
Out.print(" ,{");
JSONUtil.print(Out, "name", true, col.getName());
JSONUtil.print(Out, "type", false, col.getType().getBigQueryType());
JSONUtil.print(Out, "mode", false, col.isCollection() == true ? "REPEATED" : col._Nullable == false ? "REQUIRED" : "NULLABLE");
JSONUtil.print(Out, "description", false, col._Description);
Out.println(" }");
}
Out.println("]");
}

public static void getTableDDL(CodeGenSql CG, PrintWriter Out, Object O, boolean mainDDL, boolean keysDDL)
throws Exception
{
Expand Down
11 changes: 8 additions & 3 deletions src/tilda/generation/java8/TildaFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,8 @@ public void genClassStart(PrintWriter Out, GeneratorSession G, Object O)
Out.println(" protected Connection _C = null;");
Out.println(" protected tilda.db.processors.ObjectProcessor<" + Helper.getFullAppDataClassName(O) + "> _OP;");
Out.println(" protected ArrayListResults<" + Helper.getFullAppDataClassName(O) + "> _L = null;");
Out.println(" public void start () { if (_OP != null) _OP.start(); }");
Out.println(" public void end (boolean hasMore, int maxCount) { if (_OP == null) _L.wrapup(hasMore, maxCount); else _OP.end(hasMore, maxCount); }");
Out.println(" public void start () throws Exception { if (_OP != null) _OP.start(); }");
Out.println(" public void end (boolean hasMore, int maxCount) throws Exception { if (_OP == null) _L.wrapup(hasMore, maxCount); else _OP.end(hasMore, maxCount); }");
Out.println(" public boolean process(int count, java.sql.ResultSet RS) throws Exception");
Out.println(" {");
Out.println(" " + Helper.getFullAppDataClassName(O) + " Obj = new " + Helper.getFullAppDataClassName(O) + "();");
Expand Down Expand Up @@ -1096,9 +1096,10 @@ protected static void genMethodToJSON(PrintWriter Out, GeneratorSession G, Outpu
Out.println(" toJSON" + J._Name + "(out, obj, lead, fullObject, false);");
Out.println(" }");
Out.println();
Out.println(" public static void toJSON" + J._Name + "(java.io.Writer out, " + Helper.getFullAppDataClassName(J._ParentObject) + " obj, String lead, boolean fullObject, boolean noNullArrays) throws java.io.IOException");
Out.println(" public static void toJSON" + J._Name + "(java.io.Writer outWriter, " + Helper.getFullAppDataClassName(J._ParentObject) + " obj, String lead, boolean fullObject, boolean noNullArrays) throws java.io.IOException");
Out.println(" {");
Out.println(" long T0 = System.nanoTime();");
Out.println(" org.apache.commons.io.output.StringBuilderWriter out = new org.apache.commons.io.output.StringBuilderWriter();");
Out.println(" " + Helper.getFullBaseClassName(J._ParentObject) + " Obj = (" + Helper.getFullBaseClassName(J._ParentObject) + ") obj;");
Out.println(" if (fullObject == true)");
Out.println(" {");
Expand All @@ -1112,6 +1113,10 @@ protected static void genMethodToJSON(PrintWriter Out, GeneratorSession G, Outpu
Helper.JSONExport(Out, C);
Out.println(" if (fullObject == true)");
Out.println(" out.write(\" }\\n\");");
Out.println();
Out.println(" outWriter.append(out.getBuilder().toString());");
Out.println(" out.close();");
Out.println();
Out.println(" PerfTracker.add(TransactionType.TILDA_TOJSON, System.nanoTime() - T0);");
Out.println(" }");

Expand Down

0 comments on commit 01f5e5a

Please sign in to comment.