Skip to content

Commit

Permalink
Append mode works now also in Streaming workbooks
Browse files Browse the repository at this point in the history
  • Loading branch information
jlolling committed Sep 13, 2023
1 parent 6c7becf commit 84d7368
Show file tree
Hide file tree
Showing 7 changed files with 276 additions and 22 deletions.
137 changes: 121 additions & 16 deletions dependency-reduced-pom.xml
Original file line number Diff line number Diff line change
@@ -1,22 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>de.cimt.talendcomp</groupId>
<artifactId>cimt-talendcomp-excel</artifactId>
<name>Excel Output POI</name>
<version>7.4</version>
<groupId>de.jlo.talendcomp</groupId>
<artifactId>jlo-talendcomp-excel</artifactId>
<name>Talend Excel Components</name>
<version>13.8</version>
<organization>
<name>Jan Lolling</name>
<url>http://www.jan-lolling.de</url>
</organization>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<version>3.8.1</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
Expand All @@ -27,10 +26,122 @@
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer />
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>de.cimt.talendcomp</groupId>
<artifactId>cimt-talendcomp-maven-plugin</artifactId>
<version>3.1</version>
<executions>
<execution>
<id>tFileExcelWorkbookOpen</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelWorkbookOpen</componentName>
</configuration>
</execution>
<execution>
<id>tFileExcelWorkbookSave</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelWorkbookSave</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelSheetInput</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelSheetInput</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelSheetInputUnpivot</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelSheetInputUnpivot</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelSheetOutput</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelSheetOutput</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelSheetList</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelSheetList</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelReferencedCellInput</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelReferencedCellInput</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelReferencedCellOutput</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelReferencedCellOutput</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelNamedCellInput</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelNamedCellInput</componentName>
<noJars>true</noJars>
</configuration>
</execution>
<execution>
<id>tFileExcelNamedCellOutput</id>
<goals>
<goal>component</goal>
</goals>
<configuration>
<componentName>tFileExcelNamedCellOutput</componentName>
<noJars>true</noJars>
</configuration>
</execution>
</executions>
<configuration>
<finalName>${artifactId}-${version}</finalName>
<studioUserComponentFolder>${custom_component_folder}</studioUserComponentFolder>
<useTalendLibrariesMavenLocation>false</useTalendLibrariesMavenLocation>
</configuration>
</plugin>
</plugins>
Expand All @@ -39,7 +150,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<version>4.13.1</version>
<scope>test</scope>
<exclusions>
<exclusion>
Expand All @@ -48,12 +159,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.2.3</version>
<version>5.2.3-patched</version>
<exclusions>
<exclusion>
<artifactId>stax-api</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,5 @@ REUSE_FIRST_ROW_HEIGHT.NAME=Zeilenh
GROUP_COLS.NAME=Gruppiere Spalten (z.B. "A-G,Z-AA")
EXTEND_TABLE_CELL_RANGE.NAME=Erweitere Zellenbereich für beschriebene Tabellen
REUSE_STYLES_FOR_ALL.NAME=Kopiere Stile für alle Spalten
EXTEND_DATA_VALIDATIONS.NAME=Erweitere Datenvalidierungen um die neuen Zeilen
EXTEND_DATA_VALIDATIONS.NAME=Erweitere Datenvalidierungen um die neuen Zeilen
ROW_START_INDEX_LABEL=Aus diese Zeile werden die Formatinformationen für die nächsten Zeilen verwendet
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,5 @@ REUSE_FIRST_ROW_HEIGHT.NAME=Utiliser hauteur de 1
GROUP_COLS.NAME=Rassembler colonnes (p.ex. "A-G,Z-AA")
EXTEND_TABLE_CELL_RANGE.NAME=Augmenter taille de tableau pour tableaux remplis
REUSE_STYLES_FOR_ALL.NAME=Copier styles pour toutes les colonnes
EXTEND_DATA_VALIDATIONS.NAME=Etendre les validations de dates pour les nouvelles lignes
EXTEND_DATA_VALIDATIONS.NAME=Etendre les validations de dates pour les nouvelles lignes
ROW_START_INDEX_LABEL=Les informations de formatage pour les lignes suivantes sont utilisées à partir de cette ligne
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ TEMPLATE_SHEETS.NAME=Tabellenbl
TEMPLATE_SHEETS.ITEM.SHEET=Blatt Name oder Index
OPTIMZE_HSSF_WB_STYLES.NAME=Optimiere Stile (kann Dateigrösse reduzieren für xls)
OPTIMZE_HSSF_WB_FONTS.NAME=Optimiere Schriftarten (kann Dateigrösse reduzieren für xls)
EVALUATE_ALL_FORMULAS.NAME=Berechne all Formeln
EVALUATE_ALL_FORMULAS.NAME=Berechne alle Formeln
ENCRYPT_FILE.NAME=Schütze Datei (Liest die Ausgabedatei erneut ein und setzt ein Passwort)
PASSWORD.NAME=Passwort
5 changes: 3 additions & 2 deletions src/main/java/org/apache/poi/xssf/streaming/SXSSFSheet.java
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public SXSSFRow createRow(int rownum) {
if(_sh.getPhysicalNumberOfRows() > 0 && rownum <= _sh.getLastRowNum() ) {
throw new IllegalArgumentException(
"Attempting to write a row["+rownum+"] " +
"in the range [0," + _sh.getLastRowNum() + "] that is already written to disk.");
"in the range [0," + _sh.getLastRowNum() + "] that is already written to disk. Eventually already existing rows are ignored?");
}

SXSSFRow newRow = new SXSSFRow(this);
Expand Down Expand Up @@ -231,7 +231,8 @@ public int getFirstRowNum() {
*/
@Override
public int getLastRowNum() {
return _rows.isEmpty() ? -1 : _rows.lastKey();
// jlolling allow append
return _rows.isEmpty() ? _sh.getLastRowNum() : _rows.lastKey();
}

/**
Expand Down
146 changes: 146 additions & 0 deletions src/test/java/de/jlo/talendcomp/excel/TestExcelOutputStreaming.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package de.jlo.talendcomp.excel;

import java.util.HashMap;
import java.util.Map;

public class TestExcelOutputStreaming {

public static Map<String, Object> globalMap = new HashMap<>();

public static class row2Struct {

public String newColumn;

public String getNewColumn() {
return this.newColumn;
}

public String newColumn1;

public String getNewColumn1() {
return this.newColumn1;
}

public String newColumn2;

public String getNewColumn2() {
return this.newColumn2;
}

public String newColumn3;

public String getNewColumn3() {
return this.newColumn3;
}

}


public void testAppend() throws Exception {

de.jlo.talendcomp.excel.SpreadsheetFile tFileExcelWorkbookOpen_2 = new de.jlo.talendcomp.excel.SpreadsheetFile();
tFileExcelWorkbookOpen_2.setZipBombWarningThreshold(0.005d);
tFileExcelWorkbookOpen_2.setCreateStreamingXMLWorkbook(true);
try {
// read a excel file as template
// this file file will not used as output file
tFileExcelWorkbookOpen_2.setInputFile("/var/testdata/excel/test_streaming.xlsx",
true);
tFileExcelWorkbookOpen_2.initializeWorkbook();
} catch (Exception e) {
tFileExcelWorkbookOpen_2.error("Intialize workbook from file failed: " + e.getMessage(), e);
globalMap.put("tFileExcelWorkbookOpen_2_ERROR_MESSAGE", e.getMessage());
throw e;
}

globalMap.put("workbook_tFileExcelWorkbookOpen_2", tFileExcelWorkbookOpen_2.getWorkbook());
globalMap.put("tFileExcelWorkbookOpen_2_COUNT_SHEETS",
tFileExcelWorkbookOpen_2.getWorkbook().getNumberOfSheets());


final de.jlo.talendcomp.excel.SpreadsheetOutput tFileExcelSheetOutput_2 = new de.jlo.talendcomp.excel.SpreadsheetOutput();
tFileExcelSheetOutput_2.setWorkbook(
(org.apache.poi.ss.usermodel.Workbook) globalMap.get("workbook_tFileExcelWorkbookOpen_2"));
tFileExcelSheetOutput_2.setTargetSheetName("sheet");
globalMap.put("tFileExcelSheetOutput_2_SHEET_NAME", tFileExcelSheetOutput_2.getTargetSheetName());
tFileExcelSheetOutput_2.resetCache();
int startRowIndex_tFileExcelSheetOutput_2 = 5 - 1;
int currentSheetLastRowIndex_tFileExcelSheetOutput_2 = tFileExcelSheetOutput_2
.detectCurrentSheetLastNoneEmptyRowIndex() + 1;
tFileExcelSheetOutput_2.setRowStartIndex(
startRowIndex_tFileExcelSheetOutput_2 > currentSheetLastRowIndex_tFileExcelSheetOutput_2
? startRowIndex_tFileExcelSheetOutput_2
: currentSheetLastRowIndex_tFileExcelSheetOutput_2);
tFileExcelSheetOutput_2.setTemplateRowIndexForStyles(startRowIndex_tFileExcelSheetOutput_2);
tFileExcelSheetOutput_2.setFirstRowIsHeader(false);
// configure cell positions
tFileExcelSheetOutput_2.setColumnStart("C");
tFileExcelSheetOutput_2.setAppend(true);
tFileExcelSheetOutput_2.setReuseExistingStylesFromFirstWrittenRow(false);
tFileExcelSheetOutput_2.setSetupCellStylesForAllColumns(false);
tFileExcelSheetOutput_2.setReuseFirstRowHeight(false);
// configure cell formats
// columnIndex: 0, name: newColumn, format: , talendType: String
// columnIndex: 1, name: newColumn1, format: , talendType: String
// columnIndex: 2, name: newColumn2, format: , talendType: String
// columnIndex: 3, name: newColumn3, format: , talendType: String
tFileExcelSheetOutput_2.setWriteNullValues(false);
tFileExcelSheetOutput_2.setWriteZeroDateAsNull(true);
tFileExcelSheetOutput_2.setForbidWritingInProtectedCells(false);
// configure auto size columns and group rows by and comments
// config column 0
// config column 1
// config column 2
// config column 3
// row counter
int nb_line_tFileExcelSheetOutput_2 = 0;


row2Struct row2 = new row2Struct();
// fill schema data into the object array
Object[] dataset_tFileExcelSheetOutput_2 = new Object[4];
dataset_tFileExcelSheetOutput_2[0] = row2.newColumn;
dataset_tFileExcelSheetOutput_2[1] = row2.newColumn1;
dataset_tFileExcelSheetOutput_2[2] = row2.newColumn2;
dataset_tFileExcelSheetOutput_2[3] = row2.newColumn3;
// write dataset
try {
tFileExcelSheetOutput_2.writeRow(dataset_tFileExcelSheetOutput_2);
nb_line_tFileExcelSheetOutput_2++;
} catch (Exception e) {
tFileExcelSheetOutput_2.error("Write data row in line: " + nb_line_tFileExcelSheetOutput_2
+ " failed: " + e.getMessage(), e);
globalMap.put("tFileExcelSheetOutput_2_ERROR_MESSAGE", "Write data row in line: "
+ nb_line_tFileExcelSheetOutput_2 + " failed: " + e.getMessage());
throw e;
}

tFileExcelSheetOutput_2.setupColumnSize();
tFileExcelSheetOutput_2.closeLastGroup();
globalMap.put("tFileExcelSheetOutput_2_NB_LINE", nb_line_tFileExcelSheetOutput_2);
globalMap.put("tFileExcelSheetOutput_2_LAST_ROW_INDEX",
tFileExcelSheetOutput_2.detectCurrentSheetLastNoneEmptyRowIndex() + 1);

de.jlo.talendcomp.excel.SpreadsheetFile tFileExcelWorkbookSave_2 = new de.jlo.talendcomp.excel.SpreadsheetFile();
// set the workbook
tFileExcelWorkbookSave_2.setWorkbook(
(org.apache.poi.ss.usermodel.Workbook) globalMap.get("workbook_tFileExcelWorkbookOpen_2"));
// delete template sheets if needed
// persist workbook
try {
tFileExcelWorkbookSave_2.setOutputFile("/var/testdata/excel/test_streaming2");
tFileExcelWorkbookSave_2.createDirs();
globalMap.put("tFileExcelWorkbookSave_2_COUNT_SHEETS",
tFileExcelWorkbookSave_2.getWorkbook().getNumberOfSheets());
tFileExcelWorkbookSave_2.writeWorkbook();
// release the memory
globalMap.put("tFileExcelWorkbookSave_2_FILENAME", tFileExcelWorkbookSave_2.getOutputFile());
globalMap.remove("workbook_tFileExcelWorkbookOpen_2");
} catch (Exception e) {
globalMap.put("tFileExcelWorkbookSave_2_ERROR_MESSAGE", e.getMessage());
throw e;
}

}

}

0 comments on commit 84d7368

Please sign in to comment.