Skip to content

Commit

Permalink
Revived my test API for sending files through the ingest process with…
Browse files Browse the repository at this point in the history 
…out writing anything in the database.

During the 4.0 development I was using this API to test ingest plugins.
I want to use it agin, for batch-testing Oscar's new CSV ingest reader on the CSV files already ingested in production.
  • Loading branch information
@landreev
landreev committed Jun 29, 2017
1 parent 406906d commit 8651826
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 25 deletions.
28 changes: 9 additions & 19 deletions src/main/java/edu/harvard/iq/dataverse/api/TestIngest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
package edu.harvard.iq.dataverse.api;

import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DataFileServiceBean;
import edu.harvard.iq.dataverse.DataTable;
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.FileMetadata;
import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataFileReader;
import edu.harvard.iq.dataverse.ingest.tabulardata.TabularDataIngest;
import edu.harvard.iq.dataverse.util.FileUtil;
import edu.harvard.iq.dataverse.util.StringUtil;
import java.io.BufferedInputStream;
import java.util.logging.Logger;
import javax.ejb.EJB;
Expand All @@ -32,6 +32,7 @@
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.UriInfo;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.QueryParam;



Expand All @@ -56,49 +57,38 @@
public class TestIngest {
private static final Logger logger = Logger.getLogger(TestIngest.class.getCanonicalName());

@EJB
DataFileServiceBean dataFileService;
@EJB
DatasetServiceBean datasetService;
@EJB
IngestServiceBean ingestService;

//@EJB

@Path("test/{fileName}/{fileType}")
@Path("test/file")
@GET
@Produces({ "text/plain" })
public String datafile(@PathParam("fileName") String fileName, @PathParam("fileType") String fileType, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
public String datafile(@QueryParam("fileName") String fileName, @QueryParam("fileType") String fileType, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
String output = "";

if (fileName == null || fileType == null || "".equals(fileName) || "".equals(fileType)) {
output = output.concat("Usage: java edu.harvard.iq.dataverse.ingest.IngestServiceBean <file> <type>.");
if (StringUtil.isEmpty(fileName) || StringUtil.isEmpty(fileType)) {
output = output.concat("Usage: /api/ingest/test/file?fileName=PATH&fileType=TYPE");
return output;
}

BufferedInputStream fileInputStream = null;

String absoluteFilePath = null;
if (fileType.equals("x-stata")) {
absoluteFilePath = "/usr/share/data/retest_stata/reingest/" + fileName;
} else if (fileType.equals("x-spss-sav")) {
absoluteFilePath = "/usr/share/data/retest_sav/reingest/" + fileName;
} else if (fileType.equals("x-spss-por")) {
absoluteFilePath = "/usr/share/data/retest_por/reingest/" + fileName;
}

try {
fileInputStream = new BufferedInputStream(new FileInputStream(new File(absoluteFilePath)));
fileInputStream = new BufferedInputStream(new FileInputStream(new File(fileName)));
} catch (FileNotFoundException notfoundEx) {
fileInputStream = null;
}

if (fileInputStream == null) {
output = output.concat("Could not open file "+absoluteFilePath+".");
output = output.concat("Could not open file "+fileName+".");
return output;
}

fileType = "application/"+fileType;
TabularDataFileReader ingestPlugin = ingestService.getTabDataReaderByMimeType(fileType);

if (ingestPlugin == null) {
Expand All @@ -123,7 +113,7 @@ public String datafile(@PathParam("fileName") String fileName, @PathParam("fileT
&& tabFile != null
&& tabFile.exists()) {

String tabFilename = FileUtil.replaceExtension(absoluteFilePath, "tab");
String tabFilename = FileUtil.replaceExtension(fileName, "tab");

java.nio.file.Files.copy(Paths.get(tabFile.getAbsolutePath()), Paths.get(tabFilename), StandardCopyOption.REPLACE_EXISTING);

Expand Down
12 changes: 6 additions & 6 deletions src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,8 +544,8 @@ public void produceContinuousSummaryStatistics(DataFile dataFile, File generated
for (int i = 0; i < dataFile.getDataTable().getVarQuantity(); i++) {
if (dataFile.getDataTable().getDataVariables().get(i).isIntervalContinuous()) {
logger.fine("subsetting continuous vector");
DataFileIO dataFileIO = dataFile.getDataFileIO();
dataFileIO.open();
//DataFileIO dataFileIO = dataFile.getDataFileIO();
//dataFileIO.open();
if ("float".equals(dataFile.getDataTable().getDataVariables().get(i).getFormat())) {
Float[] variableVector = TabularSubsetGenerator.subsetFloatVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue());
logger.fine("Calculating summary statistics on a Float vector;");
Expand Down Expand Up @@ -576,8 +576,8 @@ public void produceDiscreteNumericSummaryStatistics(DataFile dataFile, File gene
if (dataFile.getDataTable().getDataVariables().get(i).isIntervalDiscrete()
&& dataFile.getDataTable().getDataVariables().get(i).isTypeNumeric()) {
logger.fine("subsetting discrete-numeric vector");
DataFileIO dataFileIO = dataFile.getDataFileIO();
dataFileIO.open();
//DataFileIO dataFileIO = dataFile.getDataFileIO();
//dataFileIO.open();
Long[] variableVector = TabularSubsetGenerator.subsetLongVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue());
// We are discussing calculating the same summary stats for
// all numerics (the same kind of sumstats that we've been calculating
Expand Down Expand Up @@ -610,8 +610,8 @@ public void produceCharacterSummaryStatistics(DataFile dataFile, File generatedT

for (int i = 0; i < dataFile.getDataTable().getVarQuantity(); i++) {
if (dataFile.getDataTable().getDataVariables().get(i).isTypeCharacter()) {
DataFileIO dataFileIO = dataFile.getDataFileIO();
dataFileIO.open();
//DataFileIO dataFileIO = dataFile.getDataFileIO();
//dataFileIO.open();
logger.fine("subsetting character vector");
String[] variableVector = TabularSubsetGenerator.subsetStringVector(new FileInputStream(generatedTabularFile), i, dataFile.getDataTable().getCaseQuantity().intValue());
//calculateCharacterSummaryStatistics(dataFile, i, variableVector);
Expand Down

0 comments on commit 8651826

Please sign in to comment.