From b828799a23edff62c9509e991a10ecac039b5ace Mon Sep 17 00:00:00 2001 From: Michael Bar-Sinai Date: Tue, 6 May 2014 13:36:10 -0400 Subject: [PATCH 01/50] Working on dataset json i/o --- .../edu/harvard/iq/dataverse/Dataset.java | 2 +- .../harvard/iq/dataverse/DatasetAuthor.java | 1 - .../iq/dataverse/DatasetFieldServiceBean.java | 19 +++++- .../iq/dataverse/DatasetFieldType.java | 9 ++- .../iq/dataverse/DatasetServiceBean.java | 6 -- .../harvard/iq/dataverse/api/Dataverses.java | 14 ++--- .../iq/dataverse/util/json/JsonParser.java | 59 +++++++++---------- 7 files changed, 57 insertions(+), 53 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index cc459605175..c075d21f198 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -101,7 +101,7 @@ public void setFiles(List files) { private List versions = new ArrayList(); public DatasetVersion getLatestVersion() { - return versions.get(0); + return getVersions().get(0); } public List getVersions() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java b/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java index 483e9ff5701..59e4549511f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java @@ -23,7 +23,6 @@ public int compare(DatasetAuthor o1, DatasetAuthor o2) { } }; - private DatasetVersion datasetVersion; public DatasetVersion getDatasetVersion() { return datasetVersion; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 195ca5c2e5b..ac8f98e9988 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -10,6 +10,7 @@ import javax.ejb.Stateless; import javax.inject.Named; import javax.persistence.EntityManager; +import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.persistence.Query; @@ -24,7 +25,7 @@ public class DatasetFieldServiceBean { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; - private static final String NAME_QUERY = "SELECT dsfType from DatasetFieldType dsfType where dsfType.name= :fieldName "; + private static final String NAME_QUERY = "SELECT dsfType from DatasetFieldType dsfType where dsfType.name= :fieldName"; private static final String FILEMETA_NAME_QUERY = "SELECT fmf from FileMetadataField fmf where fmf.name= :fieldName "; private static final String FILEMETA_NAME_FORMAT_QUERY = "SELECT fmf from FileMetadataField fmf where fmf.name= :fieldName and fmf.fileFormatName= :fileFormatName "; @@ -53,6 +54,22 @@ public DatasetFieldType findByName(String name) { return dsfType; } + /** + * Gets the dataset field type, or returns {@code null}. Does not throw exceptions. + * @param name the name do the field type + * @return the field type, or {@code null} + * @see #findByName(java.lang.String) + */ + public DatasetFieldType findByNameOpt( String name ) { + try { + return em.createNamedQuery("DatasetFieldType.findByName", DatasetFieldType.class) + .setParameter("name", name) + .getSingleResult(); + } catch ( NoResultException nre ) { + return null; + } + } + public ControlledVocabularyValue findControlledVocabularyValue(Object pk) { return (ControlledVocabularyValue) em.find(ControlledVocabularyValue.class, pk); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 39a29a2a871..0243fcc5111 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse; import java.util.Collection; @@ -19,6 +14,10 @@ * * @author Stephen Kraffmiller */ +@NamedQueries({ + @NamedQuery( name="DatasetFieldType.findByName", + query="SELECT dsfType FROM DatasetFieldType dsfType WHERE dsfType.name=:name") +}) @Entity public class DatasetFieldType implements Serializable, Comparable { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 423e8b8a116..ac843229ec3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -5,8 +5,6 @@ */ package edu.harvard.iq.dataverse; -import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.logging.Logger; import javax.ejb.EJB; @@ -15,7 +13,6 @@ import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.persistence.Query; -import org.apache.commons.lang.StringUtils; /** * @@ -31,9 +28,6 @@ public class DatasetServiceBean { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; - - - public Dataset find(Object pk) { return (Dataset) em.find(Dataset.class, pk); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 8684997b5d3..86837993673 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -41,11 +41,13 @@ import javax.json.stream.JsonParsingException; import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolationException; +import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.POST; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; @@ -58,7 +60,6 @@ public class Dataverses extends AbstractApiBean { private static final Logger logger = Logger.getLogger(Dataverses.class.getName()); - @GET public String list() { JsonArrayBuilder bld = Json.createArrayBuilder(); @@ -115,9 +116,8 @@ public String addDataverse( Dataverse d, @PathParam("identifier") String parentI } @POST - @Path("{identifier}/datasets/") - @Produces("application/json") - public Response createDataset( @PathParam("identifier") String parentIdtf, String jsonBody, @QueryParam("key") String apiKey ) { + @Path("{identifier}/datasets") + public Response createDataset( String jsonBody, @PathParam("identifier") String parentIdtf, @QueryParam("key") String apiKey ) { DataverseUser u = userSvc.findByUserName(apiKey); if ( u == null ) return errorResponse( Response.Status.UNAUTHORIZED, "Invalid apikey '" + apiKey + "'"); @@ -130,6 +130,7 @@ public Response createDataset( @PathParam("identifier") String parentIdtf, Strin try ( StringReader rdr = new StringReader(jsonBody) ) { json = Json.createReader(rdr).readObject(); } catch ( JsonParsingException jpe ) { + logger.log(Level.SEVERE, "Json: " + jsonBody); return errorResponse( Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage() ); } @@ -240,7 +241,7 @@ public String listMetadataBlocks( @PathParam("identifier") String dvIdtf, @Query @POST @Path("{identifier}/metadatablocks") - @Produces("application/json") + @Produces(MediaType.APPLICATION_JSON) public Response setMetadataBlocks( @PathParam("identifier")String dvIdtf, @QueryParam("key") String apiKey, String blockIds ) { DataverseUser u = userSvc.findByUserName(apiKey); if ( u == null ) return badApiKey(apiKey); @@ -449,9 +450,6 @@ public String deleteAssignment( @PathParam("id") long assignmentId, @PathParam(" } } - // CONTPOINT add a POST method for datasets here, and a POST method for dataset versions in the dataset part. - - @GET @Path(":gv") public String toGraphviz() { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index d686417ac3e..b43aec1f42f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -98,7 +98,6 @@ public DatasetVersion parseDatasetVersion( JsonObject obj ) throws JsonParseExce dsv.setDatasetDistributors(distros); } - return dsv; } catch (ParseException ex) { @@ -125,42 +124,40 @@ public List parseMetadataBlocks( JsonObject json ) throws JsonPars public DatasetField parseField( JsonObject json ) throws JsonParseException { if ( json == null ) return null; - try { - DatasetField ret = new DatasetField(); - DatasetFieldType type = datasetFieldSvc.findByName(json.getString("typeName","")); - - if ( type == null ) { - throw new NoResultException("Can't find type '" + json.getString("typeName","") +"'"); - } - ret.setDatasetFieldType(type); + + DatasetField ret = new DatasetField(); + DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName","")); - if ( type.isCompound() ) { - List vals = parseCompoundValue(type, json); - for ( DatasetFieldCompoundValue dsfcv : vals ) { - dsfcv.setParentDatasetField(ret); - } - ret.setDatasetFieldCompoundValues(vals); + if ( type == null ) { + throw new JsonParseException("Can't find type '" + json.getString("typeName","") +"'"); + } + + ret.setDatasetFieldType(type); - } else if ( type.isControlledVocabulary() ) { - List vals = parseControlledVocabularyValue(type, json); - for ( ControlledVocabularyValue cvv : vals ) { - cvv.setDatasetFieldType(type); - } - ret.setControlledVocabularyValues(vals); + if ( type.isCompound() ) { + List vals = parseCompoundValue(type, json); + for ( DatasetFieldCompoundValue dsfcv : vals ) { + dsfcv.setParentDatasetField(ret); + } + ret.setDatasetFieldCompoundValues(vals); - } else { - // primitive - List values = parsePrimitiveValue( json ); - for ( DatasetFieldValue val : values ) { - val.setDatasetField(ret); - } - ret.setDatasetFieldValues(values); + } else if ( type.isControlledVocabulary() ) { + List vals = parseControlledVocabularyValue(type, json); + for ( ControlledVocabularyValue cvv : vals ) { + cvv.setDatasetFieldType(type); } + ret.setControlledVocabularyValues(vals); - return ret; - } catch ( NoResultException nre ) { - throw new JsonParseException("Can't find field type named '" + json.getString("typeName","") + "'"); + } else { + // primitive + List values = parsePrimitiveValue( json ); + for ( DatasetFieldValue val : values ) { + val.setDatasetField(ret); + } + ret.setDatasetFieldValues(values); } + + return ret; } public List parseCompoundValue( DatasetFieldType compoundType, JsonObject json ) throws JsonParseException { From 73fae2cc9f0671684ae3ce7f3f4693b230cee629 Mon Sep 17 00:00:00 2001 From: mheppler Date: Tue, 6 May 2014 15:06:18 -0400 Subject: [PATCH 02/50] Updated link URL for Support, Resources, Software in the header. Commented out placeholder for Privacy link in the footer. --- src/main/webapp/dataverse_header.xhtml | 10 +++++----- src/main/webapp/dataverse_template.xhtml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/dataverse_header.xhtml b/src/main/webapp/dataverse_header.xhtml index 1262c913c3c..842d49ebbb1 100644 --- a/src/main/webapp/dataverse_header.xhtml +++ b/src/main/webapp/dataverse_header.xhtml @@ -38,17 +38,17 @@
  • - + About
  • - + Software
  • - + Resources
    • @@ -58,8 +58,8 @@
    • - - + +
    • diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index 124b72640da..d5d48a62143 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -33,7 +33,7 @@ Default Body From a07ec72927a7755a3ab66316eaff9fa54600b6fa Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 6 May 2014 15:06:50 -0400 Subject: [PATCH 03/50] Final mod for formatting of numeric values in CSV ingest. (we ain't doin' no formattin' is the final verdict). --- .../impl/plugins/csv/CSVFileReader.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java index 0399baddd4d..d03fe9bc298 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/tabulardata/impl/plugins/csv/CSVFileReader.java @@ -443,8 +443,19 @@ public int readFile(BufferedReader csvReader, DataTable dataTable, PrintWriter f } else if (valueTokens[i].equalsIgnoreCase("null")) { // By request from Gus - "NULL" is recognized as a // numeric zero: - caseRow[i] = "0"; + if (isIntegerVariable[i]) { + caseRow[i] = "0"; + } else { + caseRow[i] = "0.0"; + } } else { + /* No re-formatting is done on any other numeric values. + * We'll save them as they were, for archival purposes. + * The alternative solution - formatting in sci. notation + * is commented-out below. + */ + caseRow[i] = valueTokens[i]; + /* if (isIntegerVariable[i]) { try { Integer testIntegerValue = new Integer(valueTokens[i]); @@ -466,13 +477,11 @@ public int readFile(BufferedReader csvReader, DataTable dataTable, PrintWriter f // in a IEEE 754-like "scientific notation" - for ex., // 753.24 will be encoded as 7.5324e2 BigDecimal testBigDecimal = new BigDecimal(valueTokens[i], doubleMathContext); - /* // an experiment - what's gonna happen if we just // use the string representation of the bigdecimal object // above? - caseRow[i] = testBigDecimal.toString(); - */ - + //caseRow[i] = testBigDecimal.toString(); += caseRow[i] = String.format(FORMAT_IEEE754, testBigDecimal); // Strip meaningless zeros and extra + signs: @@ -486,6 +495,7 @@ public int readFile(BufferedReader csvReader, DataTable dataTable, PrintWriter f throw new IOException("Failed to parse a value recognized as numeric in the first pass! (?)"); } } + */ } } else if (isTimeVariable[i] || isDateVariable[i]) { // Time and Dates are stored NOT quoted (don't ask). From cc219c12eb07d5377edc886b5671f0fe2c6cbc6e Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 6 May 2014 15:25:07 -0400 Subject: [PATCH 04/50] added dataverse contact button --- src/main/webapp/dataverse.xhtml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index d475b6bccd9..8e0e9f769f8 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -282,6 +282,15 @@ + +
      + +
      +       + +
      From b38524516792ca1bb9d698316095c3a1e4772afc Mon Sep 17 00:00:00 2001 From: Gustavo Durand Date: Tue, 6 May 2014 15:38:53 -0400 Subject: [PATCH 05/50] added tooltips for datasetFieldTypes --- src/main/webapp/search/advanced.xhtml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/search/advanced.xhtml b/src/main/webapp/search/advanced.xhtml index c91b123aba9..dfd8cda1c04 100644 --- a/src/main/webapp/search/advanced.xhtml +++ b/src/main/webapp/search/advanced.xhtml @@ -61,14 +61,18 @@
      - +
      + itemLabel="#{cvs.strValue}" itemValue="#{cvs.strValue}"/>
      From 7ec4e3085f9ef1ed5f4f725df9b2fa9e6d6975e3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Tue, 6 May 2014 15:44:38 -0400 Subject: [PATCH 06/50] Changed index service bean to be more careful about not leaving the "index in progress" flag set on files which failed to ingest. (that's what was causing the endless spinner, "ingest in progress" not going away). --- .../dataverse/ingest/IngestServiceBean.java | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 3db2fad333d..4a14e8023ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -207,6 +207,14 @@ public boolean ingestAsTabular(DataFile dataFile) throws IOException { public boolean ingestAsTabular(String tempFileLocation, DataFile dataFile) throws IOException { boolean ingestSuccessful = false; + PushContext pushContext = PushContextFactory.getDefault().getPushContext(); + if (pushContext != null) { + Logger.getLogger(DatasetPage.class.getName()).log(Level.FINE, "Ingest: Obtained push context " + + pushContext.toString()); + } else { + Logger.getLogger(DatasetPage.class.getName()).log(Level.SEVERE, "Warning! Could not obtain push context."); + } + // Locate ingest plugin for the file format by looking // it up with the Ingest Service Provider Registry: //TabularDataFileReader ingestPlugin = IngestSP.getTabDataReaderByMIMEType(dFile.getContentType()); @@ -215,6 +223,11 @@ public boolean ingestAsTabular(String tempFileLocation, DataFile dataFile) throw TabularDataFileReader ingestPlugin = getTabDataReaderByMimeType(dataFile); if (ingestPlugin == null) { + dataFile.SetIngestProblem(); + dataFile = fileService.save(dataFile); + FacesMessage facesMessage = new FacesMessage("ingest failed"); + pushContext.push("/ingest"+dataFile.getOwner().getId(), facesMessage); + Logger.getLogger(DatasetPage.class.getName()).log(Level.INFO, "Ingest failure: Sent push notification to the page."); throw new IOException("Could not find ingest plugin for the file " + fileName); } @@ -223,6 +236,11 @@ public boolean ingestAsTabular(String tempFileLocation, DataFile dataFile) throw try { tempFileInputStream = new FileInputStream(new File(tempFileLocation)); } catch (FileNotFoundException notfoundEx) { + dataFile.SetIngestProblem(); + dataFile = fileService.save(dataFile); + FacesMessage facesMessage = new FacesMessage("ingest failed"); + pushContext.push("/ingest"+dataFile.getOwner().getId(), facesMessage); + Logger.getLogger(DatasetPage.class.getName()).log(Level.INFO, "Ingest failure: Sent push notification to the page."); throw new IOException("Could not open temp file "+tempFileLocation); } @@ -266,27 +284,21 @@ public boolean ingestAsTabular(String tempFileLocation, DataFile dataFile) throw try { produceSummaryStatistics(dataFile); } catch (IOException sumStatEx) { + dataFile.SetIngestProblem(); + dataFile = fileService.save(dataFile); + FacesMessage facesMessage = new FacesMessage("ingest failed"); + pushContext.push("/ingest"+dataFile.getOwner().getId(), facesMessage); + Logger.getLogger(DatasetPage.class.getName()).log(Level.INFO, "Ingest failure: Sent push notification to the page."); throw new IOException ("Ingest: failed to calculate summary statistics. "+sumStatEx.getMessage()); } - ingestSuccessful = true; - PushContext pushContext = PushContextFactory.getDefault().getPushContext(); - if (pushContext != null ) { - Logger.getLogger(DatasetPage.class.getName()).log(Level.FINE, "Ingest: Obtained push context " - + pushContext.toString()); - } else { - Logger.getLogger(DatasetPage.class.getName()).log(Level.SEVERE, "Warning! Could not obtain push context."); - } - - - - - FacesMessage facesMessage = new FacesMessage("ingest done"); - pushContext.push("/ingest"+dataFile.getOwner().getId(), facesMessage); - Logger.getLogger(DatasetPage.class.getName()).log(Level.INFO, "Ingest: Sent push notification to the page."); - + ingestSuccessful = true; } } + + FacesMessage facesMessage = new FacesMessage("ingest done"); + pushContext.push("/ingest"+dataFile.getOwner().getId(), facesMessage); + Logger.getLogger(DatasetPage.class.getName()).log(Level.INFO, "Ingest: Sent push notification to the page."); return ingestSuccessful; } From 6126ed536da48f9b8af7f461f68871975cbad9e4 Mon Sep 17 00:00:00 2001 From: Raman Prasad Date: Tue, 6 May 2014 16:34:34 -0400 Subject: [PATCH 07/50] Fix tree not open after edit metadata or edit files is cancelled. Still needs fix for edit dataverse -> cancel --- src/main/webapp/dataverse_template.xhtml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index d5d48a62143..626080b8755 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -50,6 +50,11 @@ -