From 8165481b24565ad3df12739ed67063977011630c Mon Sep 17 00:00:00 2001
From: bencomp
Date: Fri, 8 Apr 2016 18:05:18 +0200
Subject: [PATCH 01/37] Correct pom.xml indentation
---
pom.xml | 285 ++++++++++++++++++++++++++++----------------------------
1 file changed, 143 insertions(+), 142 deletions(-)
diff --git a/pom.xml b/pom.xml
index e15f5a87529..c8c46e87396 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,30 +16,30 @@
- prime-repo
- PrimeFaces Maven Repository
- http://repository.primefaces.org
- default
+ prime-repo
+ PrimeFaces Maven Repository
+ http://repository.primefaces.org
+ default
-
-
- geotk-repo
- Geo Toolkit Maven Repository
- http://maven.geotoolkit.org
- default
-
-
- central-repo
- Central Repository
- http://repo1.maven.org/maven2
- default
-
-
+
+
+ geotk-repo
+ Geo Toolkit Maven Repository
+ http://maven.geotoolkit.org
+ default
+
+
+ central-repo
+ Central Repository
+ http://repo1.maven.org/maven2
+ default
+
+
dvn.private
Local repository for hosting jars not available from network repositories.
file://${project.basedir}/local_lib
-
+
dataone.org
http://dev-testing.dataone.org/maven
@@ -51,9 +51,9 @@
-
+
-
+
junit
junit
4.8.1
@@ -117,7 +117,8 @@
gson
2.2.4
compile
-
+
+
xom
xom
@@ -158,20 +159,20 @@
1.0.10
- org.atmosphere
- atmosphere-runtime
- 2.4.2
-
+ org.atmosphere
+ atmosphere-runtime
+ 2.4.2
+
org.omnifaces
omnifaces
1.7
-
-
- org.hibernate
- hibernate-validator
- 5.0.3.Final
-
+
+
+ org.hibernate
+ hibernate-validator
+ 5.0.3.Final
+
commons-lang
commons-lang
@@ -188,103 +189,103 @@
commons-logging
1.1.3
-
- org.apache.commons
- commons-math
- 2.2
-
-
- commons-validator
- commons-validator
- 1.4.0
-
-
- colt
- colt
- 1.2.0
-
-
-
- nom.tam.fits
- fits
- 2012-10-25-generated
-
-
+
+ org.apache.commons
+ commons-math
+ 2.2
+
+
+ commons-validator
+ commons-validator
+ 1.4.0
+
+
+ colt
+ colt
+ 1.2.0
+
+
+
+ nom.tam.fits
+ fits
+ 2012-10-25-generated
+
+
net.handle
handle
2006-06-16-generated
-
-
- edu.harvard.iq.dvn
- unf5
- 5.0
-
-
-
- org.dataverse
- unf
- 6.0
-
-
-
-
- org.nuiton.thirdparty
- REngine
- 0.6-1
-
-
- org.nuiton.thirdparty
- Rserve
- 0.6-1
-
-
-
- org.apache.poi
- poi
- 3.10-FINAL
-
-
- org.apache.poi
- poi-ooxml
- 3.10-FINAL
-
-
- org.apache.poi
- poi-examples
- 3.10-FINAL
-
-
- edu.harvard.hul.ois.jhove
- jhove
- 1.11.0
-
-
- edu.harvard.hul.ois.jhove
- jhove-module
- 1.11.0
-
-
- edu.harvard.hul.ois.jhove
- jhove-handler
- 1.11.0
-
-
-
- javax.media
- jai_imageio
- 1.1.1
-
-
- javax.media
- jai_core
- 1.1.3
-
-
- javax.media
- jai_codec
- 1.1.3
-
+
+
+ edu.harvard.iq.dvn
+ unf5
+ 5.0
+
+
+
+ org.dataverse
+ unf
+ 6.0
+
+
+
+
+ org.nuiton.thirdparty
+ REngine
+ 0.6-1
+
+
+ org.nuiton.thirdparty
+ Rserve
+ 0.6-1
+
+
+
+ org.apache.poi
+ poi
+ 3.10-FINAL
+
+
+ org.apache.poi
+ poi-ooxml
+ 3.10-FINAL
+
+
+ org.apache.poi
+ poi-examples
+ 3.10-FINAL
+
+
+ edu.harvard.hul.ois.jhove
+ jhove
+ 1.11.0
+
+
+ edu.harvard.hul.ois.jhove
+ jhove-module
+ 1.11.0
+
+
+ edu.harvard.hul.ois.jhove
+ jhove-handler
+ 1.11.0
+
+
+
+ javax.media
+ jai_imageio
+ 1.1.1
+
+
+ javax.media
+ jai_core
+ 1.1.3
+
+
+ javax.media
+ jai_codec
+ 1.1.3
+
org.ocpsoft.rewrite
rewrite-servlet
@@ -295,8 +296,8 @@
rewrite-config-prettyfaces
2.0.12.Final
-
- edu.ucsb.nceas
+
+ edu.ucsb.nceas
ezid
1.0.0
jar
@@ -368,7 +369,7 @@
log4j
1.2.17
-
+
@@ -376,8 +377,8 @@
src/main/java
- *.properties
- **/mime.types
+ *.properties
+ **/mime.types
**/*.R
@@ -408,12 +409,12 @@
maven-jar-plugin
2.3
-
-
- true
- true
-
-
+
+
+ true
+ true
+
+
@@ -422,12 +423,12 @@
2.3
false
-
-
- true
- true
-
-
+
+
+ true
+ true
+
+
From 2b7232d72e5221514c5e4455922d81dd4becf6ca Mon Sep 17 00:00:00 2001
From: bmckinney
Date: Mon, 11 Apr 2016 12:57:13 -0400
Subject: [PATCH 02/37] adds "Laboratory" and "Research Group" to list of
Dataverse categories
---
src/main/java/Bundle.properties | 2 ++
src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 8 ++++++--
src/main/webapp/dataverse.xhtml | 2 ++
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties
index 5a6d571d06e..977f4c0ee9d 100755
--- a/src/main/java/Bundle.properties
+++ b/src/main/java/Bundle.properties
@@ -270,6 +270,8 @@ dataverse.type.selectTab.journals=Journal
dataverse.type.selectTab.organizationsAndInsitutions=Organization or Institution
dataverse.type.selectTab.teachingCourses=Teaching Course
dataverse.type.selectTab.uncategorized=Uncategorized
+dataverse.type.selectTab.researchGroup=Research Group
+dataverse.type.selectTab.laboratory=Laboratory
dataverse.description.title=A summary describing the purpose, nature, or scope of this dataverse.
dataverse.email=Email
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index b97d3402f81..44ab7c59f7e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -56,7 +56,7 @@
public class Dataverse extends DvObjectContainer {
public enum DataverseType {
- RESEARCHERS, RESEARCH_PROJECTS, JOURNALS, ORGANIZATIONS_INSTITUTIONS, TEACHING_COURSES, UNCATEGORIZED
+ RESEARCHERS, RESEARCH_PROJECTS, JOURNALS, ORGANIZATIONS_INSTITUTIONS, TEACHING_COURSES, UNCATEGORIZED, LABORATORY, RESEARCH_GROUP
};
private static final long serialVersionUID = 1L;
@@ -115,7 +115,11 @@ public String getFriendlyCategoryName(){
case ORGANIZATIONS_INSTITUTIONS:
return "Organization or Institution";
case TEACHING_COURSES:
- return "Teaching Course";
+ return "Teaching Course";
+ case LABORATORY:
+ return "Laboratory";
+ case RESEARCH_GROUP:
+ return "Research Group";
case UNCATEGORIZED:
return uncategorizedString;
default:
diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml
index 2992ca8c376..8a1100366be 100644
--- a/src/main/webapp/dataverse.xhtml
+++ b/src/main/webapp/dataverse.xhtml
@@ -122,6 +122,8 @@
+
+
From 6f0583f8fe2e24ea784785c15ea535605ebaee4d Mon Sep 17 00:00:00 2001
From: Eleni Castro
Date: Thu, 14 Apr 2016 21:57:32 -0400
Subject: [PATCH 03/37] update tsv files #3073
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Change from “showabovefold” to more accurate “displayoncreate” #3073
---
scripts/api/data/metadatablocks/astrophysics.tsv | 2 +-
scripts/api/data/metadatablocks/biomedical.tsv | 2 +-
scripts/api/data/metadatablocks/citation.tsv | 2 +-
scripts/api/data/metadatablocks/customARCS.tsv | 2 +-
scripts/api/data/metadatablocks/customCHIA.tsv | 2 +-
scripts/api/data/metadatablocks/customDigaai.tsv | 2 +-
scripts/api/data/metadatablocks/customGSD.tsv | 2 +-
scripts/api/data/metadatablocks/customMRA.tsv | 2 +-
scripts/api/data/metadatablocks/customPSI.tsv | 2 +-
scripts/api/data/metadatablocks/customPSRI.tsv | 2 +-
scripts/api/data/metadatablocks/custom_hbgdki.tsv | 2 +-
scripts/api/data/metadatablocks/geospatial.tsv | 2 +-
scripts/api/data/metadatablocks/journals.tsv | 2 +-
scripts/api/data/metadatablocks/social_science.tsv | 2 +-
14 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/scripts/api/data/metadatablocks/astrophysics.tsv b/scripts/api/data/metadatablocks/astrophysics.tsv
index 755cc392147..d6266d239b2 100644
--- a/scripts/api/data/metadatablocks/astrophysics.tsv
+++ b/scripts/api/data/metadatablocks/astrophysics.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
astrophysics Astronomy and Astrophysics Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
astroType Type The nature or genre of the content of the files in the dataset. text 0 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics
astroFacility Facility The observatory or facility where the data was obtained. text 1 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics
astroInstrument Instrument The instrument used to collect the data. text 2 TRUE TRUE TRUE TRUE FALSE FALSE astrophysics
diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv
index db24a804443..f45c5849845 100644
--- a/scripts/api/data/metadatablocks/biomedical.tsv
+++ b/scripts/api/data/metadatablocks/biomedical.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
biomedical Life Sciences Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
studyDesignType Design Type Design types that are based on the overall experimental design. text 0 TRUE TRUE TRUE TRUE FALSE FALSE biomedical
studyFactorType Factor Type Factors used in the Dataset. text 1 TRUE TRUE TRUE TRUE FALSE FALSE biomedical
studyAssayOrganism Organism The taxonomic name of the organism used in the Dataset or from which the starting biological material derives. text 2 TRUE TRUE TRUE TRUE FALSE FALSE biomedical
diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv
index 322cd558bfe..7ea1330c8cb 100644
--- a/scripts/api/data/metadatablocks/citation.tsv
+++ b/scripts/api/data/metadatablocks/citation.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
citation Citation Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
title Title Full title by which the Dataset is known. Enter title... text 0 TRUE FALSE FALSE FALSE TRUE TRUE citation
subtitle Subtitle A secondary title used to amplify or state certain limitations on the main title. text 1 FALSE FALSE FALSE FALSE FALSE FALSE citation
alternativeTitle Alternative Title A title by which the work is commonly referred, or an abbreviation of the title. text 2 FALSE FALSE FALSE FALSE FALSE FALSE citation
diff --git a/scripts/api/data/metadatablocks/customARCS.tsv b/scripts/api/data/metadatablocks/customARCS.tsv
index f74c1073694..e287349b830 100644
--- a/scripts/api/data/metadatablocks/customARCS.tsv
+++ b/scripts/api/data/metadatablocks/customARCS.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customARCS Alliance for Research on Corporate Sustainability Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
ARCS1 1) Were any of these data sets a) purchased, b) obtained through licensed databases, or c) provided by an organization under a nondisclosure or other agreement? Licensed agreement of deposited data. text 0 FALSE TRUE FALSE FALSE FALSE FALSE customARCS
ARCS2 2) If you responded Yes to Q1, have you ensured that sharing the data does not violate terms of the agreement? If you responded No to Q1, please enter N/A here. Data sharing does not violate terms. text 1 FALSE TRUE FALSE FALSE FALSE FALSE customARCS
ARCS3 3) Do any of these data sets include individual-level data (either collected or pre-existing in the dataset) that might make them subject to U.S. or international human subjects considerations? Human subjects consideration. text 2 FALSE TRUE FALSE FALSE FALSE FALSE customARCS
diff --git a/scripts/api/data/metadatablocks/customCHIA.tsv b/scripts/api/data/metadatablocks/customCHIA.tsv
index c2d52cf164f..255981c5418 100644
--- a/scripts/api/data/metadatablocks/customCHIA.tsv
+++ b/scripts/api/data/metadatablocks/customCHIA.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customCHIA CHIA Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
sourceCHIA Source Source - This describes the source of the data. Is it from the Bureau of Labor and Statistics? Is it data from the United Nations? text 0 TRUE FALSE FALSE TRUE FALSE FALSE customCHIA
datesAdditionalInformationCHIA Dates - Additional Information Dates - Additional Information - Note any additional information about dates or time periods in the dataset including intervals (annual, decennial, centennial, etc.) Also note the column(s) in the dataset where dates and other temporal information can be found. text 1 TRUE FALSE FALSE FALSE FALSE FALSE customCHIA
variablesCHIA Variables Variables - Define the variables in this dataset. Please note the column in the dataset where variable information can be found. textbox 2 TRUE FALSE FALSE FALSE FALSE FALSE customCHIA
diff --git a/scripts/api/data/metadatablocks/customDigaai.tsv b/scripts/api/data/metadatablocks/customDigaai.tsv
index e419e93747e..fac077e201c 100644
--- a/scripts/api/data/metadatablocks/customDigaai.tsv
+++ b/scripts/api/data/metadatablocks/customDigaai.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customDigaai Digaai Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
titulo Título Título do jornal ou revista. text 0 TRUE TRUE TRUE TRUE FALSE FALSE customDigaai
numero Número Número do jornal ou revista. text 1 TRUE FALSE FALSE TRUE FALSE FALSE customDigaai
datadePublicao Data de Publicação Entrar dia/mes/ano. dia/mes/ano text 2 TRUE FALSE FALSE TRUE FALSE FALSE customDigaai
diff --git a/scripts/api/data/metadatablocks/customGSD.tsv b/scripts/api/data/metadatablocks/customGSD.tsv
index 9a84a70cc7e..d15a4e89748 100644
--- a/scripts/api/data/metadatablocks/customGSD.tsv
+++ b/scripts/api/data/metadatablocks/customGSD.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customGSD Graduate School of Design Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
gsdStudentName Student Name Full name of the student: Last Name, First Name (example: Smith, Jane). Use the name that the GSD Administrator has on file. LastName, FirstName text 0 TRUE FALSE TRUE FALSE FALSE FALSE customGSD
gsdStudentProgram Student's Program of Study Student's program of study. text 1 TRUE TRUE TRUE TRUE FALSE FALSE customGSD
gsdCourseName Course Name Name of the course. text 2 TRUE TRUE FALSE TRUE FALSE FALSE customGSD
diff --git a/scripts/api/data/metadatablocks/customMRA.tsv b/scripts/api/data/metadatablocks/customMRA.tsv
index 5f8c1d07466..ea915575c21 100644
--- a/scripts/api/data/metadatablocks/customMRA.tsv
+++ b/scripts/api/data/metadatablocks/customMRA.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customMRA MRA Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
mraCollection Murray Research Archive Collection Browse the Murray Research Archive collection with the following terms. text 0 FALSE TRUE TRUE TRUE FALSE FALSE customMRA
#controlledVocabulary DatasetField Value identifier displayOrder
mraCollection Diversity samples: Race, Ethnicity, Sexual Orientation, Religion MRA0 0
diff --git a/scripts/api/data/metadatablocks/customPSI.tsv b/scripts/api/data/metadatablocks/customPSI.tsv
index 0d41ea7bf0a..b5103df850b 100644
--- a/scripts/api/data/metadatablocks/customPSI.tsv
+++ b/scripts/api/data/metadatablocks/customPSI.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customPSI PSI Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
psiBehavior Behavior Behavior text 0 TRUE TRUE TRUE TRUE FALSE FALSE customPSI
psiDonor Donor Donor text 1 TRUE TRUE TRUE TRUE FALSE FALSE customPSI
psiHealthArea Health Area Health Area text 2 TRUE TRUE TRUE TRUE FALSE FALSE customPSI
diff --git a/scripts/api/data/metadatablocks/customPSRI.tsv b/scripts/api/data/metadatablocks/customPSRI.tsv
index 64f2e667223..94936872adb 100644
--- a/scripts/api/data/metadatablocks/customPSRI.tsv
+++ b/scripts/api/data/metadatablocks/customPSRI.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
customPSRI Political Science Replication Initiative Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
PSRI1 Are the original data publicly available? Select from the list of options. text 0 FALSE TRUE FALSE FALSE FALSE FALSE customPSRI
PSRI2 Is the original code available? Select from the list of options. text 1 FALSE TRUE FALSE FALSE FALSE FALSE customPSRI
PSRI3 Where are the original data archived (name and url)? Answer if the data are publicly available. text 2 FALSE FALSE FALSE FALSE FALSE FALSE customPSRI
diff --git a/scripts/api/data/metadatablocks/custom_hbgdki.tsv b/scripts/api/data/metadatablocks/custom_hbgdki.tsv
index 577b5d90652..bbb098d7689 100644
--- a/scripts/api/data/metadatablocks/custom_hbgdki.tsv
+++ b/scripts/api/data/metadatablocks/custom_hbgdki.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
custom_hbgdki HBGDki HBGDki Custom Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
hbgdkiStudyName Name of Study Name of the study. Limit to 20 characters. text 0 TRUE FALSE FALSE FALSE TRUE FALSE custom_hbgdki
hbgdkiStudyRegistry Study Registry Which study registry was used? none 1 FALSE FALSE TRUE FALSE TRUE FALSE custom_hbgdki
hbgdkiStudyRegistryType ID Type Which study registry was used? text 2 TRUE TRUE FALSE FALSE TRUE FALSE hbgdkiStudyRegistry custom_hbgdki
diff --git a/scripts/api/data/metadatablocks/geospatial.tsv b/scripts/api/data/metadatablocks/geospatial.tsv
index a3c56130a6d..7464d51dc94 100644
--- a/scripts/api/data/metadatablocks/geospatial.tsv
+++ b/scripts/api/data/metadatablocks/geospatial.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
geospatial Geospatial Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
geographicCoverage Geographic Coverage Information on the geographic coverage of the data. Includes the total geographic scope of the data. none 0 FALSE FALSE TRUE FALSE FALSE FALSE geospatial
country Country / Nation The country or nation that the Dataset is about. text 1 TRUE TRUE FALSE TRUE FALSE FALSE geographicCoverage geospatial
state State / Province The state or province that the Dataset is about. Use GeoNames for correct spelling and avoid abbreviations. text 2 TRUE FALSE FALSE TRUE FALSE FALSE geographicCoverage geospatial
diff --git a/scripts/api/data/metadatablocks/journals.tsv b/scripts/api/data/metadatablocks/journals.tsv
index ab56655803f..097f0293ba8 100644
--- a/scripts/api/data/metadatablocks/journals.tsv
+++ b/scripts/api/data/metadatablocks/journals.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
journal Journal Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
journalVolumeIssue Journal Indicates the volume, issue and date of a journal, which this Dataset is associated with. none 0 FALSE FALSE TRUE FALSE FALSE FALSE journal
journalVolume Volume The journal volume which this Dataset is associated with (e.g., Volume 4). text 1 TRUE FALSE FALSE TRUE FALSE FALSE journalVolumeIssue journal
journalIssue Issue The journal issue number which this Dataset is associated with (e.g., Number 2, Autumn). text 2 TRUE FALSE FALSE TRUE FALSE FALSE journalVolumeIssue journal
diff --git a/scripts/api/data/metadatablocks/social_science.tsv b/scripts/api/data/metadatablocks/social_science.tsv
index 29467751e2e..b9fec245a1f 100644
--- a/scripts/api/data/metadatablocks/social_science.tsv
+++ b/scripts/api/data/metadatablocks/social_science.tsv
@@ -1,6 +1,6 @@
#metadataBlock name dataverseAlias displayName
socialscience Social Science and Humanities Metadata
-#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable showabovefold required parent metadatablock_id
+#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id
unitOfAnalysis Unit of Analysis Basic unit of analysis or observation that this Dataset describes, such as individuals, families/households, groups, institutions/organizations, administrative units, and more. For information about the DDI's controlled vocabulary for this element, please refer to the DDI web page at http://www.ddialliance.org/Specification/DDI-CV/. textbox 0 TRUE FALSE TRUE TRUE FALSE FALSE socialscience
universe Universe Description of the population covered by the data in the file; the group of people or other elements that are the object of the study and to which the study results refer. Age, nationality, and residence commonly help to delineate a given universe, but any number of other factors may be used, such as age limits, sex, marital status, race, ethnic group, nationality, income, veteran status, criminal convictions, and more. The universe may consist of elements other than persons, such as housing units, court cases, deaths, countries, and so on. In general, it should be possible to tell from the description of the universe whether a given individual or element is a member of the population under study. Also known as the universe of interest, population of interest, and target population. textbox 1 TRUE FALSE TRUE TRUE FALSE FALSE socialscience
timeMethod Time Method The time method or time dimension of the data collection, such as panel, cross-sectional, trend, time- series, or other. text 2 TRUE FALSE FALSE TRUE FALSE FALSE socialscience
From cda1b541d75663092b4f6ac7f6ce1d4f9081799f Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Mon, 18 Apr 2016 01:23:20 -0400
Subject: [PATCH 04/37] First push into the Harvesting branch. This is a
rudimentary framework for the harvesting client (incomplete!)
---
pom.xml | 12 +-
.../iq/dataverse/DataverseServiceBean.java | 85 +++
.../dataverse/HarvestingDataverseConfig.java | 127 ++++
.../harvest/client/FastGetRecord.java | 520 ++++++++++++++
.../harvest/client/HarvestTimerInfo.java | 97 +++
.../harvest/client/HarvesterServiceBean.java | 671 ++++++++++++++++++
.../timer/DataverseTimerServiceBean.java | 226 ++++++
7 files changed, 1737 insertions(+), 1 deletion(-)
create mode 100644 src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
create mode 100644 src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestTimerInfo.java
create mode 100644 src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
create mode 100644 src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
diff --git a/pom.xml b/pom.xml
index e15f5a87529..2fbef369667 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,7 +39,7 @@
Local repository for hosting jars not available from network repositories.
file://${project.basedir}/local_lib
-
+
dataone.org
http://dev-testing.dataone.org/maven
@@ -49,6 +49,7 @@
true
+
@@ -368,6 +369,15 @@
log4j
1.2.17
+
+
+
+
+
+ org.dspace
+ oclc-harvester2
+ 0.1.12
+
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index b5f5d78f716..ee08517fc96 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -30,6 +30,8 @@
import java.util.jar.Manifest;
import javax.ejb.EJB;
import javax.ejb.Stateless;
+import javax.ejb.TransactionAttribute;
+import javax.ejb.TransactionAttributeType;
import javax.inject.Inject;
import javax.inject.Named;
import javax.persistence.EntityManager;
@@ -450,6 +452,10 @@ public Map getAllHarvestedDataverseDescriptions(){
return ret;
}
+
+ public List getAllHarvestedDataverses() {
+ return em.createQuery("SELECT object(d) FROM Dataverse d, harvestingDataverseConfig c AS d WHERE c.dataverse_id IS NOT null AND c.dataverse_id=d.id order by d.id").getResultList();
+ }
public void populateDvSearchCard(SolrSearchResult solrSearchResult) {
@@ -499,4 +505,83 @@ public void populateDvSearchCard(SolrSearchResult solrSearchResult) {
}
}
}
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestResult(Long hdId, String result) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ hd.getHarvestingDataverseConfig().setHarvestResult(result);
+ }
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void resetHarvestingStatus(Long hdId) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ hd.getHarvestingDataverseConfig().setHarvestingNow(false);
+ }
+
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestInProgress(Long hdId, boolean inProgress) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ hd.getHarvestingDataverseConfig().setHarvestingNow(inProgress);
+ }
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setLastHarvestTime(Long hdId, Date lastHarvestTime) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ hd.getHarvestingDataverseConfig().setLastHarvestTime(lastHarvestTime);
+ }
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestSuccess(Long hdId, Date currentTime, int harvestedCount, int failedCount) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ /* TODO:
+ hd.getHarvestingDataverseConfig().setLastSuccessfulHarvestTime(currentTime);
+ hd.getHarvestingDataverseConfig().setHarvestedStudyCount(new Long(harvestedCount));
+ hd.getHarvestingDataverseConfig().setFailedStudyCount(new Long(failedCount));
+ */
+ hd.getHarvestingDataverseConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_SUCCESS);
+ }
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestSuccessNotEmpty(Long hdId, Date currentTime, int harvestedCount, int failedCount) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ /* TODO:
+ hd.getHarvestingDataverseConfig().setLastSuccessfulNonZeroHarvestTime(currentTime);
+ hd.getHarvestingDataverseConfig().setHarvestedStudyCountNonZero(new Long(harvestedCount));
+ hd.getHarvestingDataverseConfig().setFailedStudyCountNonZero(new Long(failedCount));
+ */
+ }
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestFailure(Long hdId, int harvestedStudyCount, int failedCount) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ /* TODO:
+ hd.getHarvestingDataverseConfig().setHarvestedStudyCount(new Long(harvestedStudyCount));
+ hd.getHarvestingDataverseConfig().setFailedStudyCount(new Long(failedCount));
+ */
+ hd.getHarvestingDataverseConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_FAILED);
+ }
+
+ }
+
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
index 28df6e19e65..6ded994902d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
@@ -6,6 +6,10 @@
package edu.harvard.iq.dataverse;
import java.io.Serializable;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
@@ -16,6 +20,8 @@
import javax.persistence.JoinColumn;
import javax.persistence.OneToOne;
import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
/**
*
@@ -56,6 +62,9 @@ public void setId(Long id) {
public static final String REMOTE_ARCHIVE_URL_LEVEL_DATASET="dataset";
public static final String REMOTE_ARCHIVE_URL_LEVEL_FILE="file";
+ public static final String SCHEDULE_PERIOD_DAILY="daily";
+ public static final String SCHEDULE_PERIOD_WEEKLY="weekly";
+
public HarvestingDataverseConfig() {
this.harvestType = HARVEST_TYPE_OAI; // default harvestType
this.harvestStyle = HARVEST_STYLE_DATAVERSE; // default harvestStyle
@@ -84,6 +93,10 @@ public void setHarvestType(String harvestType) {
this.harvestType = harvestType;
}
+ public boolean isOai() {
+ return HARVEST_TYPE_OAI.equals(harvestType);
+ }
+
String harvestStyle;
public String getHarvestStyle() {
@@ -134,10 +147,124 @@ public String getHarvestingSet() {
public void setHarvestingSet(String harvestingSet) {
this.harvestingSet = harvestingSet;
}
+
+ private String metadataPrefix;
+
+ public String getMetadataPrefix() {
+ return metadataPrefix;
+ }
+
+ public void setMetadataPrefix(String metadataPrefix) {
+ this.metadataPrefix = metadataPrefix;
+ }
+
+ private String harvestResult;
+
+ public String getHarvestResult() {
+ return harvestResult;
+ }
+
+ public void setHarvestResult(String harvestResult) {
+ this.harvestResult = harvestResult;
+ }
+
+ @Temporal(value = TemporalType.TIMESTAMP)
+ private Date lastHarvestTime;
+
+ public Date getLastHarvestTime() {
+ return lastHarvestTime;
+ }
+
+ public void setLastHarvestTime(Date lastHarvestTime) {
+ this.lastHarvestTime = lastHarvestTime;
+ }
+
+ @Temporal(value = TemporalType.TIMESTAMP)
+ private Date lastSuccessfulHarvestTime;
+
+ public Date getLastSuccessfulHarvestTime() {
+ return lastSuccessfulHarvestTime;
+ }
+
+ public void setLastSuccessfulHarvestTime(Date lastSuccessfulHarvestTime) {
+ this.lastSuccessfulHarvestTime = lastSuccessfulHarvestTime;
+ }
+
+ private Long harvestedDatasetCount;
+ private Long failedDatasetCount;
+ @Temporal(value = TemporalType.TIMESTAMP)
+ private Date lastSuccessfulNonEmptyHarvestTime;
+ private Long lastNonEmptyHarvestedDatasetCount;
+ private Long lastNonEmptyFailedDatasetCount;
+
+ private boolean scheduled;
+
+ public boolean isScheduled() {
+ return this.scheduled;
+ }
+
+ public void setScheduled(boolean scheduled) {
+ this.scheduled = scheduled;
+ }
+
+ private String schedulePeriod;
+
+ public String getSchedulePeriod() {
+ return schedulePeriod;
+ }
+
+ public void setSchedulePeriod(String schedulePeriod) {
+ this.schedulePeriod = schedulePeriod;
+ }
+
+ private Integer scheduleHourOfDay;
+
+ public Integer getScheduleHourOfDay() {
+ return scheduleHourOfDay;
+ }
+
+ public void setScheduleHourOfDay(Integer scheduleHourOfDay) {
+ this.scheduleHourOfDay = scheduleHourOfDay;
+ }
+
+ private Integer scheduleDayOfWeek;
+
+ public Integer getScheduleDayOfWeek() {
+ return scheduleDayOfWeek;
+ }
+ public void setScheduleDayOfWeek(Integer scheduleDayOfWeek) {
+ this.scheduleDayOfWeek = scheduleDayOfWeek;
+ }
+ public String getScheduleDescription() {
+ Date date = new Date();
+ Calendar cal = new GregorianCalendar();
+ cal.setTime(date);
+ SimpleDateFormat weeklyFormat = new SimpleDateFormat(" E h a ");
+ SimpleDateFormat dailyFormat = new SimpleDateFormat(" h a ");
+ String desc = "Not Scheduled";
+ if (schedulePeriod!=null && schedulePeriod!="") {
+ cal.set(Calendar.HOUR_OF_DAY, scheduleHourOfDay);
+ if (schedulePeriod.equals(this.SCHEDULE_PERIOD_WEEKLY)) {
+ cal.set(Calendar.DAY_OF_WEEK,scheduleDayOfWeek);
+ desc="Weekly, "+weeklyFormat.format(cal.getTime());
+ } else {
+ desc="Daily, "+dailyFormat.format(cal.getTime());
+ }
+ }
+ return desc;
+ }
+ private boolean harvestingNow;
+ public boolean isHarvestingNow() {
+ return this.harvestingNow;
+ }
+
+ public void setHarvestingNow(boolean harvestingNow) {
+ this.harvestingNow = harvestingNow;
+ }
@Override
public int hashCode() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
new file mode 100644
index 00000000000..f87e182ecf8
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
@@ -0,0 +1,520 @@
+/*
+ Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Dataverse Network - A web application to share, preserve and analyze research data.
+ Developed at the Institute for Quantitative Social Science, Harvard University.
+ Version 3.0.
+*/
+package edu.harvard.iq.dataverse.harvest.client;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+
+import java.io.InputStream;
+import java.io.StringReader;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+import java.io.File;
+
+import java.io.FileOutputStream;
+
+import java.io.PrintWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
+import java.util.zip.ZipInputStream;
+
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+import org.xml.sax.SAXException;
+
+//import org.xml.sax.InputSource;
+
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import javax.xml.stream.XMLInputFactory;
+
+/*
+ * This is an optimized implementation of OAIPMH GetRecord method.
+ * Some code is borrowed from the OCLC implementation.
+ * It handles the retrieval of the record in a drastically different manner:
+ * It parses and validates the top, "administrative" portion of the record using
+ * an event-driven parser. Once it reaches the "payload", the actual metadata
+ * record enclosed in ... tags, it just reads it line by
+ * line without parsing and saves it in a temp file. (The record will be parsed
+ * and validated in the next step, when we attempt to import it).
+ * On a very large record, for example, a DDI of a Dataset with a large number
+ * of associated data variables, even event-driven XML parsing can end up
+ * being rather expensive.
+ * This optimized version was originally written for DVN 3.*.
+ * Added in Dataverse 4: custom protocol extension for sending the metadata
+ * record as a pre-declared numbe of bytes.
+ * @author Leonid Andreev
+ *
+*/
+
+public class FastGetRecord {
+
+ /**
+ * Client-side GetRecord verb constructor
+ *
+ * @param baseURL the baseURL of the server to be queried
+ * @exception MalformedURLException the baseURL is bad
+ * @exception SAXException the xml response is bad
+ * @exception IOException an I/O error occurred
+ */
+
+ public FastGetRecord(String baseURL, String identifier, String metadataPrefix)
+ throws IOException, ParserConfigurationException, SAXException,
+ TransformerException {
+ harvestRecord (baseURL, identifier, metadataPrefix);
+
+ }
+
+ private String errorMessage = null;
+ private File savedMetadataFile = null;
+ private XMLInputFactory xmlInputFactory = null;
+ private boolean recordDeleted = false;
+
+ // TODO: logging
+
+ public String getErrorMessage () {
+ return errorMessage;
+ }
+
+ public File getMetadataFile () {
+ return savedMetadataFile;
+ }
+
+ public boolean isDeleted () {
+ return this.recordDeleted;
+ }
+
+
+ public void harvestRecord(String baseURL, String identifier, String metadataPrefix) throws IOException,
+ ParserConfigurationException, SAXException, TransformerException {
+
+ xmlInputFactory = javax.xml.stream.XMLInputFactory.newInstance();
+
+ String requestURL = getRequestURL(baseURL, identifier, metadataPrefix);
+
+ InputStream in = null;
+ URL url = new URL(requestURL);
+ HttpURLConnection con = null;
+ int responseCode = 0;
+
+ con = (HttpURLConnection) url.openConnection();
+ con.setRequestProperty("User-Agent", "OAIHarvester/2.0");
+ con.setRequestProperty("Accept-Encoding",
+ "compress, gzip, identify");
+ try {
+ responseCode = con.getResponseCode();
+ //logger.debug("responseCode=" + responseCode);
+ } catch (FileNotFoundException e) {
+ //logger.info(requestURL, e);
+ responseCode = HttpURLConnection.HTTP_UNAVAILABLE;
+ }
+
+ // TODO: -- L.A.
+ //
+ // support for cookies;
+ // support for limited retry attempts -- ?
+ // implement reading of the stream as filterinputstream -- ?
+ // -- that could make it a little faster still. -- L.A.
+
+
+
+ if (responseCode == 200) {
+
+ String contentEncoding = con.getHeaderField("Content-Encoding");
+ //logger.debug("contentEncoding=" + contentEncoding);
+
+ // support for the standard compress/gzip/deflate compression
+ // schemes:
+
+ if ("compress".equals(contentEncoding)) {
+ ZipInputStream zis = new ZipInputStream(con.getInputStream());
+ zis.getNextEntry();
+ in = zis;
+ } else if ("gzip".equals(contentEncoding)) {
+ in = new GZIPInputStream(con.getInputStream());
+ } else if ("deflate".equals(contentEncoding)) {
+ in = new InflaterInputStream(con.getInputStream());
+ } else {
+ in = con.getInputStream();
+ }
+
+ // We are going to read the OAI header and SAX-parse it for the
+ // error messages and other protocol information;
+ // The metadata section we're going to simply save in a temporary
+ // file, unparsed.
+
+ BufferedReader rd = new BufferedReader(new InputStreamReader(in));
+
+ String line = null;
+ String oaiResponseHeader = "";
+ boolean metadataFlag = false;
+ boolean metadataWritten = false;
+ boolean schemaChecked = false;
+
+ savedMetadataFile = File.createTempFile("meta", ".tmp");
+ FileOutputStream tempFileStream = new FileOutputStream(savedMetadataFile);
+ PrintWriter metadataOut = new PrintWriter (tempFileStream, true);
+
+ metadataOut.println("");
+
+ int mopen = 0;
+ int mclose = 0;
+
+ while ( ( line = rd.readLine () ) != null) {
+ if (!metadataFlag) {
+ if (line.matches(".*.*")) {
+ String lineCopy = line;
+
+ int i = line.indexOf("");
+ line = line.substring(i+10);
+
+ oaiResponseHeader = oaiResponseHeader.concat(lineCopy.replaceAll(".*", ""));
+
+ metadataFlag = true;
+ }
+ }
+
+ if (metadataFlag) {
+ if (!metadataWritten) {
+ // Inside an OAI-PMH GetRecord response, the metadata
+ // record returned is enclosed in ...
+ // tags, after the OAI service sections that provide the
+ // date, identifier and other protocol-level information.
+ // However, it is possible for the metadata record itself
+ // to have tags of its own. So we have no
+ // choice but to count the opening and closing tags in
+ // order to recognize the one terminating the metadata
+ // section.
+ // This code isn't pretty, but on seriously large records
+ // the savings from not fully parsing the XML are
+ // significant.
+ // -- L.A.
+
+ if (line.matches(" -1) {
+ if (!line.substring(i).matches("^]*/")) {
+ // don't count if it's a closed, empty tag:
+ //
+ mopen++;
+ }
+ i+=10;
+ }
+ }
+ if (line.matches(".*.*")) {
+ int i = 0;
+ while ((i = line.indexOf("", i)) > -1) {
+ i+=11;
+ mclose++;
+ }
+
+ if ( mclose > mopen ) {
+ line = line.substring(0, line.lastIndexOf(""));
+ metadataWritten = true;
+ }
+ }
+
+ if (!schemaChecked) {
+ // if the top-level XML element lacks the schema definition,
+ // insert the generic xmlns and xmlns:xsi attributes; these
+ // may be needed by the transform stylesheets.
+ // this mimicks the behaviour of the OCLC GetRecord
+ // client implementation.
+ // -L.A.
+
+ int offset = 0;
+
+ // However, there may be one or more XML comments before
+ // the first "real" XML element (of the form
+ // ). So we need to skip these!
+
+ while ( (line.indexOf('<', offset) > -1)
+ &&
+ "':
+
+ while (line != null
+ &&
+ ((offset = line.indexOf("-->",offset)) < 0)) {
+ line = line.replaceAll("[\n\r]", " ");
+ offset = line.length();
+ line = line.concat(rd.readLine());
+ }
+
+ offset += 3;
+ }
+
+ // if we have skipped some comments, is there another
+ // XML element left in the buffered line?
+ int firstElementStart = -1;
+
+ if ((firstElementStart = line.indexOf('<', offset)) > -1 ) {
+ // OK, looks like there is.
+ // is it terminated?
+ // if not, let's read the stream until
+ // we find the closing '>':
+
+ int firstElementEnd = -1;
+ offset = firstElementStart;
+
+ while (line != null
+ &&
+ ((firstElementEnd = line.indexOf('>',offset)) < 0)) {
+
+ line = line.replaceAll("[\n\r]", "");
+ offset = line.length();
+ line = line.concat(rd.readLine());
+ }
+
+ if (firstElementEnd < 0) {
+ // this should not happen!
+ // we've reached the end of the XML stream
+ // without encountering a single valid XML tag -- ??
+
+ this.errorMessage = "Malformed GetRecord response; reached the end of the stream but couldn't find a single valid XML element in the metadata section.";
+ } else {
+
+ // OK, we now have a line that contains a complete,
+ // terminated (possibly multi-line) first XML element
+ // that starts at [offset].
+
+ int i = firstElementStart;
+
+ if (!line.substring(i).matches("^<[^>]*xmlns.*")) {
+ String head = line.substring(0, i);
+ String tail = line.substring(i);
+ tail = tail.replaceFirst(">", " xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">");
+ line = head + tail;
+ }
+
+ schemaChecked = true;
+ }
+ } else {
+ // there was no "real" XML elements, only comments.
+ // We'll perform this schema check in the next
+ // iteration.
+ }
+ }
+
+ metadataOut.println(line);
+ }
+ } else {
+ oaiResponseHeader = oaiResponseHeader.concat(line);
+ }
+ }
+
+ // parse the OAI Record header:
+
+ XMLStreamReader xmlr = null;
+
+ try {
+ StringReader reader = new StringReader(oaiResponseHeader);
+ xmlr = xmlInputFactory.createXMLStreamReader(reader);
+ processOAIheader(xmlr);
+
+ } catch (XMLStreamException ex) {
+ //Logger.getLogger("global").log(Level.SEVERE, null, ex);
+ if (this.errorMessage == null) {
+ this.errorMessage = "Malformed GetRecord response: " + oaiResponseHeader;
+ }
+
+ // delete the temp metadata file; we won't need it:
+ if (savedMetadataFile != null) {
+ //savedMetadataFile.delete();
+ }
+
+ }
+
+ try {
+ if (xmlr != null) {
+ xmlr.close();
+ }
+ } catch (Exception ed) {
+ // seems OK to ignore;
+ }
+
+
+ if (rd != null) {
+ rd.close();
+ }
+
+ if (metadataOut != null) {
+ metadataOut.close();
+ }
+
+ if (!(metadataWritten) && !(this.isDeleted())) {
+ this.errorMessage = "Failed to parse GetRecord response; "+oaiResponseHeader;
+ //savedMetadataFile.delete();
+ }
+
+ if (this.isDeleted()) {
+ //savedMetadataFile.delete();
+ }
+
+
+ } else {
+ this.errorMessage = "GetRecord request failed. HTTP error code "+responseCode;
+ }
+ }
+
+ /**
+ * Construct the query portion of the http request
+ * (borrowed from OCLC implementation)
+ *
+ * @return a String containing the query portion of the http request
+ */
+ private static String getRequestURL(String baseURL,
+ String identifier,
+ String metadataPrefix) {
+
+ StringBuffer requestURL = new StringBuffer(baseURL);
+ requestURL.append("?verb=GetRecord");
+ requestURL.append("&identifier=").append(identifier);
+ requestURL.append("&metadataPrefix=").append(metadataPrefix);
+
+ return requestURL.toString();
+ }
+
+ private void processOAIheader (XMLStreamReader xmlr) throws XMLStreamException {
+
+ // is this really a GetRecord response?
+ xmlr.nextTag();
+ xmlr.require(XMLStreamConstants.START_ELEMENT, null, "OAI-PMH");
+ processOAIPMH(xmlr);
+
+ }
+
+ private void processOAIPMH (XMLStreamReader xmlr) throws XMLStreamException {
+
+ for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+ if (event == XMLStreamConstants.START_ELEMENT) {
+ // TODO:
+ // process all the fields currently skipped -- ? -- L.A.
+ if (xmlr.getLocalName().equals("responseDate")) {}
+ else if (xmlr.getLocalName().equals("request")) {}
+ else if (xmlr.getLocalName().equals("error")) {
+ String errorCode = xmlr.getAttributeValue(null, "code");
+ String errorMessageText = getElementText(xmlr);
+
+ if (errorCode != null) {
+ this.errorMessage = "GetRecord error code: "+errorCode+"; ";
+ }
+
+ if (errorCode != null) {
+ this.errorMessage = this.errorMessage + "GetRecord error message: "+errorMessageText+"; ";
+ }
+ throw new XMLStreamException(this.errorMessage);
+
+ }
+ else if (xmlr.getLocalName().equals("GetRecord")) processGetRecordSection(xmlr);
+ } else if (event == XMLStreamConstants.END_ELEMENT) {
+ if (xmlr.getLocalName().equals("OAI-PMH")) return;
+ }
+ }
+ }
+
+ private void processGetRecordSection (XMLStreamReader xmlr) throws XMLStreamException {
+ for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+ if (event == XMLStreamConstants.START_ELEMENT) {
+ if (xmlr.getLocalName().equals("record")) {processRecord(xmlr);}
+ } else if (event == XMLStreamConstants.END_ELEMENT) {
+ if (xmlr.getLocalName().equals("GetRecord")) return;
+ }
+ }
+
+ }
+
+ private void processRecord (XMLStreamReader xmlr) throws XMLStreamException {
+ for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+ if (event == XMLStreamConstants.START_ELEMENT) {
+ if (xmlr.getLocalName().equals("header")) {
+ if ("deleted".equals( xmlr.getAttributeValue(null, "status"))) {
+ this.recordDeleted = true;
+ }
+ processHeader(xmlr);
+ } else if (xmlr.getLocalName().equals("metadata")) {/*do nothing;*/}
+ } else if (event == XMLStreamConstants.END_ELEMENT) {
+ if (xmlr.getLocalName().equals("record")) return;
+ }
+ }
+ }
+
+ private void processHeader (XMLStreamReader xmlr) throws XMLStreamException {
+ for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+ if (event == XMLStreamConstants.START_ELEMENT) {
+ if (xmlr.getLocalName().equals("identifier")) {/*do nothing*/}
+ else if (xmlr.getLocalName().equals("datestamp")) {/*do nothing -- ?*/}
+ else if (xmlr.getLocalName().equals("setSpec")) {/*do nothing*/}
+
+
+ } else if (event == XMLStreamConstants.END_ELEMENT) {
+ if (xmlr.getLocalName().equals("header")) return;
+ }
+ }
+ }
+
+
+ // (from Gustavo's ddiServiceBean -- L.A.)
+ //
+ /* We had to add this method because the ref getElementText has a bug where it
+ * would append a null before the text, if there was an escaped apostrophe; it appears
+ * that the code finds an null ENTITY_REFERENCE in this case which seems like a bug;
+ * the workaround for the moment is to comment or handling ENTITY_REFERENCE in this case
+ */
+ private String getElementText(XMLStreamReader xmlr) throws XMLStreamException {
+ if(xmlr.getEventType() != XMLStreamConstants.START_ELEMENT) {
+ throw new XMLStreamException("parser must be on START_ELEMENT to read next text", xmlr.getLocation());
+ }
+ int eventType = xmlr.next();
+ StringBuffer content = new StringBuffer();
+ while(eventType != XMLStreamConstants.END_ELEMENT ) {
+ if(eventType == XMLStreamConstants.CHARACTERS
+ || eventType == XMLStreamConstants.CDATA
+ || eventType == XMLStreamConstants.SPACE
+ /* || eventType == XMLStreamConstants.ENTITY_REFERENCE*/) {
+ content.append(xmlr.getText());
+ } else if(eventType == XMLStreamConstants.PROCESSING_INSTRUCTION
+ || eventType == XMLStreamConstants.COMMENT
+ || eventType == XMLStreamConstants.ENTITY_REFERENCE) {
+ // skipping
+ } else if(eventType == XMLStreamConstants.END_DOCUMENT) {
+ throw new XMLStreamException("unexpected end of document when reading element text content");
+ } else if(eventType == XMLStreamConstants.START_ELEMENT) {
+ throw new XMLStreamException("element text content may not contain START_ELEMENT", xmlr.getLocation());
+ } else {
+ throw new XMLStreamException("Unexpected event type "+eventType, xmlr.getLocation());
+ }
+ eventType = xmlr.next();
+ }
+ return content.toString();
+ }
+
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestTimerInfo.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestTimerInfo.java
new file mode 100644
index 00000000000..96599352cf2
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestTimerInfo.java
@@ -0,0 +1,97 @@
+/*
+ Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ Dataverse Network - A web application to share, preserve and analyze research data.
+ Developed at the Institute for Quantitative Social Science, Harvard University.
+ Version 3.0.
+*/
+package edu.harvard.iq.dataverse.harvest.client;
+
+import java.io.Serializable;
+
+/**
+ * This class is used when creating an EJB Timer for scheduling Harvesting.
+ * We use this class rather than the HarvestingDataverse entity because
+ * the class must be Serializable, and there is too much info associated with the HarvestingDataverse
+ * in order to realistically serialize it. (We can't make related mapped entities transient.)
+ *
+ * Based on the DVN 3 implementation,
+ * original
+ * @author Ellen Kraffmiller
+ * incorporated into Dataverse 4 by
+ * @author Leonid Andreev
+ */
+public class HarvestTimerInfo implements Serializable {
+ private Long harvestingDataverseId;
+ private String name;
+ private String schedulePeriod;
+ private Integer scheduleHourOfDay;
+
+ public HarvestTimerInfo() {
+
+ }
+
+
+ public HarvestTimerInfo(Long harvestingDataverseId, String name, String schedulePeriod, Integer scheduleHourOfDay, Integer scheduleDayOfWeek) {
+ this.harvestingDataverseId=harvestingDataverseId;
+ this.name=name;
+ this.schedulePeriod=schedulePeriod;
+ this.scheduleDayOfWeek=scheduleDayOfWeek;
+ this.scheduleHourOfDay=scheduleHourOfDay;
+ }
+
+
+ public Long getHarvestingDataverseId() {
+ return harvestingDataverseId;
+ }
+
+ public void setHarvestingDataverseId(Long harvestingDataverseId) {
+ this.harvestingDataverseId = harvestingDataverseId;
+ }
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getSchedulePeriod() {
+ return schedulePeriod;
+ }
+
+ public void setSchedulePeriod(String schedulePeriod) {
+ this.schedulePeriod = schedulePeriod;
+ }
+
+ public Integer getScheduleHourOfDay() {
+ return scheduleHourOfDay;
+ }
+
+ public void setScheduleHourOfDay(Integer scheduleHourOfDay) {
+ this.scheduleHourOfDay = scheduleHourOfDay;
+ }
+
+ public Integer getScheduleDayOfWeek() {
+ return scheduleDayOfWeek;
+ }
+
+ public void setScheduleDayOfWeek(Integer scheduleDayOfWeek) {
+ this.scheduleDayOfWeek = scheduleDayOfWeek;
+ }
+ private Integer scheduleDayOfWeek;
+
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
new file mode 100644
index 00000000000..5f7cfb98e4a
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -0,0 +1,671 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse.harvest.client;
+
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetServiceBean;
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseServiceBean;
+import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
+import edu.harvard.iq.dataverse.timer.DataverseTimerServiceBean;
+import edu.harvard.iq.dataverse.util.FileUtil;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.net.MalformedURLException;
+import java.net.URLEncoder;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.logging.FileHandler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Resource;
+import javax.ejb.EJB;
+import javax.ejb.EJBException;
+import javax.ejb.Stateless;
+import javax.ejb.Timeout;
+import javax.ejb.Timer;
+import javax.ejb.TransactionAttribute;
+import javax.ejb.TransactionAttributeType;
+import javax.faces.bean.ManagedBean;
+import javax.inject.Named;
+import javax.persistence.Query;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBElement;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.TransformerException;
+import org.apache.commons.lang.mutable.MutableBoolean;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+/**
+ *
+ * @author Leonid Andreev
+ */
+@Stateless(name = "harvesterService")
+@Named
+@ManagedBean
+public class HarvesterServiceBean {
+ @EJB
+ DataverseServiceBean dataverseService;
+ @EJB
+ DatasetServiceBean datasetService;
+ @Resource
+ javax.ejb.TimerService timerService;
+ @EJB
+ DataverseTimerServiceBean dataverseTimerService;
+
+ private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean");
+ private static final String HARVEST_TIMER = "HarvestTimer";
+ private static final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
+ private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
+
+ public static final String HARVEST_RESULT_SUCCESS="success";
+ public static final String HARVEST_RESULT_FAILED="failed";
+
+
+ private JAXBContext jaxbContext;
+ private Unmarshaller unmarshaller;
+
+ private long processedSizeThisBatch = 0;
+ private List harvestedDatasetIdsThisBatch = null;
+ public HarvesterServiceBean() {
+
+ }
+
+ /**
+ * Called to run an "On Demand" harvest.
+ * This method creates a timer that will go off immediately,
+ * which will start an immediate asynchronous harvest.
+ * @param dataverse
+ */
+ public void doAsyncHarvest(Dataverse harvestedDataverse) {
+ HarvestingDataverseConfig harvestedDataverseConfig = harvestedDataverse.getHarvestingDataverseConfig();
+
+ if (harvestedDataverseConfig == null) {
+ logger.info("ERROR: No Harvesting Configuration found for dataverse id="+harvestedDataverse.getId());
+ return;
+ }
+
+ Calendar cal = Calendar.getInstance();
+
+ timerService.createTimer(cal.getTime(), new HarvestTimerInfo(harvestedDataverse.getId(), harvestedDataverse.getName(), harvestedDataverseConfig.getSchedulePeriod(), harvestedDataverseConfig.getScheduleHourOfDay(), harvestedDataverseConfig.getScheduleDayOfWeek()));
+ }
+
+ public void createScheduledHarvestTimers() {
+ logger.log(Level.INFO, "HarvesterService: going to (re)create Scheduled harvest timers.");
+ dataverseTimerService.removeHarvestTimers();
+
+ List dataverses = dataverseService.getAllHarvestedDataverses();
+ for (Iterator it = dataverses.iterator(); it.hasNext();) {
+ Dataverse dataverse = (Dataverse) it.next();
+ HarvestingDataverseConfig harvestingConfig = dataverse.getHarvestingDataverseConfig();
+ if (harvestingConfig == null) {
+ logger.warning("ERROR: no harvesting config found for dataverse id="+dataverse.getId());
+ } else if (harvestingConfig.isScheduled()) {
+ createHarvestTimer(dataverse);
+ }
+ }
+ }
+
+ public void removeHarvestTimer(Dataverse dataverse) {
+ dataverseTimerService.removeHarvestTimer(dataverse);
+ }
+
+ public void updateHarvestTimer(Dataverse harvestedDataverse) {
+ removeHarvestTimer(harvestedDataverse);
+ createHarvestTimer(harvestedDataverse);
+ }
+
+ public List getHarvestTimers() {
+ ArrayList timers = new ArrayList();
+ // Clear dataverse timer, if one exists
+ for (Iterator it = timerService.getTimers().iterator(); it.hasNext();) {
+ Timer timer = (Timer) it.next();
+ if (timer.getInfo() instanceof HarvestTimerInfo) {
+ HarvestTimerInfo info = (HarvestTimerInfo) timer.getInfo();
+ timers.add(info);
+ }
+ }
+ return timers;
+ }
+
+ private void createHarvestTimer(Dataverse harvestingDataverse) {
+ HarvestingDataverseConfig harvestingDataverseConfig = harvestingDataverse.getHarvestingDataverseConfig();
+
+ if (harvestingDataverseConfig == null) {
+ logger.info("ERROR: No Harvesting Configuration found for dataverse id="+harvestingDataverse.getId());
+ return;
+ }
+
+ if (harvestingDataverseConfig.isScheduled()) {
+ long intervalDuration = 0;
+ Calendar initExpiration = Calendar.getInstance();
+ initExpiration.set(Calendar.MINUTE, 0);
+ initExpiration.set(Calendar.SECOND, 0);
+ if (harvestingDataverseConfig.getSchedulePeriod().equals(harvestingDataverseConfig.SCHEDULE_PERIOD_DAILY)) {
+ intervalDuration = 1000 * 60 * 60 * 24;
+ initExpiration.set(Calendar.HOUR_OF_DAY, harvestingDataverseConfig.getScheduleHourOfDay());
+
+ } else if (harvestingDataverseConfig.getSchedulePeriod().equals(harvestingDataverseConfig.SCHEDULE_PERIOD_WEEKLY)) {
+ intervalDuration = 1000 * 60 * 60 * 24 * 7;
+ initExpiration.set(Calendar.HOUR_OF_DAY, harvestingDataverseConfig.getScheduleHourOfDay());
+ initExpiration.set(Calendar.DAY_OF_WEEK, harvestingDataverseConfig.getScheduleDayOfWeek());
+
+ } else {
+ logger.log(Level.WARNING, "Could not set timer for dataverse id, " + harvestingDataverse.getId() + ", unknown schedule period: " + harvestingDataverseConfig.getSchedulePeriod());
+ return;
+ }
+ Date initExpirationDate = initExpiration.getTime();
+ Date currTime = new Date();
+ if (initExpirationDate.before(currTime)) {
+ initExpirationDate.setTime(initExpiration.getTimeInMillis() + intervalDuration);
+ }
+ logger.log(Level.INFO, "Setting timer for dataverse " + harvestingDataverse.getName() + ", initial expiration: " + initExpirationDate);
+ dataverseTimerService.createTimer(initExpirationDate, intervalDuration, new HarvestTimerInfo(harvestingDataverse.getId(), harvestingDataverse.getName(), harvestingDataverseConfig.getSchedulePeriod(), harvestingDataverseConfig.getScheduleHourOfDay(), harvestingDataverseConfig.getScheduleDayOfWeek()));
+ }
+ }
+
+ /**
+ * This method is called whenever an EJB Timer goes off.
+ * Check to see if this is a Harvest Timer, and if it is
+ * Run the harvest for the given (scheduled) dataverse
+ * @param timer
+ */
+ @Timeout
+ @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
+ public void handleTimeout(javax.ejb.Timer timer) {
+ // We have to put all the code in a try/catch block because
+ // if an exception is thrown from this method, Glassfish will automatically
+ // call the method a second time. (The minimum number of re-tries for a Timer method is 1)
+
+ if (timer.getInfo() instanceof HarvestTimerInfo) {
+ HarvestTimerInfo info = (HarvestTimerInfo) timer.getInfo();
+ try {
+ // First, check if we are in read-only mode:
+ /*
+ if (...) {
+ logger.log(Level.ALL, "Dataverse is in read-only mode.");
+ return;
+
+ }
+ */
+
+ // Proceeding with the scheduled harvest:
+
+ logger.log(Level.INFO, "DO HARVESTING of dataverse " + info.getHarvestingDataverseId());
+ doHarvesting(info.getHarvestingDataverseId());
+
+ } catch (Throwable e) {
+ dataverseService.setHarvestResult(info.getHarvestingDataverseId(), this.HARVEST_RESULT_FAILED);
+ /*mailService.sendHarvestErrorNotification(...getSystemEmail(), ...);*/
+ logException(e, logger);
+ }
+ }
+ }
+
+ /**
+ * Harvest an individual Dataverse
+ * @param dataverseId
+ */
+ public void doHarvesting(Long dataverseId) throws IOException {
+ Dataverse harvestingDataverse = dataverseService.find(dataverseId);
+
+ if (harvestingDataverse == null) {
+ throw new IOException("No such Dataverse: id="+dataverseId);
+ }
+
+ HarvestingDataverseConfig harvestingDataverseConfig = harvestingDataverse.getHarvestingDataverseConfig();
+
+ if (harvestingDataverseConfig == null) {
+ throw new IOException("Could not find Harvesting Config for Dataverse id="+dataverseId);
+ }
+
+ MutableBoolean harvestErrorOccurred = new MutableBoolean(false);
+ String logTimestamp = logFormatter.format(new Date());
+ Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingDataverse.getAlias() + logTimestamp);
+ String logFileName = /* TODO: !!!! FileUtil.getImportFileDir() +*/ File.separator + "harvest_" + harvestingDataverse.getAlias() + logTimestamp + ".log";
+ FileHandler fileHandler = new FileHandler(logFileName);
+ hdLogger.addHandler(fileHandler);
+ List harvestedDatasetIds = null;
+
+ this.processedSizeThisBatch = 0;
+ this.harvestedDatasetIdsThisBatch = new ArrayList();
+
+ List failedIdentifiers = new ArrayList();
+ try {
+ boolean harvestingNow = harvestingDataverseConfig.isHarvestingNow();
+
+ if (harvestingNow) {
+ harvestErrorOccurred.setValue(true);
+ hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested.");
+
+ } else {
+ dataverseService.resetHarvestingStatus(harvestingDataverse.getId());
+ String until = null; // If we don't set until date, we will get all the changes since the last harvest.
+ String from = null;
+ Date lastSuccessfulHarvestTime = harvestingDataverseConfig.getLastSuccessfulHarvestTime();
+ if (lastSuccessfulHarvestTime != null) {
+ from = formatter.format(lastSuccessfulHarvestTime);
+ }
+ dataverseService.setHarvestInProgress(harvestingDataverse.getId(), true);
+ Date currentTime = new Date();
+ dataverseService.setLastHarvestTime(harvestingDataverse.getId(), currentTime);
+
+ hdLogger.log(Level.INFO, "BEGIN HARVEST..., oaiUrl=" + harvestingDataverseConfig.getArchiveUrl() + ",set=" + harvestingDataverseConfig.getHarvestingSet() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix() + ", from=" + from + ", until=" + until);
+
+ if (harvestingDataverseConfig.isOai()) {
+ harvestedDatasetIds = harvestOAI(harvestingDataverse, hdLogger, from, until, harvestErrorOccurred, failedIdentifiers);
+
+ } else {
+ throw new IOException("Unsupported harvest type");
+ }
+ dataverseService.setHarvestSuccess(harvestingDataverse.getId(),currentTime, harvestedDatasetIds.size(), failedIdentifiers.size());
+ hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingDataverseConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix());
+
+ /* Last "non-empty" harvest: */
+ if (harvestedDatasetIds.size() > 0) {
+ dataverseService.setHarvestSuccessNotEmpty(harvestingDataverse.getId(),currentTime, harvestedDatasetIds.size(), failedIdentifiers.size());
+ hdLogger.log(Level.INFO, "COMPLETED HARVEST with results");
+ }
+
+ // now index all studies (need to modify for update)
+ /* (TODO: !!!)
+ if (this.processedSizeThisBatch > 0) {
+ hdLogger.log(Level.INFO, "POST HARVEST, reindexing the remaining studies.");
+ if (this.harvestedDatasetIdsThisBatch != null) {
+ hdLogger.log(Level.INFO, this.harvestedDatasetIdsThisBatch.size()+" studies in the batch");
+ }
+ hdLogger.log(Level.INFO, this.processedSizeThisBatch + " bytes of content");
+ indexService.updateIndexList(this.harvestedDatasetIdsThisBatch);
+ hdLogger.log(Level.INFO, "POST HARVEST, calls to index finished.");
+ } else {
+ hdLogger.log(Level.INFO, "(All harvested content already reindexed)");
+ }
+ */
+ }
+ //mailService.sendHarvestNotification(...getSystemEmail(), harvestingDataverse.getName(), logFileName, logTimestamp, harvestErrorOccurred.booleanValue(), harvestedDatasetIds.size(), failedIdentifiers);
+ } catch (Throwable e) {
+ harvestErrorOccurred.setValue(true);
+ String message = "Exception processing harvest, server= " + harvestingDataverseConfig.getArchiveUrl() + ",format=" + harvestingDataverseConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage();
+ hdLogger.log(Level.SEVERE, message);
+ logException(e, hdLogger);
+ hdLogger.log(Level.INFO, "HARVEST NOT COMPLETED DUE TO UNEXPECTED ERROR.");
+ dataverseService.setHarvestFailure(harvestingDataverse.getId(), harvestedDatasetIds.size(), failedIdentifiers.size());
+
+
+ } finally {
+ dataverseService.setHarvestInProgress(harvestingDataverse.getId(), false);
+ fileHandler.close();
+ hdLogger.removeHandler(fileHandler);
+ }
+ }
+
+ /**
+ *
+ * @param dataverse the dataverse to harvest into
+ * @param from get updated studies from this beginning date
+ * @param until get updated studies until this end date
+ * @param harvestErrorOccurred have we encountered any errors during harvest?
+ * @param failedIdentifiers Study Identifiers for failed "GetRecord" requests
+ */
+ private List harvestOAI(Dataverse dataverse, Logger hdLogger, String from, String until, MutableBoolean harvestErrorOccurred, List failedIdentifiers)
+ throws IOException, ParserConfigurationException,SAXException, TransformerException, JAXBException {
+
+ List harvestedDatasetIds = new ArrayList();
+
+ /*
+ ResumptionTokenType resumptionToken = null;
+
+ do {
+ //resumptionToken = harvesterService.harvestFromIdentifiers(hdLogger, resumptionToken, dataverse, from, until, harvestedDatasetIds, failedIdentifiers, harvestErrorOccurred
+ resumptionToken = harvestFromIdentifiers(hdLogger, resumptionToken, dataverse, from, until, harvestedDatasetIds, failedIdentifiers, harvestErrorOccurred);
+ } while (resumptionToken != null && !resumptionToken.equals(""));
+
+ hdLogger.log(Level.INFO, "COMPLETED HARVEST, oaiUrl=" + dataverse.getServerUrl() + ",set=" + dataverse.getHarvestingSet() + ", metadataPrefix=" + dataverse.getHarvestFormatType().getMetadataPrefix() + ", from=" + from + ", until=" + until);
+
+ */
+ return harvestedDatasetIds;
+
+ }
+
+ /*
+ @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
+ public ResumptionTokenType harvestFromIdentifiers(Logger hdLogger, ResumptionTokenType resumptionToken, HarvestingDataverse dataverse, String from, String until, List harvestedDatasetIds, List failedIdentifiers, MutableBoolean harvestErrorOccurred)
+ throws java.io.IOException, ParserConfigurationException, SAXException, TransformerException, JAXBException {
+ String encodedSet = dataverse.getHarvestingSet() == null ? null : URLEncoder.encode(dataverse.getHarvestingSet(), "UTF-8");
+ ListIdentifiers listIdentifiers = null;
+
+ if (resumptionToken == null) {
+ listIdentifiers = new ListIdentifiers(dataverse.getServerUrl(),
+ from,
+ until,
+ encodedSet,
+ URLEncoder.encode(dataverse.getHarvestFormatType().getMetadataPrefix(), "UTF-8"));
+ } else {
+ hdLogger.log(Level.INFO, "harvestFromIdentifiers(), resumptionToken=" + resumptionToken.getValue());
+ listIdentifiers = new ListIdentifiers(dataverse.getServerUrl(), resumptionToken.getValue());
+ }
+
+ Document doc = listIdentifiers.getDocument();
+
+ // JAXBContext jc = JAXBContext.newInstance("edu.harvard.hmdc.vdcnet.jaxb.oai");
+ // Unmarshaller unmarshaller = jc.createUnmarshaller();
+ JAXBElement unmarshalObj = (JAXBElement) unmarshaller.unmarshal(doc);
+ OAIPMHtype oaiObj = (OAIPMHtype) unmarshalObj.getValue();
+
+ if (oaiObj.getError() != null && oaiObj.getError().size() > 0) {
+ if (oaiObj.getError().get(0).getCode().equals(OAIPMHerrorcodeType.NO_RECORDS_MATCH)) {
+ hdLogger.info("ListIdentifiers returned NO_RECORDS_MATCH - no studies found to be harvested.");
+ } else {
+ handleOAIError(hdLogger, oaiObj, "calling listIdentifiers, oaiServer= " + dataverse.getServerUrl() + ",from=" + from + ",until=" + until + ",encodedSet=" + encodedSet + ",format=" + dataverse.getHarvestFormatType().getMetadataPrefix());
+ throw new EJBException("Received OAI Error response calling ListIdentifiers");
+ }
+ } else {
+ ListIdentifiersType listIdentifiersType = oaiObj.getListIdentifiers();
+ if (listIdentifiersType != null) {
+ resumptionToken = listIdentifiersType.getResumptionToken();
+ for (Iterator it = listIdentifiersType.getHeader().iterator(); it.hasNext();) {
+ HeaderType header = (HeaderType) it.next();
+ MutableBoolean getRecordErrorOccurred = new MutableBoolean(false);
+ Long studyId = getRecord(hdLogger, dataverse, header.getIdentifier(), dataverse.getHarvestFormatType().getMetadataPrefix(), getRecordErrorOccurred);
+ if (studyId != null) {
+ harvestedDatasetIds.add(studyId);
+ }
+ if (getRecordErrorOccurred.booleanValue()==true) {
+ failedIdentifiers.add(header.getIdentifier());
+ }
+
+ }
+
+ }
+ }
+ String logMsg = "Returning from harvestFromIdentifiers";
+
+ if (resumptionToken == null) {
+ logMsg += " resumptionToken is null";
+ } else if (!StringUtil.isEmpty(resumptionToken.getValue())) {
+ logMsg += " resumptionToken is " + resumptionToken.getValue();
+ } else {
+ // Some OAIServers return an empty resumptionToken element when all
+ // the identifiers have been sent, so need to check for this, and
+ // treat it as if resumptiontoken is null.
+ logMsg += " resumptionToken is empty, setting return value to null.";
+ resumptionToken = null;
+ }
+ hdLogger.info(logMsg);
+ return resumptionToken;
+ }
+ */
+
+ /*
+ private void handleOAIError(Logger hdLogger, OAIPMHtype oaiObj, String message) {
+ for (Iterator it = oaiObj.getError().iterator(); it.hasNext();) {
+ OAIPMHerrorType error = (OAIPMHerrorType) it.next();
+ message += ", error code: " + error.getCode();
+ message += ", error value: " + error.getValue();
+ hdLogger.log(Level.SEVERE, message);
+
+ }
+ }
+ */
+
+ /*
+ @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
+ public Long getRecord(HarvestingDataverse dataverse, String identifier, String metadataPrefix) {
+ return getRecord(logger, dataverse, identifier, metadataPrefix, null);
+ }
+ */
+
+ @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
+ public Long getRecord(Logger hdLogger, Dataverse dataverse, String identifier, String metadataPrefix, MutableBoolean recordErrorOccurred) {
+ String errMessage = null;
+
+ HarvestingDataverseConfig harvestingConfig = dataverse.getHarvestingDataverseConfig();
+
+ if (harvestingConfig == null) {
+ errMessage = "Could not find Harvesting Config for Dataverse id="+dataverse.getId();
+ hdLogger.log(Level.SEVERE, errMessage);
+ return null;
+ }
+
+ Dataset harvestedDataset = null;
+ String oaiUrl = harvestingConfig.getHarvestingUrl();
+ try {
+ hdLogger.log(Level.INFO, "Calling GetRecord: oaiUrl =" + oaiUrl + "?verb=GetRecord&identifier=" + identifier + "&metadataPrefix=" + metadataPrefix);
+
+ FastGetRecord record = new FastGetRecord(oaiUrl, identifier, metadataPrefix);
+ errMessage = record.getErrorMessage();
+ //errMessage=null;
+
+ if (errMessage != null) {
+ hdLogger.log(Level.SEVERE, "Error calling GetRecord - " + errMessage);
+ } else if (record.isDeleted()) {
+ hdLogger.log(Level.INFO, "Received 'deleted' status from OAI Server.");
+ Dataset dataset = null; //TODO: !!! datasetService.getDatasetByHarvestInfo(dataverse, identifier);
+ if (dataset != null) {
+ hdLogger.log(Level.INFO, "Deleting study " + dataset.getGlobalId());
+ // TODO: !!! datasetService.deleteDataset(dataset.getId());
+ } else {
+ hdLogger.log(Level.INFO, "No study found for this record, skipping delete. ");
+ }
+
+ } else {
+ hdLogger.log(Level.INFO, "Successfully retreived GetRecord response.");
+
+
+ harvestedDataset = null; // TODO: !!! import
+ hdLogger.log(Level.INFO, "Harvest Successful for identifier " + identifier);
+
+ this.processedSizeThisBatch += record.getMetadataFile().length();
+ if ( this.harvestedDatasetIdsThisBatch == null ) {
+ this.harvestedDatasetIdsThisBatch = new ArrayList();
+ }
+ this.harvestedDatasetIdsThisBatch.add(harvestedDataset.getId());
+
+ // reindexing in batches? - this is from DVN 3;
+ // we may not need it anymore.
+ if ( this.processedSizeThisBatch > 10000000 ) {
+
+ hdLogger.log(Level.INFO, "REACHED CONTENT BATCH SIZE LIMIT; calling index ("+this.harvestedDatasetIdsThisBatch.size()+" studies in the batch).");
+ //indexService.updateIndexList(this.harvestedDatasetIdsThisBatch);
+ hdLogger.log(Level.INFO, "REINDEX DONE.");
+
+
+ this.processedSizeThisBatch = 0;
+ this.harvestedDatasetIdsThisBatch = null;
+ }
+ }
+ } catch (Throwable e) {
+ errMessage = "Exception processing getRecord(), oaiUrl=" + oaiUrl + ",identifier=" + identifier + " " + e.getClass().getName() + " " + e.getMessage();
+ hdLogger.log(Level.SEVERE, errMessage);
+ logException(e, hdLogger);
+
+ }
+
+ // If we got an Error from the OAI server or an exception happened during import, then
+ // set recordErrorOccurred to true (if recordErrorOccurred is being used)
+ // otherwise throw an exception (if recordErrorOccurred is not used, i.e null)
+ if (errMessage != null) {
+ if (recordErrorOccurred != null) {
+ recordErrorOccurred.setValue(true);
+ } else {
+ throw new EJBException(errMessage);
+ }
+ }
+
+ return harvestedDataset != null ? harvestedDataset.getId() : null;
+ }
+
+
+ /*
+ public List getMetadataFormats(String oaiUrl) {
+ JAXBElement unmarshalObj;
+ try {
+
+ Document doc = new ListMetadataFormats(oaiUrl).getDocument();
+ JAXBContext jc = JAXBContext.newInstance("edu.harvard.hmdc.vdcnet.jaxb.oai");
+ Unmarshaller unmarshaller = jc.createUnmarshaller();
+ unmarshalObj = (JAXBElement) unmarshaller.unmarshal(doc);
+ } catch (TransformerException ex) {
+ throw new EJBException(ex);
+ } catch (ParserConfigurationException ex) {
+ throw new EJBException(ex);
+ } catch (JAXBException ex) {
+ throw new EJBException(ex);
+ } catch (SAXException ex) {
+ throw new EJBException(ex);
+ } catch (IOException ex) {
+ throw new EJBException(ex);
+ }
+
+ OAIPMHtype OAIObj = (OAIPMHtype) unmarshalObj.getValue();
+ if (OAIObj.getError()!=null && OAIObj.getError().size()>0) {
+ List errList = OAIObj.getError();
+ String errMessage="";
+ for (OAIPMHerrorType error : OAIObj.getError()){
+ errMessage += error.getCode()+ " " +error.getValue();
+ }
+ throw new EJBException(errMessage);
+ }
+ ListMetadataFormatsType listMetadataFormats = OAIObj.getListMetadataFormats();
+ List formats = null;
+ if (listMetadataFormats != null) {
+ formats = new ArrayList();
+ for (Iterator it = listMetadataFormats.getMetadataFormat().iterator(); it.hasNext();) {
+ // Object elem = it.next();
+ MetadataFormatType elem = (MetadataFormatType) it.next();
+ formats.add(elem.getMetadataPrefix());
+ }
+ }
+ return formats;
+ }
+ */
+
+ /**
+ *
+ * SetDetailBean returned rather than the ListSetsType because we get strange errors when trying
+ * to refer to JAXB generated classes in both Web and EJB tiers.
+ */
+ /*
+ public List getSets(String oaiUrl) {
+ JAXBElement unmarshalObj = null;
+
+ try {
+ ListSets listSets = new ListSets(oaiUrl);
+ int nodeListLength = listSets.getErrors().getLength();
+ if (nodeListLength==1) {
+ System.out.println("err Node: "+ listSets.getErrors().item(0));
+ }
+
+
+ Document doc = new ListSets(oaiUrl).getDocument();
+ JAXBContext jc = JAXBContext.newInstance("edu.harvard.hmdc.vdcnet.jaxb.oai");
+ Unmarshaller unmarshaller = jc.createUnmarshaller();
+ unmarshalObj = (JAXBElement) unmarshaller.unmarshal(doc);
+ } catch (ParserConfigurationException ex) {
+ throw new EJBException(ex);
+ } catch (SAXException ex) {
+ throw new EJBException(ex);
+ } catch (TransformerException ex) {
+ throw new EJBException(ex);
+ } catch (IOException ex) {
+ throw new EJBException(ex);
+ } catch (JAXBException ex) {
+ throw new EJBException(ex);
+ }
+ List sets = null;
+ Object value = unmarshalObj.getValue();
+
+ Package valPackage = value.getClass().getPackage();
+ if (value instanceof edu.harvard.hmdc.vdcnet.jaxb.oai.OAIPMHtype) {
+ OAIPMHtype OAIObj = (OAIPMHtype) value;
+ if (OAIObj.getError()!=null && OAIObj.getError().size()>0 ) {
+ List errList = OAIObj.getError();
+ String errMessage="";
+ for (OAIPMHerrorType error : OAIObj.getError()){
+ // NO_SET_HIERARCHY is not an error from the perspective of the DVN,
+ // it just means that the OAI server doesn't support sets.
+ if (!error.getCode().equals(OAIPMHerrorcodeType.NO_SET_HIERARCHY)) {
+ errMessage += error.getCode()+ " " +error.getValue();
+ }
+ }
+ if (errMessage!="") {
+ throw new EJBException(errMessage);
+ }
+
+ }
+
+ ListSetsType listSetsType = OAIObj.getListSets();
+ if (listSetsType != null) {
+ sets = new ArrayList();
+ for (Iterator it = listSetsType.getSet().iterator(); it.hasNext();) {
+ SetType elem = (SetType) it.next();
+ SetDetailBean setDetail = new SetDetailBean();
+ setDetail.setName(elem.getSetName());
+ setDetail.setSpec(elem.getSetSpec());
+ sets.add(setDetail);
+ }
+ }
+ }
+ return sets;
+ }
+ */
+
+
+ private void logException(Throwable e, Logger logger) {
+
+ boolean cause = false;
+ String fullMessage = "";
+ do {
+ String message = e.getClass().getName() + " " + e.getMessage();
+ if (cause) {
+ message = "\nCaused By Exception.................... " + e.getClass().getName() + " " + e.getMessage();
+ }
+ StackTraceElement[] ste = e.getStackTrace();
+ message += "\nStackTrace: \n";
+ for (int m = 0; m < ste.length; m++) {
+ message += ste[m].toString() + "\n";
+ }
+ fullMessage += message;
+ cause = true;
+ } while ((e = e.getCause()) != null);
+ logger.severe(fullMessage);
+ }
+
+ /*
+ Most likely not needed any more:
+ public List findAllHarvestFormatTypes() {
+ String queryStr = "SELECT f FROM HarvestFormatType f";
+ Query query = em.createQuery(queryStr);
+ return query.getResultList();
+ }
+
+ public HarvestFormatType findHarvestFormatTypeByMetadataPrefix(String metadataPrefix) {
+ String queryStr = "SELECT f FROM HarvestFormatType f WHERE f.metadataPrefix = '" + metadataPrefix + "'";
+ Query query = em.createQuery(queryStr);
+ List resultList = query.getResultList();
+ HarvestFormatType hft = null;
+ if (resultList.size() > 1) {
+ throw new EJBException("More than one HarvestFormatType found with metadata Prefix= '" + metadataPrefix + "'");
+ }
+ if (resultList.size() == 1) {
+ hft = (HarvestFormatType) resultList.get(0);
+ }
+ return hft;
+ }
+*/
+
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
new file mode 100644
index 00000000000..6a4159d11a8
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
@@ -0,0 +1,226 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse.timer;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseServiceBean;
+import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
+import edu.harvard.iq.dataverse.harvest.client.HarvestTimerInfo;
+import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.annotation.Resource;
+import javax.ejb.EJB;
+import javax.ejb.Stateless;
+import javax.ejb.Timeout;
+import javax.ejb.Timer;
+import javax.ejb.TransactionAttribute;
+import javax.ejb.TransactionAttributeType;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+
+/**
+ *
+ * @author roberttreacy
+ */
+@Stateless
+public class DataverseTimerServiceBean implements Serializable {
+ @Resource
+ javax.ejb.TimerService timerService;
+ @PersistenceContext(unitName = "VDCNet-ejbPU")
+ private EntityManager em;
+ private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.timer.DataverseTimerServiceBean");
+ @EJB
+ HarvesterServiceBean harvesterService;
+ @EJB
+ DataverseServiceBean dataverseService;
+
+ /*@EJB
+ StudyServiceLocal studyService;*/
+
+
+ public void createTimer(Date initialExpiration, long intervalDuration, Serializable info) {
+ try {
+ logger.log(Level.INFO,"Creating timer on " + InetAddress.getLocalHost().getCanonicalHostName());
+ } catch (UnknownHostException ex) {
+ Logger.getLogger(DataverseTimerServiceBean.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ timerService.createTimer(initialExpiration, intervalDuration, info);
+ }
+
+
+ /**
+ * This method is called whenever an EJB Timer goes off.
+ * Check to see if this is a Harvest Timer, and if it is
+ * Run the harvest for the given (scheduled) dataverse
+ * @param timer
+ */
+ @Timeout
+ @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
+ public void handleTimeout(javax.ejb.Timer timer) {
+ // We have to put all the code in a try/catch block because
+ // if an exception is thrown from this method, Glassfish will automatically
+ // call the method a second time. (The minimum number of re-tries for a Timer method is 1)
+
+ try {
+ logger.log(Level.INFO,"Handling timeout on " + InetAddress.getLocalHost().getCanonicalHostName());
+ } catch (UnknownHostException ex) {
+ Logger.getLogger(DataverseTimerServiceBean.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ if (timer.getInfo() instanceof HarvestTimerInfo) {
+ HarvestTimerInfo info = (HarvestTimerInfo) timer.getInfo();
+ try {
+
+ logger.log(Level.INFO, "DO HARVESTING of dataverse " + info.getHarvestingDataverseId());
+ harvesterService.doHarvesting(info.getHarvestingDataverseId());
+
+ } catch (Throwable e) {
+ dataverseService.setHarvestResult(info.getHarvestingDataverseId(), harvesterService.HARVEST_RESULT_FAILED);
+ //mailService.sendHarvestErrorNotification(dataverseService.find().getSystemEmail(), dataverseService.find().getName());
+ logException(e, logger);
+ }
+ }
+ /* Export timers: (not yet implemented!) -- L.A.
+ if (timer.getInfo() instanceof ExportTimerInfo) {
+ try {
+ ExportTimerInfo info = (ExportTimerInfo) timer.getInfo();
+ logger.info("handling timeout");
+ studyService.exportUpdatedStudies();
+ } catch (Throwable e) {
+ mailService.sendExportErrorNotification(vdcNetworkService.find().getSystemEmail(), vdcNetworkService.find().getName());
+ logException(e, logger);
+ }
+ }
+ */
+
+ }
+
+ public void removeHarvestTimers() {
+ // Remove all the harvest timers, if exist:
+ //
+ // (the logging messages below are set to level INFO; it's ok,
+ // since this code is only called on startup of the application,
+ // and it may be useful to know what existing timers were encountered).
+
+ logger.log(Level.INFO,"Removing existing harvest timers..");
+
+ int i = 1;
+ for (Iterator it = timerService.getTimers().iterator(); it.hasNext();) {
+
+ Timer timer = (Timer) it.next();
+ logger.log(Level.INFO, "HarvesterService: checking timer "+i);
+
+ if (timer.getInfo() instanceof HarvestTimerInfo) {
+ logger.log(Level.INFO, "HarvesterService: timer "+i+" is a harvesting one; removing.");
+ timer.cancel();
+ }
+
+ i++;
+ }
+ }
+
+ private void createHarvestTimer(Dataverse harvestedDataverse) {
+ HarvestingDataverseConfig harvestedDataverseConfig = harvestedDataverse.getHarvestingDataverseConfig();
+
+ if (harvestedDataverseConfig == null) {
+ logger.info("ERROR: No Harvesting Configuration found for dataverse id="+harvestedDataverse.getId());
+ return;
+ }
+
+ if (harvestedDataverseConfig.isScheduled()) {
+ long intervalDuration = 0;
+ Calendar initExpiration = Calendar.getInstance();
+ initExpiration.set(Calendar.MINUTE, 0);
+ initExpiration.set(Calendar.SECOND, 0);
+ if (harvestedDataverseConfig.getSchedulePeriod().equals(HarvestingDataverseConfig.SCHEDULE_PERIOD_DAILY)) {
+ intervalDuration = 1000 * 60 * 60 * 24;
+ initExpiration.set(Calendar.HOUR_OF_DAY, harvestedDataverseConfig.getScheduleHourOfDay());
+
+ } else if (harvestedDataverseConfig.getSchedulePeriod().equals(harvestedDataverseConfig.SCHEDULE_PERIOD_WEEKLY)) {
+ intervalDuration = 1000 * 60 * 60 * 24 * 7;
+ initExpiration.set(Calendar.HOUR_OF_DAY, harvestedDataverseConfig.getScheduleHourOfDay());
+ initExpiration.set(Calendar.DAY_OF_WEEK, harvestedDataverseConfig.getScheduleDayOfWeek());
+
+ } else {
+ logger.log(Level.WARNING, "Could not set timer for harvestedDataverse id, " + harvestedDataverse.getId() + ", unknown schedule period: " + harvestedDataverseConfig.getSchedulePeriod());
+ return;
+ }
+ Date initExpirationDate = initExpiration.getTime();
+ Date currTime = new Date();
+ if (initExpirationDate.before(currTime)) {
+ initExpirationDate.setTime(initExpiration.getTimeInMillis() + intervalDuration);
+ }
+ logger.log(Level.INFO, "Setting timer for dataverse " + harvestedDataverse.getName() + ", initial expiration: " + initExpirationDate);
+ createTimer(initExpirationDate, intervalDuration, new HarvestTimerInfo(harvestedDataverse.getId(), harvestedDataverse.getName(), harvestedDataverseConfig.getSchedulePeriod(), harvestedDataverseConfig.getScheduleHourOfDay(), harvestedDataverseConfig.getScheduleDayOfWeek()));
+ }
+ }
+
+ public void updateHarvestTimer(Dataverse harvestedDataverse) {
+ removeHarvestTimer(harvestedDataverse);
+ createHarvestTimer(harvestedDataverse);
+ }
+
+
+ public void removeHarvestTimer(Dataverse harvestedDataverse) {
+ // Clear dataverse timer, if one exists
+ try {
+ logger.log(Level.INFO,"Removing harvest timer on " + InetAddress.getLocalHost().getCanonicalHostName());
+ } catch (UnknownHostException ex) {
+ Logger.getLogger(DataverseTimerServiceBean.class.getName()).log(Level.SEVERE, null, ex);
+ }
+ for (Iterator it = timerService.getTimers().iterator(); it.hasNext();) {
+ Timer timer = (Timer) it.next();
+ if (timer.getInfo() instanceof HarvestTimerInfo) {
+ HarvestTimerInfo info = (HarvestTimerInfo) timer.getInfo();
+ if (info.getHarvestingDataverseId().equals(harvestedDataverse.getId())) {
+ timer.cancel();
+ }
+ }
+ }
+ }
+
+ public void createExportTimer() {
+ /* Not yet implemented. The DVN 3 implementation can be used as a model */
+
+ }
+
+ public void createExportTimer(Dataverse dataverse) {
+ /* Not yet implemented. The DVN 3 implementation can be used as a model */
+
+ }
+
+ public void removeExportTimer() {
+ /* Not yet implemented. The DVN 3 implementation can be used as a model */
+ }
+
+ /* Utility methods: */
+ private void logException(Throwable e, Logger logger) {
+
+ boolean cause = false;
+ String fullMessage = "";
+ do {
+ String message = e.getClass().getName() + " " + e.getMessage();
+ if (cause) {
+ message = "\nCaused By Exception.................... " + e.getClass().getName() + " " + e.getMessage();
+ }
+ StackTraceElement[] ste = e.getStackTrace();
+ message += "\nStackTrace: \n";
+ for (int m = 0; m < ste.length; m++) {
+ message += ste[m].toString() + "\n";
+ }
+ fullMessage += message;
+ cause = true;
+ } while ((e = e.getCause()) != null);
+ logger.severe(fullMessage);
+ }
+
+}
\ No newline at end of file
From 91315ed4d3a5b33b66332cddbe345ae2147d0a0c Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Mon, 18 Apr 2016 18:13:36 -0400
Subject: [PATCH 05/37] Initial framework for the Harvesting REST API.
---
.../iq/dataverse/DataverseServiceBean.java | 31 +++--
.../dataverse/HarvestingDataverseConfig.java | 54 ++++++++-
.../iq/dataverse/api/BatchServiceBean.java | 2 +-
.../harvard/iq/dataverse/api/Harvesting.java | 65 +++++++++++
.../harvest/client/HarvesterServiceBean.java | 109 +++++-------------
.../timer/DataverseTimerServiceBean.java | 2 +-
6 files changed, 163 insertions(+), 100 deletions(-)
create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index ee08517fc96..33ce58ff250 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -548,15 +548,19 @@ public void setHarvestSuccess(Long hdId, Date currentTime, int harvestedCount, i
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- /* TODO:
- hd.getHarvestingDataverseConfig().setLastSuccessfulHarvestTime(currentTime);
- hd.getHarvestingDataverseConfig().setHarvestedStudyCount(new Long(harvestedCount));
- hd.getHarvestingDataverseConfig().setFailedStudyCount(new Long(failedCount));
- */
+ hd.getHarvestingDataverseConfig().setLastHarvestTime(currentTime);
+ hd.getHarvestingDataverseConfig().setLastSuccessfulHarvestTime(currentTime);
hd.getHarvestingDataverseConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_SUCCESS);
+
+ if (harvestedCount > 0 || failedCount > 0) {
+ hd.getHarvestingDataverseConfig().setLastNonEmptyHarvestTime(currentTime);
+ hd.getHarvestingDataverseConfig().setHarvestedDatasetCount(new Long(harvestedCount));
+ hd.getHarvestingDataverseConfig().setFailedDatasetCount(new Long(failedCount));
+ /*TODO: record the number of deleted datasets! */
+ }
}
}
-
+/*
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
public void setHarvestSuccessNotEmpty(Long hdId, Date currentTime, int harvestedCount, int failedCount) {
Dataverse hd = em.find(Dataverse.class, hdId);
@@ -566,22 +570,17 @@ public void setHarvestSuccessNotEmpty(Long hdId, Date currentTime, int harvested
hd.getHarvestingDataverseConfig().setLastSuccessfulNonZeroHarvestTime(currentTime);
hd.getHarvestingDataverseConfig().setHarvestedStudyCountNonZero(new Long(harvestedCount));
hd.getHarvestingDataverseConfig().setFailedStudyCountNonZero(new Long(failedCount));
- */
+ *
}
- }
+ }*/
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestFailure(Long hdId, int harvestedStudyCount, int failedCount) {
+ public void setHarvestFailure(Long hdId, Date currentTime) {
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- /* TODO:
- hd.getHarvestingDataverseConfig().setHarvestedStudyCount(new Long(harvestedStudyCount));
- hd.getHarvestingDataverseConfig().setFailedStudyCount(new Long(failedCount));
- */
+ hd.getHarvestingDataverseConfig().setLastHarvestTime(currentTime);
hd.getHarvestingDataverseConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_FAILED);
}
-
- }
-
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
index 6ded994902d..9d2537375e1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
@@ -168,6 +168,10 @@ public void setHarvestResult(String harvestResult) {
this.harvestResult = harvestResult;
}
+ // "Last Harvest Time" is the last time we *attempted* to harvest
+ // from this remote resource.
+ // It wasn't necessarily a successful attempt!
+
@Temporal(value = TemporalType.TIMESTAMP)
private Date lastHarvestTime;
@@ -179,6 +183,12 @@ public void setLastHarvestTime(Date lastHarvestTime) {
this.lastHarvestTime = lastHarvestTime;
}
+ // This the last "successful harvest" - i.e., the last time we
+ // tried to harvest, and got a response from the remote server.
+ // We may not have necessarily harvested any useful content though;
+ // the result may have been a "no content" or "no changes since the last harvest"
+ // response.
+
@Temporal(value = TemporalType.TIMESTAMP)
private Date lastSuccessfulHarvestTime;
@@ -190,12 +200,48 @@ public void setLastSuccessfulHarvestTime(Date lastSuccessfulHarvestTime) {
this.lastSuccessfulHarvestTime = lastSuccessfulHarvestTime;
}
+ // Finally, this is the time stamp from the last "non-empty" harvest.
+ // I.e. the last time we ran a harvest that actually resulted in
+ // some Datasets created, updated or deleted:
+
+ private Date lastNonEmptyHarvestTime;
+ @Temporal(value = TemporalType.TIMESTAMP)
+ public Date getLastNonEmptyHarvestTime() {
+ return lastNonEmptyHarvestTime;
+ }
+
+ public void setLastNonEmptyHarvestTime(Date lastNonEmptyHarvestTime) {
+ this.lastNonEmptyHarvestTime = lastNonEmptyHarvestTime;
+ }
+
+ // And these are the Dataset counts from that last "non-empty" harvest:
private Long harvestedDatasetCount;
private Long failedDatasetCount;
- @Temporal(value = TemporalType.TIMESTAMP)
- private Date lastSuccessfulNonEmptyHarvestTime;
- private Long lastNonEmptyHarvestedDatasetCount;
- private Long lastNonEmptyFailedDatasetCount;
+ private Long deletedDatasetCount;
+
+ public Long getHarvestedDatasetCount() {
+ return harvestedDatasetCount;
+ }
+
+ public void setHarvestedDatasetCount(Long harvestedDatasetCount) {
+ this.harvestedDatasetCount = harvestedDatasetCount;
+ }
+
+ public Long getFailedDatasetCount() {
+ return failedDatasetCount;
+ }
+
+ public void setFailedDatasetCount(Long failedDatasetCount) {
+ this.failedDatasetCount = failedDatasetCount;
+ }
+
+ public Long getDeletedDatasetCount() {
+ return deletedDatasetCount;
+ }
+
+ public void setDeletedDatasetCount(Long deletedDatasetCount) {
+ this.deletedDatasetCount = deletedDatasetCount;
+ }
private boolean scheduled;
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java
index 7bd0635cf7f..d41fed2652d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/BatchServiceBean.java
@@ -5,7 +5,6 @@
import edu.harvard.iq.dataverse.DataverseServiceBean;
import edu.harvard.iq.dataverse.api.imports.ImportException;
import edu.harvard.iq.dataverse.api.imports.ImportUtil;
-import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import java.io.File;
import java.io.FileWriter;
@@ -107,5 +106,6 @@ public JsonArrayBuilder handleDirectory(DataverseRequest dataverseRequest, File
}
return status;
}
+
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
new file mode 100644
index 00000000000..7d2b91af87f
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
@@ -0,0 +1,65 @@
+package edu.harvard.iq.dataverse.api;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseServiceBean;
+
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
+import java.io.IOException;
+import javax.ejb.EJB;
+import javax.ejb.Stateless;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+
+@Stateless
+@Path("harvest")
+public class Harvesting extends AbstractApiBean {
+
+
+ @EJB
+ DataverseServiceBean dataverseService;
+ @EJB
+ HarvesterServiceBean harvesterService;
+
+
+ @GET
+ @Path("run/{dataverseAlias}")
+ public Response startHarvestingJob(@PathParam("dataverseAlias") String dataverseAlias, @QueryParam("key") String apiKey) throws IOException {
+
+ try {
+ AuthenticatedUser authenticatedUser = null;
+
+ try {
+ authenticatedUser = findAuthenticatedUserOrDie();
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+
+ if (authenticatedUser == null || !authenticatedUser.isSuperuser()) {
+ return errorResponse(Response.Status.FORBIDDEN, "Only the Dataverse Admin user can run harvesting jobs");
+ }
+
+ Dataverse dataverse = dataverseService.findByAlias(dataverseAlias);
+
+ if (dataverse == null) {
+ return errorResponse(Response.Status.NOT_FOUND, "No such dataverse: "+dataverseAlias);
+ }
+
+ if (!dataverse.isHarvested()) {
+ return errorResponse(Response.Status.BAD_REQUEST, "Not a HARVESTING dataverse: "+dataverseAlias);
+ }
+
+ //DataverseRequest dataverseRequest = createDataverseRequest(authenticatedUser);
+
+ harvesterService.doAsyncHarvest(dataverse);
+
+ } catch (Exception e) {
+ return this.errorResponse(Response.Status.BAD_REQUEST, "Exception thrown when running a Harvest on dataverse \""+dataverseAlias+"\" via REST API; " + e.getMessage());
+ }
+ return this.accepted();
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index 5f7cfb98e4a..deb33234916 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -29,6 +29,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Resource;
+import javax.ejb.Asynchronous;
import javax.ejb.EJB;
import javax.ejb.EJBException;
import javax.ejb.Stateless;
@@ -86,21 +87,15 @@ public HarvesterServiceBean() {
/**
* Called to run an "On Demand" harvest.
- * This method creates a timer that will go off immediately,
- * which will start an immediate asynchronous harvest.
- * @param dataverse
*/
- public void doAsyncHarvest(Dataverse harvestedDataverse) {
- HarvestingDataverseConfig harvestedDataverseConfig = harvestedDataverse.getHarvestingDataverseConfig();
+ @Asynchronous
+ public void doAsyncHarvest(Dataverse harvestingDataverse) {
- if (harvestedDataverseConfig == null) {
- logger.info("ERROR: No Harvesting Configuration found for dataverse id="+harvestedDataverse.getId());
- return;
+ try {
+ doHarvest(harvestingDataverse.getId());
+ } catch (Exception e) {
+ logger.info("Caught exception running an asynchronous harvest (dataverse \""+harvestingDataverse.getAlias()+"\")");
}
-
- Calendar cal = Calendar.getInstance();
-
- timerService.createTimer(cal.getTime(), new HarvestTimerInfo(harvestedDataverse.getId(), harvestedDataverse.getName(), harvestedDataverseConfig.getSchedulePeriod(), harvestedDataverseConfig.getScheduleHourOfDay(), harvestedDataverseConfig.getScheduleDayOfWeek()));
}
public void createScheduledHarvestTimers() {
@@ -176,50 +171,12 @@ private void createHarvestTimer(Dataverse harvestingDataverse) {
dataverseTimerService.createTimer(initExpirationDate, intervalDuration, new HarvestTimerInfo(harvestingDataverse.getId(), harvestingDataverse.getName(), harvestingDataverseConfig.getSchedulePeriod(), harvestingDataverseConfig.getScheduleHourOfDay(), harvestingDataverseConfig.getScheduleDayOfWeek()));
}
}
-
- /**
- * This method is called whenever an EJB Timer goes off.
- * Check to see if this is a Harvest Timer, and if it is
- * Run the harvest for the given (scheduled) dataverse
- * @param timer
- */
- @Timeout
- @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
- public void handleTimeout(javax.ejb.Timer timer) {
- // We have to put all the code in a try/catch block because
- // if an exception is thrown from this method, Glassfish will automatically
- // call the method a second time. (The minimum number of re-tries for a Timer method is 1)
-
- if (timer.getInfo() instanceof HarvestTimerInfo) {
- HarvestTimerInfo info = (HarvestTimerInfo) timer.getInfo();
- try {
- // First, check if we are in read-only mode:
- /*
- if (...) {
- logger.log(Level.ALL, "Dataverse is in read-only mode.");
- return;
-
- }
- */
-
- // Proceeding with the scheduled harvest:
-
- logger.log(Level.INFO, "DO HARVESTING of dataverse " + info.getHarvestingDataverseId());
- doHarvesting(info.getHarvestingDataverseId());
-
- } catch (Throwable e) {
- dataverseService.setHarvestResult(info.getHarvestingDataverseId(), this.HARVEST_RESULT_FAILED);
- /*mailService.sendHarvestErrorNotification(...getSystemEmail(), ...);*/
- logException(e, logger);
- }
- }
- }
/**
- * Harvest an individual Dataverse
+ * Run a harvest for an individual harvesting Dataverse
* @param dataverseId
*/
- public void doHarvesting(Long dataverseId) throws IOException {
+ public void doHarvest(Long dataverseId) throws IOException {
Dataverse harvestingDataverse = dataverseService.find(dataverseId);
if (harvestingDataverse == null) {
@@ -244,9 +201,11 @@ public void doHarvesting(Long dataverseId) throws IOException {
this.harvestedDatasetIdsThisBatch = new ArrayList();
List failedIdentifiers = new ArrayList();
+ Date harvestStartTime = new Date();
+
try {
boolean harvestingNow = harvestingDataverseConfig.isHarvestingNow();
-
+
if (harvestingNow) {
harvestErrorOccurred.setValue(true);
hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested.");
@@ -259,29 +218,24 @@ public void doHarvesting(Long dataverseId) throws IOException {
if (lastSuccessfulHarvestTime != null) {
from = formatter.format(lastSuccessfulHarvestTime);
}
- dataverseService.setHarvestInProgress(harvestingDataverse.getId(), true);
- Date currentTime = new Date();
- dataverseService.setLastHarvestTime(harvestingDataverse.getId(), currentTime);
-
- hdLogger.log(Level.INFO, "BEGIN HARVEST..., oaiUrl=" + harvestingDataverseConfig.getArchiveUrl() + ",set=" + harvestingDataverseConfig.getHarvestingSet() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix() + ", from=" + from + ", until=" + until);
+ dataverseService.setHarvestInProgress(harvestingDataverse.getId(), true);
- if (harvestingDataverseConfig.isOai()) {
- harvestedDatasetIds = harvestOAI(harvestingDataverse, hdLogger, from, until, harvestErrorOccurred, failedIdentifiers);
+ dataverseService.setLastHarvestTime(harvestingDataverse.getId(), harvestStartTime);
- } else {
- throw new IOException("Unsupported harvest type");
- }
- dataverseService.setHarvestSuccess(harvestingDataverse.getId(),currentTime, harvestedDatasetIds.size(), failedIdentifiers.size());
- hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingDataverseConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix());
+ hdLogger.log(Level.INFO, "BEGIN HARVEST..., oaiUrl=" + harvestingDataverseConfig.getArchiveUrl() + ",set=" + harvestingDataverseConfig.getHarvestingSet() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix() + ", from=" + from + ", until=" + until);
- /* Last "non-empty" harvest: */
- if (harvestedDatasetIds.size() > 0) {
- dataverseService.setHarvestSuccessNotEmpty(harvestingDataverse.getId(),currentTime, harvestedDatasetIds.size(), failedIdentifiers.size());
- hdLogger.log(Level.INFO, "COMPLETED HARVEST with results");
- }
-
- // now index all studies (need to modify for update)
- /* (TODO: !!!)
+ if (harvestingDataverseConfig.isOai()) {
+ harvestedDatasetIds = harvestOAI(harvestingDataverse, hdLogger, from, until, harvestErrorOccurred, failedIdentifiers);
+
+ } else {
+ throw new IOException("Unsupported harvest type");
+ }
+ dataverseService.setHarvestSuccess(harvestingDataverse.getId(), harvestStartTime, harvestedDatasetIds.size(), failedIdentifiers.size());
+ hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingDataverseConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix());
+ hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: [TODO:], datasets failed: " + failedIdentifiers.size());
+
+ // now index all studies (need to modify for update)
+ /* (TODO: !!!)
if (this.processedSizeThisBatch > 0) {
hdLogger.log(Level.INFO, "POST HARVEST, reindexing the remaining studies.");
if (this.harvestedDatasetIdsThisBatch != null) {
@@ -293,18 +247,17 @@ public void doHarvesting(Long dataverseId) throws IOException {
} else {
hdLogger.log(Level.INFO, "(All harvested content already reindexed)");
}
- */
+ */
}
//mailService.sendHarvestNotification(...getSystemEmail(), harvestingDataverse.getName(), logFileName, logTimestamp, harvestErrorOccurred.booleanValue(), harvestedDatasetIds.size(), failedIdentifiers);
- } catch (Throwable e) {
+ } catch (Throwable e) {
harvestErrorOccurred.setValue(true);
String message = "Exception processing harvest, server= " + harvestingDataverseConfig.getArchiveUrl() + ",format=" + harvestingDataverseConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage();
hdLogger.log(Level.SEVERE, message);
logException(e, hdLogger);
hdLogger.log(Level.INFO, "HARVEST NOT COMPLETED DUE TO UNEXPECTED ERROR.");
- dataverseService.setHarvestFailure(harvestingDataverse.getId(), harvestedDatasetIds.size(), failedIdentifiers.size());
-
-
+ dataverseService.setHarvestFailure(harvestingDataverse.getId(), harvestStartTime);
+
} finally {
dataverseService.setHarvestInProgress(harvestingDataverse.getId(), false);
fileHandler.close();
diff --git a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
index 6a4159d11a8..106eefa5753 100644
--- a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
@@ -81,7 +81,7 @@ public void handleTimeout(javax.ejb.Timer timer) {
try {
logger.log(Level.INFO, "DO HARVESTING of dataverse " + info.getHarvestingDataverseId());
- harvesterService.doHarvesting(info.getHarvestingDataverseId());
+ harvesterService.doHarvest(info.getHarvestingDataverseId());
} catch (Throwable e) {
dataverseService.setHarvestResult(info.getHarvestingDataverseId(), harvesterService.HARVEST_RESULT_FAILED);
From b6845e067f34fd39c9ddf9674d6acd548296c05a Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Tue, 19 Apr 2016 16:43:35 -0400
Subject: [PATCH 06/37] Removed unused imports from the harvester service bean.
---
.../harvest/client/HarvesterServiceBean.java | 14 +-------------
1 file changed, 1 insertion(+), 13 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index deb33234916..1ee6009c975 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -12,13 +12,8 @@
import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
import edu.harvard.iq.dataverse.timer.DataverseTimerServiceBean;
import edu.harvard.iq.dataverse.util.FileUtil;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.Writer;
-import java.net.MalformedURLException;
-import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
@@ -33,21 +28,15 @@
import javax.ejb.EJB;
import javax.ejb.EJBException;
import javax.ejb.Stateless;
-import javax.ejb.Timeout;
import javax.ejb.Timer;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.faces.bean.ManagedBean;
import javax.inject.Named;
-import javax.persistence.Query;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBElement;
-import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import org.apache.commons.lang.mutable.MutableBoolean;
-import org.w3c.dom.Document;
import org.xml.sax.SAXException;
/**
@@ -76,7 +65,6 @@ public class HarvesterServiceBean {
public static final String HARVEST_RESULT_FAILED="failed";
- private JAXBContext jaxbContext;
private Unmarshaller unmarshaller;
private long processedSizeThisBatch = 0;
@@ -274,7 +262,7 @@ public void doHarvest(Long dataverseId) throws IOException {
* @param failedIdentifiers Study Identifiers for failed "GetRecord" requests
*/
private List harvestOAI(Dataverse dataverse, Logger hdLogger, String from, String until, MutableBoolean harvestErrorOccurred, List failedIdentifiers)
- throws IOException, ParserConfigurationException,SAXException, TransformerException, JAXBException {
+ throws IOException, ParserConfigurationException,SAXException, TransformerException {
List harvestedDatasetIds = new ArrayList();
From 340b944e688ba81e9cab093d639d4d056f8dd4a3 Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Thu, 21 Apr 2016 21:49:30 -0400
Subject: [PATCH 07/37] new /api/harvest/status method with JSON output.
---
.../dataverse/HarvestingDataverseConfig.java | 26 ++++++--
.../harvard/iq/dataverse/api/Harvesting.java | 60 ++++++++++++++++++-
2 files changed, 79 insertions(+), 7 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
index 9d2537375e1..9ef7a49b802 100644
--- a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
@@ -27,15 +27,17 @@
*
* @author Leonid Andreev
*/
-@Entity
+
@Table(indexes = {@Index(columnList="dataverse_id")
, @Index(columnList="harvesttype")
, @Index(columnList="harveststyle")
, @Index(columnList="harvestingurl")})
+@Entity
public class HarvestingDataverseConfig implements Serializable {
private static final long serialVersionUID = 1L;
+
@Id
- @GeneratedValue(strategy = GenerationType.AUTO)
+ @GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
public Long getId() {
@@ -45,7 +47,7 @@ public Long getId() {
public void setId(Long id) {
this.id = id;
}
-
+
public static final String HARVEST_TYPE_OAI="oai";
public static final String HARVEST_TYPE_NESSTAR="nesstar";
@@ -83,7 +85,17 @@ public void setDataverse(Dataverse dataverse) {
this.dataverse = dataverse;
}
- String harvestType;
+ private String name;
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ private String harvestType;
public String getHarvestType() {
return harvestType;
@@ -93,11 +105,12 @@ public void setHarvestType(String harvestType) {
this.harvestType = harvestType;
}
+
public boolean isOai() {
return HARVEST_TYPE_OAI.equals(harvestType);
}
- String harvestStyle;
+ private String harvestStyle;
public String getHarvestStyle() {
return harvestStyle;
@@ -204,8 +217,9 @@ public void setLastSuccessfulHarvestTime(Date lastSuccessfulHarvestTime) {
// I.e. the last time we ran a harvest that actually resulted in
// some Datasets created, updated or deleted:
- private Date lastNonEmptyHarvestTime;
@Temporal(value = TemporalType.TIMESTAMP)
+ private Date lastNonEmptyHarvestTime;
+
public Date getLastNonEmptyHarvestTime() {
return lastNonEmptyHarvestTime;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
index 7d2b91af87f..818bbef4bbc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
@@ -2,12 +2,20 @@
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseServiceBean;
+import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
+import javax.json.JsonObjectBuilder;
+import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
+import edu.harvard.iq.dataverse.util.json.JsonPrinter;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
import javax.ejb.EJB;
import javax.ejb.Stateless;
+import javax.json.Json;
+import javax.json.JsonArrayBuilder;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -35,7 +43,7 @@ public Response startHarvestingJob(@PathParam("dataverseAlias") String dataverse
try {
authenticatedUser = findAuthenticatedUserOrDie();
} catch (WrappedResponse wr) {
- return wr.getResponse();
+ return errorResponse(Response.Status.UNAUTHORIZED, "Authentication required to use this API method");
}
if (authenticatedUser == null || !authenticatedUser.isSuperuser()) {
@@ -61,5 +69,55 @@ public Response startHarvestingJob(@PathParam("dataverseAlias") String dataverse
}
return this.accepted();
}
+
+ /*
+ * /api/harvest/status
+ * will, by default, return a JSON record with the information about the
+ * configured remote archives.
+ * optionally, plain text output will [/may] be provided as well.
+ */
+ @GET
+ @Path("status")
+ public Response harvestingStatus() throws IOException {
+ //return this.accepted();
+
+ List harvestingDataverses = dataverseService.getAllHarvestedDataverses();
+ if (harvestingDataverses == null) {
+ return okResponse("");
+ }
+
+ return okResponse(jsonObjectBuilder().add("remoteArchives", harvestingConfigsAsJsonArray(harvestingDataverses)));
+ }
+ public static JsonArrayBuilder harvestingConfigsAsJsonArray(List harvestingDataverses) {
+ JsonArrayBuilder hdArr = Json.createArrayBuilder();
+
+ for (Dataverse hd : harvestingDataverses) {
+ hdArr.add(harvestingConfigAsJson(hd));
+ }
+ return hdArr;
+ }
+
+ public static JsonObjectBuilder harvestingConfigAsJson(Dataverse dataverse) {
+ HarvestingDataverseConfig harvestingConfig = dataverse.getHarvestingDataverseConfig();
+ if (harvestingConfig == null) {
+ return null;
+ }
+
+ return jsonObjectBuilder().add("nickname", harvestingConfig.getName()).
+ add("dataverseAlias", dataverse.getAlias()).
+ add("type", harvestingConfig.getHarvestType()).
+ add("harvestURL", harvestingConfig.getHarvestingUrl()).
+ add("metadataFormat", harvestingConfig.getMetadataPrefix()).
+ add("set", harvestingConfig.getHarvestingSet() == null ? "N/A" : harvestingConfig.getHarvestingSet()).
+ add("schedule", harvestingConfig.isScheduled() ? harvestingConfig.getScheduleDescription() : "none").
+ add("inProgress", harvestingConfig.isHarvestingNow() ? "yes" : "-").
+ add("lastHarvest", harvestingConfig.getLastHarvestTime() == null ? "N/A" : harvestingConfig.getLastHarvestTime().toString()).
+ add("lastSuccessful", harvestingConfig.getLastSuccessfulHarvestTime() == null ? "N/A" : harvestingConfig.getLastSuccessfulHarvestTime().toString()).
+ add("lastNonEmpty", harvestingConfig.getLastNonEmptyHarvestTime() == null ? "N/A" : harvestingConfig.getLastNonEmptyHarvestTime().toString()).
+ add("lastResult", harvestingConfig.getHarvestResult()).
+ add("datasetsHarveted", harvestingConfig.getHarvestedDatasetCount() == null ? "N/A" : harvestingConfig.getHarvestedDatasetCount().toString()).
+ add("datasetsDeleted", harvestingConfig.getDeletedDatasetCount() == null ? "N/A" : harvestingConfig.getDeletedDatasetCount().toString()).
+ add("datasetsFailed", harvestingConfig.getFailedDatasetCount() == null ? "N/A" : harvestingConfig.getFailedDatasetCount().toString());
+ }
}
From e4b90e5ec078655ba18737a7b6a2599c13efbf8c Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Tue, 26 Apr 2016 19:16:01 -0400
Subject: [PATCH 08/37] Refactored the code, replacing
"HarvestingDataverseConfig" with "HarvestingClient".
---
.../edu/harvard/iq/dataverse/Dataset.java | 29 ++--
.../harvard/iq/dataverse/DatasetVersion.java | 7 +-
.../edu/harvard/iq/dataverse/Dataverse.java | 13 +-
.../iq/dataverse/DataverseServiceBean.java | 30 ++--
.../harvard/iq/dataverse/api/Datasets.java | 3 +
.../harvard/iq/dataverse/api/Dataverses.java | 3 +
.../harvard/iq/dataverse/api/Harvesting.java | 4 +-
.../impl/CreateHarvestingClientCommand.java | 33 +++++
.../impl/DeleteHarvestingClientCommand.java | 38 +++++
.../impl/UpdateHarvestingClientCommand.java | 33 +++++
.../harvest/client/HarvesterServiceBean.java | 9 +-
.../client/HarvestingClient.java} | 13 +-
.../harvest/client/HarvestingClientRun.java | 132 ++++++++++++++++++
.../client/HarvestingClientServiceBean.java | 24 ++++
.../timer/DataverseTimerServiceBean.java | 6 +-
15 files changed, 324 insertions(+), 53 deletions(-)
create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateHarvestingClientCommand.java
create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteHarvestingClientCommand.java
create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestingClientCommand.java
rename src/main/java/edu/harvard/iq/dataverse/{HarvestingDataverseConfig.java => harvest/client/HarvestingClient.java} (96%)
create mode 100644 src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java
create mode 100644 src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 6b019ab9a4a..3c0fd8e70d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -1,5 +1,6 @@
package edu.harvard.iq.dataverse;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
@@ -521,24 +522,24 @@ public boolean isHarvested() {
public String getRemoteArchiveURL() {
if (isHarvested()) {
- if (HarvestingDataverseConfig.HARVEST_STYLE_DATAVERSE.equals(this.getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
- return this.getOwner().getHarvestingDataverseConfig().getArchiveUrl() + "/dataset.xhtml?persistentId=" + getGlobalId();
- } else if (HarvestingDataverseConfig.HARVEST_STYLE_VDC.equals(this.getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
- String rootArchiveUrl = this.getOwner().getHarvestingDataverseConfig().getHarvestingUrl();
+ if (HarvestingClient.HARVEST_STYLE_DATAVERSE.equals(this.getOwner().getHarvestingClientConfig().getHarvestStyle())) {
+ return this.getOwner().getHarvestingClientConfig().getArchiveUrl() + "/dataset.xhtml?persistentId=" + getGlobalId();
+ } else if (HarvestingClient.HARVEST_STYLE_VDC.equals(this.getOwner().getHarvestingClientConfig().getHarvestStyle())) {
+ String rootArchiveUrl = this.getOwner().getHarvestingClientConfig().getHarvestingUrl();
int c = rootArchiveUrl.indexOf("/OAIHandler");
if (c > 0) {
rootArchiveUrl = rootArchiveUrl.substring(0, c);
return rootArchiveUrl + "/faces/study/StudyPage.xhtml?globalId=" + getGlobalId();
}
- } else if (HarvestingDataverseConfig.HARVEST_STYLE_ICPSR.equals(this.getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
+ } else if (HarvestingClient.HARVEST_STYLE_ICPSR.equals(this.getOwner().getHarvestingClientConfig().getHarvestStyle())) {
// For the ICPSR, it turns out that the best thing to do is to
// rely on the DOI to send the user to the right landing page for
// the study:
//String icpsrId = identifier;
- //return this.getOwner().getHarvestingDataverseConfig().getArchiveUrl() + "/icpsrweb/ICPSR/studies/"+icpsrId+"?q="+icpsrId+"&searchSource=icpsr-landing";
+ //return this.getOwner().getHarvestingClientConfig().getArchiveUrl() + "/icpsrweb/ICPSR/studies/"+icpsrId+"?q="+icpsrId+"&searchSource=icpsr-landing";
return "http://doi.org/" + authority + "/" + identifier;
- } else if (HarvestingDataverseConfig.HARVEST_STYLE_NESSTAR.equals(this.getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
- String nServerURL = this.getOwner().getHarvestingDataverseConfig().getArchiveUrl();
+ } else if (HarvestingClient.HARVEST_STYLE_NESSTAR.equals(this.getOwner().getHarvestingClientConfig().getHarvestStyle())) {
+ String nServerURL = this.getOwner().getHarvestingClientConfig().getArchiveUrl();
// chop any trailing slashes in the server URL - or they will result
// in multiple slashes in the final URL pointing to the study
// on server of origin; Nesstar doesn't like it, apparently.
@@ -556,9 +557,9 @@ public String getRemoteArchiveURL() {
+ "&top=yes";
return NesstarWebviewPage;
- } else if (HarvestingDataverseConfig.HARVEST_STYLE_ROPER.equals(this.getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
- return this.getOwner().getHarvestingDataverseConfig().getArchiveUrl() + "/CFIDE/cf/action/catalog/abstract.cfm?archno=" + identifier;
- } else if (HarvestingDataverseConfig.HARVEST_STYLE_HGL.equals(this.getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
+ } else if (HarvestingClient.HARVEST_STYLE_ROPER.equals(this.getOwner().getHarvestingClientConfig().getHarvestStyle())) {
+ return this.getOwner().getHarvestingClientConfig().getArchiveUrl() + "/CFIDE/cf/action/catalog/abstract.cfm?archno=" + identifier;
+ } else if (HarvestingClient.HARVEST_STYLE_HGL.equals(this.getOwner().getHarvestingClientConfig().getHarvestStyle())) {
// a bit of a hack, true.
// HGL documents, when turned into Dataverse studies/datasets
// all 1 datafile; the location ("storage identifier") of the file
@@ -574,9 +575,9 @@ public String getRemoteArchiveURL() {
}
}
}
- return this.getOwner().getHarvestingDataverseConfig().getArchiveUrl();
+ return this.getOwner().getHarvestingClientConfig().getArchiveUrl();
}else {
- return this.getOwner().getHarvestingDataverseConfig().getArchiveUrl();
+ return this.getOwner().getHarvestingClientConfig().getArchiveUrl();
}
}
@@ -585,7 +586,7 @@ public String getRemoteArchiveURL() {
public String getHarvestingDescription() {
if (isHarvested()) {
- return this.getOwner().getHarvestingDataverseConfig().getArchiveDescription();
+ return this.getOwner().getHarvestingClientConfig().getArchiveDescription();
}
return null;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
index 311851eda6b..2d61c2ef53e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
@@ -1,5 +1,6 @@
package edu.harvard.iq.dataverse;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.util.MarkupChecker;
import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
import edu.harvard.iq.dataverse.util.StringUtil;
@@ -811,9 +812,9 @@ public String getCitation(boolean isOnlineVersion) {
// It is always part of the citation for the local datasets;
// And for *some* harvested datasets.
if (!this.getDataset().isHarvested()
- || HarvestingDataverseConfig.HARVEST_STYLE_VDC.equals(this.getDataset().getOwner().getHarvestingDataverseConfig().getHarvestStyle())
- || HarvestingDataverseConfig.HARVEST_STYLE_ICPSR.equals(this.getDataset().getOwner().getHarvestingDataverseConfig().getHarvestStyle())
- || HarvestingDataverseConfig.HARVEST_STYLE_DATAVERSE.equals(this.getDataset().getOwner().getHarvestingDataverseConfig().getHarvestStyle())) {
+ || HarvestingClient.HARVEST_STYLE_VDC.equals(this.getDataset().getOwner().getHarvestingClientConfig().getHarvestStyle())
+ || HarvestingClient.HARVEST_STYLE_ICPSR.equals(this.getDataset().getOwner().getHarvestingClientConfig().getHarvestStyle())
+ || HarvestingClient.HARVEST_STYLE_DATAVERSE.equals(this.getDataset().getOwner().getHarvestingClientConfig().getHarvestStyle())) {
if (!StringUtil.isEmpty(this.getDataset().getIdentifier())) {
if (!StringUtil.isEmpty(str)) {
str += ", ";
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index b97d3402f81..c3580f807fe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -1,5 +1,6 @@
package edu.harvard.iq.dataverse;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.authorization.DataverseRole;
import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch;
import java.util.ArrayList;
@@ -277,18 +278,18 @@ public void setGuestbooks(List guestbooks) {
}
@OneToOne (mappedBy="dataverse", cascade={CascadeType.PERSIST, CascadeType.REMOVE})
- private HarvestingDataverseConfig harvestingDataverseConfig;
+ private HarvestingClient harvestingClient;
- public HarvestingDataverseConfig getHarvestingDataverseConfig() {
- return this.harvestingDataverseConfig;
+ public HarvestingClient getHarvestingClientConfig() {
+ return this.harvestingClient;
}
- public void setHarvestingDataverseConfig(HarvestingDataverseConfig harvestingDataverseConfig) {
- this.harvestingDataverseConfig = harvestingDataverseConfig;
+ public void setHarvestingClientConfig(HarvestingClient harvestingClient) {
+ this.harvestingClient = harvestingClient;
}
public boolean isHarvested() {
- return harvestingDataverseConfig != null;
+ return harvestingClient != null;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 33ce58ff250..4a95f79e065 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -511,7 +511,7 @@ public void setHarvestResult(Long hdId, String result) {
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- hd.getHarvestingDataverseConfig().setHarvestResult(result);
+ hd.getHarvestingClientConfig().setHarvestResult(result);
}
}
@@ -520,7 +520,7 @@ public void resetHarvestingStatus(Long hdId) {
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- hd.getHarvestingDataverseConfig().setHarvestingNow(false);
+ hd.getHarvestingClientConfig().setHarvestingNow(false);
}
}
@@ -530,7 +530,7 @@ public void setHarvestInProgress(Long hdId, boolean inProgress) {
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- hd.getHarvestingDataverseConfig().setHarvestingNow(inProgress);
+ hd.getHarvestingClientConfig().setHarvestingNow(inProgress);
}
}
@@ -539,7 +539,7 @@ public void setLastHarvestTime(Long hdId, Date lastHarvestTime) {
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- hd.getHarvestingDataverseConfig().setLastHarvestTime(lastHarvestTime);
+ hd.getHarvestingClientConfig().setLastHarvestTime(lastHarvestTime);
}
}
@@ -548,14 +548,14 @@ public void setHarvestSuccess(Long hdId, Date currentTime, int harvestedCount, i
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- hd.getHarvestingDataverseConfig().setLastHarvestTime(currentTime);
- hd.getHarvestingDataverseConfig().setLastSuccessfulHarvestTime(currentTime);
- hd.getHarvestingDataverseConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_SUCCESS);
+ hd.getHarvestingClientConfig().setLastHarvestTime(currentTime);
+ hd.getHarvestingClientConfig().setLastSuccessfulHarvestTime(currentTime);
+ hd.getHarvestingClientConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_SUCCESS);
if (harvestedCount > 0 || failedCount > 0) {
- hd.getHarvestingDataverseConfig().setLastNonEmptyHarvestTime(currentTime);
- hd.getHarvestingDataverseConfig().setHarvestedDatasetCount(new Long(harvestedCount));
- hd.getHarvestingDataverseConfig().setFailedDatasetCount(new Long(failedCount));
+ hd.getHarvestingClientConfig().setLastNonEmptyHarvestTime(currentTime);
+ hd.getHarvestingClientConfig().setHarvestedDatasetCount(new Long(harvestedCount));
+ hd.getHarvestingClientConfig().setFailedDatasetCount(new Long(failedCount));
/*TODO: record the number of deleted datasets! */
}
}
@@ -567,9 +567,9 @@ public void setHarvestSuccessNotEmpty(Long hdId, Date currentTime, int harvested
em.refresh(hd);
if (hd.isHarvested()) {
/* TODO:
- hd.getHarvestingDataverseConfig().setLastSuccessfulNonZeroHarvestTime(currentTime);
- hd.getHarvestingDataverseConfig().setHarvestedStudyCountNonZero(new Long(harvestedCount));
- hd.getHarvestingDataverseConfig().setFailedStudyCountNonZero(new Long(failedCount));
+ hd.getHarvestingClientConfig().setLastSuccessfulNonZeroHarvestTime(currentTime);
+ hd.getHarvestingClientConfig().setHarvestedStudyCountNonZero(new Long(harvestedCount));
+ hd.getHarvestingClientConfig().setFailedStudyCountNonZero(new Long(failedCount));
*
}
}*/
@@ -579,8 +579,8 @@ public void setHarvestFailure(Long hdId, Date currentTime) {
Dataverse hd = em.find(Dataverse.class, hdId);
em.refresh(hd);
if (hd.isHarvested()) {
- hd.getHarvestingDataverseConfig().setLastHarvestTime(currentTime);
- hd.getHarvestingDataverseConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_FAILED);
+ hd.getHarvestingClientConfig().setLastHarvestTime(currentTime);
+ hd.getHarvestingClientConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_FAILED);
}
}
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 7f5f8a9533c..e8e77d7dc1f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -57,6 +57,9 @@
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import static edu.harvard.iq.dataverse.api.AbstractApiBean.errorResponse;
+import static edu.harvard.iq.dataverse.api.AbstractApiBean.errorResponse;
+import static edu.harvard.iq.dataverse.api.AbstractApiBean.errorResponse;
@Path("datasets")
public class Datasets extends AbstractApiBean {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index a787f3e26ee..ab8ce890332 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -79,6 +79,9 @@
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
/**
* A REST API for dataverses.
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
index 818bbef4bbc..c279201a0b7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
@@ -2,7 +2,7 @@
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseServiceBean;
-import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
@@ -99,7 +99,7 @@ public static JsonArrayBuilder harvestingConfigsAsJsonArray(List harv
}
public static JsonObjectBuilder harvestingConfigAsJson(Dataverse dataverse) {
- HarvestingDataverseConfig harvestingConfig = dataverse.getHarvestingDataverseConfig();
+ HarvestingClient harvestingConfig = dataverse.getHarvestingClientConfig();
if (harvestingConfig == null) {
return null;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateHarvestingClientCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateHarvestingClientCommand.java
new file mode 100644
index 00000000000..c4d09b2ef6e
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateHarvestingClientCommand.java
@@ -0,0 +1,33 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+
+/**
+ *
+ * @author Leonid Andreev
+ */
+@RequiredPermissions( Permission.EditDataverse )
+public class CreateHarvestingClientCommand extends AbstractCommand {
+
+ private final Dataverse dv;
+
+ public CreateHarvestingClientCommand(DataverseRequest aRequest, Dataverse motherDataverse) {
+ super(aRequest, motherDataverse);
+ dv = motherDataverse;
+ }
+
+ @Override
+ public HarvestingClient execute(CommandContext ctxt) throws CommandException {
+ // TODO: check if the harvesting client config is legit;
+ // and that it is new.
+ return ctxt.dataverses().save(dv).getHarvestingClientConfig();
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteHarvestingClientCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteHarvestingClientCommand.java
new file mode 100644
index 00000000000..38999fb45c8
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteHarvestingClientCommand.java
@@ -0,0 +1,38 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+
+/**
+ *
+ * @author Leonid Andreev
+ */
+@RequiredPermissions( Permission.EditDataverse )
+public class DeleteHarvestingClientCommand extends AbstractVoidCommand {
+
+ private final Dataverse motherDataverse;
+
+ public DeleteHarvestingClientCommand(DataverseRequest aRequest, Dataverse motherDataverse) {
+ super(aRequest, motherDataverse);
+ this.motherDataverse = motherDataverse;
+ }
+
+ @Override
+ public void executeImpl(CommandContext ctxt) throws CommandException {
+ HarvestingClient harvestingClient = motherDataverse.getHarvestingClientConfig();
+ if (harvestingClient == null) {
+ throw new IllegalCommandException("No harvesting client is configured for dataverse "+motherDataverse.getAlias(), this);
+ }
+ motherDataverse.setHarvestingClientConfig(null);
+ ctxt.em().remove(harvestingClient);
+ ctxt.em().merge(motherDataverse);
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestingClientCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestingClientCommand.java
new file mode 100644
index 00000000000..77c8bf75e57
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestingClientCommand.java
@@ -0,0 +1,33 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+
+/**
+ *
+ * @author Leonid Andreev
+ */
+@RequiredPermissions( Permission.EditDataverse )
+public class UpdateHarvestingClientCommand extends AbstractCommand {
+
+ private final Dataverse dv;
+
+ public UpdateHarvestingClientCommand(DataverseRequest aRequest, Dataverse motherDataverse) {
+ super(aRequest, motherDataverse);
+ dv = motherDataverse;
+ }
+
+ @Override
+ public HarvestingClient execute(CommandContext ctxt) throws CommandException {
+ // TODO: check if the harvesting client config attached to the dataverse
+ // is legit; and that it already exists.
+ return ctxt.em().merge(dv).getHarvestingClientConfig();
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index 1ee6009c975..e647f287db7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -9,7 +9,6 @@
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseServiceBean;
-import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
import edu.harvard.iq.dataverse.timer.DataverseTimerServiceBean;
import edu.harvard.iq.dataverse.util.FileUtil;
import java.io.File;
@@ -93,7 +92,7 @@ public void createScheduledHarvestTimers() {
List dataverses = dataverseService.getAllHarvestedDataverses();
for (Iterator it = dataverses.iterator(); it.hasNext();) {
Dataverse dataverse = (Dataverse) it.next();
- HarvestingDataverseConfig harvestingConfig = dataverse.getHarvestingDataverseConfig();
+ HarvestingClient harvestingConfig = dataverse.getHarvestingClientConfig();
if (harvestingConfig == null) {
logger.warning("ERROR: no harvesting config found for dataverse id="+dataverse.getId());
} else if (harvestingConfig.isScheduled()) {
@@ -125,7 +124,7 @@ public List getHarvestTimers() {
}
private void createHarvestTimer(Dataverse harvestingDataverse) {
- HarvestingDataverseConfig harvestingDataverseConfig = harvestingDataverse.getHarvestingDataverseConfig();
+ HarvestingClient harvestingDataverseConfig = harvestingDataverse.getHarvestingClientConfig();
if (harvestingDataverseConfig == null) {
logger.info("ERROR: No Harvesting Configuration found for dataverse id="+harvestingDataverse.getId());
@@ -171,7 +170,7 @@ public void doHarvest(Long dataverseId) throws IOException {
throw new IOException("No such Dataverse: id="+dataverseId);
}
- HarvestingDataverseConfig harvestingDataverseConfig = harvestingDataverse.getHarvestingDataverseConfig();
+ HarvestingClient harvestingDataverseConfig = harvestingDataverse.getHarvestingClientConfig();
if (harvestingDataverseConfig == null) {
throw new IOException("Could not find Harvesting Config for Dataverse id="+dataverseId);
@@ -373,7 +372,7 @@ public Long getRecord(HarvestingDataverse dataverse, String identifier, String m
public Long getRecord(Logger hdLogger, Dataverse dataverse, String identifier, String metadataPrefix, MutableBoolean recordErrorOccurred) {
String errMessage = null;
- HarvestingDataverseConfig harvestingConfig = dataverse.getHarvestingDataverseConfig();
+ HarvestingClient harvestingConfig = dataverse.getHarvestingClientConfig();
if (harvestingConfig == null) {
errMessage = "Could not find Harvesting Config for Dataverse id="+dataverse.getId();
diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
similarity index 96%
rename from src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
rename to src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
index 9ef7a49b802..98d9ac8b3cb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/HarvestingDataverseConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
@@ -3,8 +3,9 @@
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
-package edu.harvard.iq.dataverse;
+package edu.harvard.iq.dataverse.harvest.client;
+import edu.harvard.iq.dataverse.Dataverse;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
@@ -33,7 +34,7 @@
, @Index(columnList="harveststyle")
, @Index(columnList="harvestingurl")})
@Entity
-public class HarvestingDataverseConfig implements Serializable {
+public class HarvestingClient implements Serializable {
private static final long serialVersionUID = 1L;
@Id
@@ -67,7 +68,7 @@ public void setId(Long id) {
public static final String SCHEDULE_PERIOD_DAILY="daily";
public static final String SCHEDULE_PERIOD_WEEKLY="weekly";
- public HarvestingDataverseConfig() {
+ public HarvestingClient() {
this.harvestType = HARVEST_TYPE_OAI; // default harvestType
this.harvestStyle = HARVEST_STYLE_DATAVERSE; // default harvestStyle
}
@@ -171,6 +172,7 @@ public void setMetadataPrefix(String metadataPrefix) {
this.metadataPrefix = metadataPrefix;
}
+ /* move the fields below to the new HarvestingClientRun class: */
private String harvestResult;
public String getHarvestResult() {
@@ -256,6 +258,7 @@ public Long getDeletedDatasetCount() {
public void setDeletedDatasetCount(Long deletedDatasetCount) {
this.deletedDatasetCount = deletedDatasetCount;
}
+ /**/
private boolean scheduled;
@@ -336,10 +339,10 @@ public int hashCode() {
@Override
public boolean equals(Object object) {
// TODO: Warning - this method won't work in the case the id fields are not set
- if (!(object instanceof HarvestingDataverseConfig)) {
+ if (!(object instanceof HarvestingClient)) {
return false;
}
- HarvestingDataverseConfig other = (HarvestingDataverseConfig) object;
+ HarvestingClient other = (HarvestingClient) object;
if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
return false;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java
new file mode 100644
index 00000000000..404ace6cd42
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java
@@ -0,0 +1,132 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse.harvest.client;
+
+import java.io.Serializable;
+import java.util.Date;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+/**
+ *
+ * @author Leonid Andreev
+ *
+ * This is a record of an attempted harvesting client run.
+ * (Should it be named HarvestingClientRunResult instead?)
+ */
+@Entity
+public class HarvestingClientRun implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ @Id
+ @GeneratedValue(strategy = GenerationType.AUTO)
+ private Long id;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ private String harvestResult; // TODO: should this me an enum instead? -- L.A. 4.4
+
+ public String getHarvestResult() {
+ return harvestResult;
+ }
+
+ public void setHarvestResult(String harvestResult) {
+ this.harvestResult = harvestResult;
+ }
+
+ // Time of this harvest attempt:
+
+ @Temporal(value = TemporalType.TIMESTAMP)
+ private Date startTime;
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(Date startTime) {
+ this.startTime = startTime;
+ }
+
+ @Temporal(value = TemporalType.TIMESTAMP)
+ private Date finishTime;
+
+ public Date getFinishTime() {
+ return finishTime;
+ }
+
+ public void setFinishTime(Date finishTime) {
+ this.finishTime = finishTime;
+ }
+
+
+ // Tese are the Dataset counts from that last harvest:
+ // (TODO: do we need to differentiate between *created* (new), and *updated*
+ // harvested datasets? -- L.A. 4.4
+
+ private Long harvestedDatasetCount;
+ private Long failedDatasetCount;
+ private Long deletedDatasetCount;
+
+ public Long getHarvestedDatasetCount() {
+ return harvestedDatasetCount;
+ }
+
+ public void setHarvestedDatasetCount(Long harvestedDatasetCount) {
+ this.harvestedDatasetCount = harvestedDatasetCount;
+ }
+
+ public Long getFailedDatasetCount() {
+ return failedDatasetCount;
+ }
+
+ public void setFailedDatasetCount(Long failedDatasetCount) {
+ this.failedDatasetCount = failedDatasetCount;
+ }
+
+ public Long getDeletedDatasetCount() {
+ return deletedDatasetCount;
+ }
+
+ public void setDeletedDatasetCount(Long deletedDatasetCount) {
+ this.deletedDatasetCount = deletedDatasetCount;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 0;
+ hash += (id != null ? id.hashCode() : 0);
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ // TODO: Warning - this method won't work in the case the id fields are not set
+ if (!(object instanceof HarvestingClientRun)) {
+ return false;
+ }
+ HarvestingClientRun other = (HarvestingClientRun) object;
+ if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "edu.harvard.iq.dataverse.harvest.client.HarvestingClientRun[ id=" + id + " ]";
+ }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
new file mode 100644
index 00000000000..7783e64e8ff
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
@@ -0,0 +1,24 @@
+package edu.harvard.iq.dataverse.harvest.client;
+
+import edu.harvard.iq.dataverse.DataverseServiceBean;
+import java.util.logging.Logger;
+import javax.ejb.EJB;
+import javax.ejb.Stateless;
+import javax.faces.bean.ManagedBean;
+import javax.inject.Named;
+
+/**
+ *
+ * @author Leonid Andreev
+ *
+ * Dedicated service for managing Harvesting Client Configurations
+ */
+@Stateless(name = "harvesterService")
+@Named
+@ManagedBean
+public class HarvestingClientServiceBean {
+ @EJB
+ DataverseServiceBean dataverseService;
+
+ private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvestingClinetServiceBean");
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
index 106eefa5753..f438bd10c14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
@@ -7,7 +7,7 @@
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseServiceBean;
-import edu.harvard.iq.dataverse.HarvestingDataverseConfig;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.harvest.client.HarvestTimerInfo;
import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
import java.io.Serializable;
@@ -129,7 +129,7 @@ public void removeHarvestTimers() {
}
private void createHarvestTimer(Dataverse harvestedDataverse) {
- HarvestingDataverseConfig harvestedDataverseConfig = harvestedDataverse.getHarvestingDataverseConfig();
+ HarvestingClient harvestedDataverseConfig = harvestedDataverse.getHarvestingClientConfig();
if (harvestedDataverseConfig == null) {
logger.info("ERROR: No Harvesting Configuration found for dataverse id="+harvestedDataverse.getId());
@@ -141,7 +141,7 @@ private void createHarvestTimer(Dataverse harvestedDataverse) {
Calendar initExpiration = Calendar.getInstance();
initExpiration.set(Calendar.MINUTE, 0);
initExpiration.set(Calendar.SECOND, 0);
- if (harvestedDataverseConfig.getSchedulePeriod().equals(HarvestingDataverseConfig.SCHEDULE_PERIOD_DAILY)) {
+ if (harvestedDataverseConfig.getSchedulePeriod().equals(HarvestingClient.SCHEDULE_PERIOD_DAILY)) {
intervalDuration = 1000 * 60 * 60 * 24;
initExpiration.set(Calendar.HOUR_OF_DAY, harvestedDataverseConfig.getScheduleHourOfDay());
From 6315e1924a9470b3df4bd03e6016960c9b5d85fb Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Wed, 27 Apr 2016 16:10:31 -0400
Subject: [PATCH 09/37] more refactoring; added a Command for retrieving a
HarvstingClient; more fixes;
---
.../iq/dataverse/DataverseServiceBean.java | 78 ----------
.../harvard/iq/dataverse/api/Harvesting.java | 10 +-
.../impl/GetHarvestingClientCommand.java | 51 +++++++
...ngClientRun.java => ClientHarvestRun.java} | 104 ++++++++++----
.../harvest/client/HarvesterServiceBean.java | 41 +++---
.../harvest/client/HarvestingClient.java | 134 ++++++++++++++++--
.../client/HarvestingClientServiceBean.java | 114 ++++++++++++++-
.../timer/DataverseTimerServiceBean.java | 14 +-
8 files changed, 408 insertions(+), 138 deletions(-)
create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetHarvestingClientCommand.java
rename src/main/java/edu/harvard/iq/dataverse/harvest/client/{HarvestingClientRun.java => ClientHarvestRun.java} (56%)
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 4a95f79e065..003df01f87b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -505,82 +505,4 @@ public void populateDvSearchCard(SolrSearchResult solrSearchResult) {
}
}
}
-
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestResult(Long hdId, String result) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- hd.getHarvestingClientConfig().setHarvestResult(result);
- }
- }
-
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void resetHarvestingStatus(Long hdId) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- hd.getHarvestingClientConfig().setHarvestingNow(false);
- }
-
- }
-
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestInProgress(Long hdId, boolean inProgress) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- hd.getHarvestingClientConfig().setHarvestingNow(inProgress);
- }
- }
-
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setLastHarvestTime(Long hdId, Date lastHarvestTime) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- hd.getHarvestingClientConfig().setLastHarvestTime(lastHarvestTime);
- }
- }
-
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestSuccess(Long hdId, Date currentTime, int harvestedCount, int failedCount) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- hd.getHarvestingClientConfig().setLastHarvestTime(currentTime);
- hd.getHarvestingClientConfig().setLastSuccessfulHarvestTime(currentTime);
- hd.getHarvestingClientConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_SUCCESS);
-
- if (harvestedCount > 0 || failedCount > 0) {
- hd.getHarvestingClientConfig().setLastNonEmptyHarvestTime(currentTime);
- hd.getHarvestingClientConfig().setHarvestedDatasetCount(new Long(harvestedCount));
- hd.getHarvestingClientConfig().setFailedDatasetCount(new Long(failedCount));
- /*TODO: record the number of deleted datasets! */
- }
- }
- }
-/*
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestSuccessNotEmpty(Long hdId, Date currentTime, int harvestedCount, int failedCount) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- /* TODO:
- hd.getHarvestingClientConfig().setLastSuccessfulNonZeroHarvestTime(currentTime);
- hd.getHarvestingClientConfig().setHarvestedStudyCountNonZero(new Long(harvestedCount));
- hd.getHarvestingClientConfig().setFailedStudyCountNonZero(new Long(failedCount));
- *
- }
- }*/
-
- @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public void setHarvestFailure(Long hdId, Date currentTime) {
- Dataverse hd = em.find(Dataverse.class, hdId);
- em.refresh(hd);
- if (hd.isHarvested()) {
- hd.getHarvestingClientConfig().setLastHarvestTime(currentTime);
- hd.getHarvestingClientConfig().setHarvestResult(edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.HARVEST_RESULT_FAILED);
- }
- }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
index c279201a0b7..444bed6e390 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Harvesting.java
@@ -111,13 +111,13 @@ public static JsonObjectBuilder harvestingConfigAsJson(Dataverse dataverse) {
add("metadataFormat", harvestingConfig.getMetadataPrefix()).
add("set", harvestingConfig.getHarvestingSet() == null ? "N/A" : harvestingConfig.getHarvestingSet()).
add("schedule", harvestingConfig.isScheduled() ? harvestingConfig.getScheduleDescription() : "none").
- add("inProgress", harvestingConfig.isHarvestingNow() ? "yes" : "-").
+ add("status", harvestingConfig.isHarvestingNow() ? "inProgress" : "inActive").
add("lastHarvest", harvestingConfig.getLastHarvestTime() == null ? "N/A" : harvestingConfig.getLastHarvestTime().toString()).
+ add("lastResult", harvestingConfig.getLastResult()).
add("lastSuccessful", harvestingConfig.getLastSuccessfulHarvestTime() == null ? "N/A" : harvestingConfig.getLastSuccessfulHarvestTime().toString()).
add("lastNonEmpty", harvestingConfig.getLastNonEmptyHarvestTime() == null ? "N/A" : harvestingConfig.getLastNonEmptyHarvestTime().toString()).
- add("lastResult", harvestingConfig.getHarvestResult()).
- add("datasetsHarveted", harvestingConfig.getHarvestedDatasetCount() == null ? "N/A" : harvestingConfig.getHarvestedDatasetCount().toString()).
- add("datasetsDeleted", harvestingConfig.getDeletedDatasetCount() == null ? "N/A" : harvestingConfig.getDeletedDatasetCount().toString()).
- add("datasetsFailed", harvestingConfig.getFailedDatasetCount() == null ? "N/A" : harvestingConfig.getFailedDatasetCount().toString());
+ add("lastDatasetsHarvested", harvestingConfig.getLastHarvestedDatasetCount() == null ? "N/A" : harvestingConfig.getLastHarvestedDatasetCount().toString()).
+ add("lastDatasetsDeleted", harvestingConfig.getLastDeletedDatasetCount() == null ? "N/A" : harvestingConfig.getLastDeletedDatasetCount().toString()).
+ add("lastDatasetsFailed", harvestingConfig.getLastFailedDatasetCount() == null ? "N/A" : harvestingConfig.getLastFailedDatasetCount().toString());
}
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetHarvestingClientCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetHarvestingClientCommand.java
new file mode 100644
index 00000000000..d0929e2ec1a
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetHarvestingClientCommand.java
@@ -0,0 +1,51 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ *
+ * @author Leonid Andreev
+ */
+// One can view the configuration of a Harvesting Client if and only if
+// they have the permission to view the dataverse that owns the harvesting
+// client. And for a Dataverse, we cannot define the permission with a
+// @RequiredPermission annotation - because the decision has to be made dynamically:
+// Everybody can view a published Dataverse; otherwise, an explicit
+// ViewUnpublishedDataverse is needed.
+// This is defined in the getRequiredPermissions() method, below.
+public class GetHarvestingClientCommand extends AbstractCommand{
+ private final Dataverse ownerDataverse;
+
+ public GetHarvestingClientCommand(DataverseRequest aRequest, Dataverse ownerDataverse) {
+ super(aRequest, ownerDataverse);
+ this.ownerDataverse = ownerDataverse;
+ }
+
+ @Override
+ public HarvestingClient execute(CommandContext ctxt) throws CommandException {
+ if (ownerDataverse == null) {
+ throw new IllegalCommandException("GetHarvestingClientCommand called on a null dataverse object", this);
+ }
+ if (ownerDataverse.getHarvestingClientConfig() == null) {
+ throw new IllegalCommandException("No harvesting client is configured for dataverse "+ownerDataverse.getAlias(), this);
+ }
+ return ownerDataverse.getHarvestingClientConfig();
+ }
+
+ @Override
+ public Map> getRequiredPermissions() {
+ return Collections.singletonMap("",
+ ownerDataverse.isReleased() ? Collections.emptySet()
+ : Collections.singleton(Permission.ViewUnpublishedDataverse));
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
similarity index 56%
rename from src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java
rename to src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
index 404ace6cd42..7cd8c4e603d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientRun.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java
@@ -11,20 +11,23 @@
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
/**
*
* @author Leonid Andreev
- *
- * This is a record of an attempted harvesting client run.
- * (Should it be named HarvestingClientRunResult instead?)
+ *
+ * This is a record of an attempted harvesting client run. (Should it be named
+ * HarvestingClientRunResult instead?)
*/
@Entity
-public class HarvestingClientRun implements Serializable {
+public class ClientHarvestRun implements Serializable {
private static final long serialVersionUID = 1L;
+
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
private Long id;
@@ -37,18 +40,71 @@ public void setId(Long id) {
this.id = id;
}
- private String harvestResult; // TODO: should this me an enum instead? -- L.A. 4.4
+ public enum RunResultType { SUCCESS, FAILURE, INPROGRESS };
+
+ private static String RESULT_LABEL_SUCCESS = "SUCCESS";
+ private static String RESULT_LABEL_FAILURE = "FAILED";
+ private static String RESULT_LABEL_INPROGRESS = "INPROGRESS";
- public String getHarvestResult() {
+ @ManyToOne
+ @JoinColumn(nullable = false)
+ private HarvestingClient harvestingClient;
+
+ public HarvestingClient getHarvestingClient() {
+ return harvestingClient;
+ }
+
+ public void setHarvestingClient(HarvestingClient harvestingClient) {
+ this.harvestingClient = harvestingClient;
+ }
+
+ private RunResultType harvestResult;
+
+ public RunResultType getResult() {
return harvestResult;
}
+
+ public String getResultLabel() {
+ if (isSuccess()) {
+ return RESULT_LABEL_SUCCESS;
+ } else if (isFailed()) {
+ return RESULT_LABEL_FAILURE;
+ } else if (isInProgress()) {
+ return RESULT_LABEL_INPROGRESS;
+ }
+ return null;
+ }
- public void setHarvestResult(String harvestResult) {
+ public void setResult(RunResultType harvestResult) {
this.harvestResult = harvestResult;
}
+
+ public boolean isSuccess() {
+ return RunResultType.SUCCESS == harvestResult;
+ }
+
+ public void setSuccess() {
+ harvestResult = RunResultType.SUCCESS;
+ }
+
+ public boolean isFailed() {
+ return RunResultType.FAILURE == harvestResult;
+ }
+
+ public void setFailed() {
+ harvestResult = RunResultType.FAILURE;
+ }
- // Time of this harvest attempt:
+ public boolean isInProgress() {
+ return RunResultType.INPROGRESS == harvestResult ||
+ (harvestResult == null && startTime != null && finishTime == null);
+ }
+ public void setInProgress() {
+ harvestResult = RunResultType.INPROGRESS;
+ }
+
+ // Time of this harvest attempt:
@Temporal(value = TemporalType.TIMESTAMP)
private Date startTime;
@@ -57,12 +113,12 @@ public Date getStartTime() {
}
public void setStartTime(Date startTime) {
- this.startTime = startTime;
+ this.startTime = startTime;
}
-
+
@Temporal(value = TemporalType.TIMESTAMP)
- private Date finishTime;
-
+ private Date finishTime;
+
public Date getFinishTime() {
return finishTime;
}
@@ -70,16 +126,14 @@ public Date getFinishTime() {
public void setFinishTime(Date finishTime) {
this.finishTime = finishTime;
}
-
-
+
// Tese are the Dataset counts from that last harvest:
// (TODO: do we need to differentiate between *created* (new), and *updated*
// harvested datasets? -- L.A. 4.4
-
- private Long harvestedDatasetCount;
- private Long failedDatasetCount;
- private Long deletedDatasetCount;
-
+ private Long harvestedDatasetCount = 0L;
+ private Long failedDatasetCount = 0L;
+ private Long deletedDatasetCount = 0L;
+
public Long getHarvestedDatasetCount() {
return harvestedDatasetCount;
}
@@ -87,7 +141,7 @@ public Long getHarvestedDatasetCount() {
public void setHarvestedDatasetCount(Long harvestedDatasetCount) {
this.harvestedDatasetCount = harvestedDatasetCount;
}
-
+
public Long getFailedDatasetCount() {
return failedDatasetCount;
}
@@ -95,7 +149,7 @@ public Long getFailedDatasetCount() {
public void setFailedDatasetCount(Long failedDatasetCount) {
this.failedDatasetCount = failedDatasetCount;
}
-
+
public Long getDeletedDatasetCount() {
return deletedDatasetCount;
}
@@ -103,7 +157,7 @@ public Long getDeletedDatasetCount() {
public void setDeletedDatasetCount(Long deletedDatasetCount) {
this.deletedDatasetCount = deletedDatasetCount;
}
-
+
@Override
public int hashCode() {
int hash = 0;
@@ -114,10 +168,10 @@ public int hashCode() {
@Override
public boolean equals(Object object) {
// TODO: Warning - this method won't work in the case the id fields are not set
- if (!(object instanceof HarvestingClientRun)) {
+ if (!(object instanceof ClientHarvestRun)) {
return false;
}
- HarvestingClientRun other = (HarvestingClientRun) object;
+ ClientHarvestRun other = (ClientHarvestRun) object;
if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
return false;
}
@@ -128,5 +182,5 @@ public boolean equals(Object object) {
public String toString() {
return "edu.harvard.iq.dataverse.harvest.client.HarvestingClientRun[ id=" + id + " ]";
}
-
+
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
index e647f287db7..24cf96fbd65 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java
@@ -54,9 +54,10 @@ public class HarvesterServiceBean {
javax.ejb.TimerService timerService;
@EJB
DataverseTimerServiceBean dataverseTimerService;
+ @EJB
+ HarvestingClientServiceBean harvestingClientService;
private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean");
- private static final String HARVEST_TIMER = "HarvestTimer";
private static final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
@@ -64,8 +65,6 @@ public class HarvesterServiceBean {
public static final String HARVEST_RESULT_FAILED="failed";
- private Unmarshaller unmarshaller;
-
private long processedSizeThisBatch = 0;
private List harvestedDatasetIdsThisBatch = null;
public HarvesterServiceBean() {
@@ -170,9 +169,9 @@ public void doHarvest(Long dataverseId) throws IOException {
throw new IOException("No such Dataverse: id="+dataverseId);
}
- HarvestingClient harvestingDataverseConfig = harvestingDataverse.getHarvestingClientConfig();
+ HarvestingClient harvestingClientConfig = harvestingDataverse.getHarvestingClientConfig();
- if (harvestingDataverseConfig == null) {
+ if (harvestingClientConfig == null) {
throw new IOException("Could not find Harvesting Config for Dataverse id="+dataverseId);
}
@@ -191,37 +190,36 @@ public void doHarvest(Long dataverseId) throws IOException {
Date harvestStartTime = new Date();
try {
- boolean harvestingNow = harvestingDataverseConfig.isHarvestingNow();
+ boolean harvestingNow = harvestingClientConfig.isHarvestingNow();
if (harvestingNow) {
harvestErrorOccurred.setValue(true);
hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested.");
} else {
- dataverseService.resetHarvestingStatus(harvestingDataverse.getId());
+ harvestingClientService.resetHarvestInProgress(harvestingDataverse.getId());
String until = null; // If we don't set until date, we will get all the changes since the last harvest.
String from = null;
- Date lastSuccessfulHarvestTime = harvestingDataverseConfig.getLastSuccessfulHarvestTime();
+ // TODO: should it be last *non-empty* time? -- L.A. 4.4
+ Date lastSuccessfulHarvestTime = harvestingClientConfig.getLastSuccessfulHarvestTime();
if (lastSuccessfulHarvestTime != null) {
from = formatter.format(lastSuccessfulHarvestTime);
}
- dataverseService.setHarvestInProgress(harvestingDataverse.getId(), true);
-
- dataverseService.setLastHarvestTime(harvestingDataverse.getId(), harvestStartTime);
+ harvestingClientService.setHarvestInProgress(harvestingDataverse.getId(), harvestStartTime);
- hdLogger.log(Level.INFO, "BEGIN HARVEST..., oaiUrl=" + harvestingDataverseConfig.getArchiveUrl() + ",set=" + harvestingDataverseConfig.getHarvestingSet() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix() + ", from=" + from + ", until=" + until);
+ hdLogger.log(Level.INFO, "BEGIN HARVEST..., oaiUrl=" + harvestingClientConfig.getArchiveUrl() + ",set=" + harvestingClientConfig.getHarvestingSet() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix() + ", from=" + from + ", until=" + until);
- if (harvestingDataverseConfig.isOai()) {
+ if (harvestingClientConfig.isOai()) {
harvestedDatasetIds = harvestOAI(harvestingDataverse, hdLogger, from, until, harvestErrorOccurred, failedIdentifiers);
} else {
throw new IOException("Unsupported harvest type");
}
- dataverseService.setHarvestSuccess(harvestingDataverse.getId(), harvestStartTime, harvestedDatasetIds.size(), failedIdentifiers.size());
- hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingDataverseConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingDataverseConfig.getMetadataPrefix());
+ harvestingClientService.setHarvestSuccess(harvestingDataverse.getId(), new Date(), harvestedDatasetIds.size(), failedIdentifiers.size());
+ hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingClientConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix());
hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: [TODO:], datasets failed: " + failedIdentifiers.size());
- // now index all studies (need to modify for update)
+ // now index all the datasets we have harvested - created, modified or deleted:
/* (TODO: !!!)
if (this.processedSizeThisBatch > 0) {
hdLogger.log(Level.INFO, "POST HARVEST, reindexing the remaining studies.");
@@ -239,14 +237,19 @@ public void doHarvest(Long dataverseId) throws IOException {
//mailService.sendHarvestNotification(...getSystemEmail(), harvestingDataverse.getName(), logFileName, logTimestamp, harvestErrorOccurred.booleanValue(), harvestedDatasetIds.size(), failedIdentifiers);
} catch (Throwable e) {
harvestErrorOccurred.setValue(true);
- String message = "Exception processing harvest, server= " + harvestingDataverseConfig.getArchiveUrl() + ",format=" + harvestingDataverseConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage();
+ String message = "Exception processing harvest, server= " + harvestingClientConfig.getArchiveUrl() + ",format=" + harvestingClientConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage();
hdLogger.log(Level.SEVERE, message);
logException(e, hdLogger);
hdLogger.log(Level.INFO, "HARVEST NOT COMPLETED DUE TO UNEXPECTED ERROR.");
- dataverseService.setHarvestFailure(harvestingDataverse.getId(), harvestStartTime);
+ // TODO:
+ // even though this harvesting run failed, we may have had successfully
+ // processed some number of datasets, by the time the exception was thrown.
+ // We should record that number too. And the number of the datasets that
+ // had failed, that we may have counted. -- L.A. 4.4
+ harvestingClientService.setHarvestFailure(harvestingDataverse.getId(), new Date());
} finally {
- dataverseService.setHarvestInProgress(harvestingDataverse.getId(), false);
+ harvestingClientService.resetHarvestInProgress(harvestingDataverse.getId());
fileHandler.close();
hdLogger.removeHandler(fileHandler);
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
index 98d9ac8b3cb..2d31d8067a3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
@@ -11,6 +11,7 @@
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
+import java.util.List;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
@@ -19,7 +20,9 @@
import javax.persistence.Id;
import javax.persistence.Index;
import javax.persistence.JoinColumn;
+import javax.persistence.OneToMany;
import javax.persistence.OneToOne;
+import javax.persistence.OrderBy;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@@ -172,14 +175,129 @@ public void setMetadataPrefix(String metadataPrefix) {
this.metadataPrefix = metadataPrefix;
}
- /* move the fields below to the new HarvestingClientRun class: */
+ // TODO: do we need "orphanRemoval=true"? -- L.A. 4.4
+ // TODO: should it be @OrderBy("startTime")? -- L.A. 4.4
+ @OneToMany(mappedBy="harvestingClient", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
+ @OrderBy("id")
+ private List harvestHistory;
+
+ List getRunHistory() {
+ return harvestHistory;
+ }
+
+ void setRunHistory(List harvestHistory) {
+ this.harvestHistory = harvestHistory;
+ }
+
+ public String getLastResult() {
+ if (harvestHistory == null || harvestHistory.size() == 0) {
+ return null;
+ }
+ return harvestHistory.get(harvestHistory.size() - 1).getResultLabel();
+ }
+
+ public ClientHarvestRun getLastRun() {
+ if (harvestHistory == null || harvestHistory.size() == 0) {
+ return null;
+ }
+
+ return harvestHistory.get(harvestHistory.size() - 1);
+ }
+
+ public ClientHarvestRun getLastSuccessfulRun() {
+ if (harvestHistory == null || harvestHistory.size() == 0) {
+ return null;
+ }
+
+ ClientHarvestRun harvestRun = null;
+ int i = harvestHistory.size() - 1;
+
+ while (i > 0) {
+ if (harvestHistory.get(i).isSuccess()) {
+ return harvestHistory.get(i);
+ }
+ i--;
+ }
+
+ return null;
+ }
+
+ ClientHarvestRun getLastNonEmptyRun() {
+ if (harvestHistory == null || harvestHistory.size() == 0) {
+ return null;
+ }
+
+ ClientHarvestRun harvestRun = null;
+ int i = harvestHistory.size() - 1;
+
+ while (i > 0) {
+ if (harvestHistory.get(i).isSuccess()) {
+ if (harvestHistory.get(i).getHarvestedDatasetCount().longValue() > 0 ||
+ harvestHistory.get(i).getDeletedDatasetCount().longValue() > 0) {
+ return harvestHistory.get(i);
+ }
+ }
+ i--;
+ }
+ return null;
+ }
+
+ public Date getLastHarvestTime() {
+ ClientHarvestRun lastHarvest = getLastRun();
+ if ( lastHarvest != null) {
+ return lastHarvest.getStartTime();
+ }
+ return null;
+ }
+
+ public Date getLastSuccessfulHarvestTime() {
+ ClientHarvestRun lastSuccessfulHarvest = getLastSuccessfulRun();
+ if ( lastSuccessfulHarvest != null) {
+ return lastSuccessfulHarvest.getStartTime();
+ }
+ return null;
+ }
+
+ public Date getLastNonEmptyHarvestTime() {
+ ClientHarvestRun lastNonEmptyHarvest = getLastNonEmptyRun();
+ if ( lastNonEmptyHarvest != null) {
+ return lastNonEmptyHarvest.getStartTime();
+ }
+ return null;
+ }
+
+ public Long getLastHarvestedDatasetCount() {
+ ClientHarvestRun lastNonEmptyHarvest = getLastNonEmptyRun();
+ if ( lastNonEmptyHarvest != null) {
+ return lastNonEmptyHarvest.getHarvestedDatasetCount();
+ }
+ return null;
+ }
+
+ public Long getLastFailedDatasetCount() {
+ ClientHarvestRun lastNonEmptyHarvest = getLastNonEmptyRun();
+ if ( lastNonEmptyHarvest != null) {
+ return lastNonEmptyHarvest.getFailedDatasetCount();
+ }
+ return null;
+ }
+
+ public Long getLastDeletedDatasetCount() {
+ ClientHarvestRun lastNonEmptyHarvest = getLastNonEmptyRun();
+ if ( lastNonEmptyHarvest != null) {
+ return lastNonEmptyHarvest.getDeletedDatasetCount();
+ }
+ return null;
+ }
+
+ /* move the fields below to the new HarvestingClientRun class:
private String harvestResult;
- public String getHarvestResult() {
+ public String getResult() {
return harvestResult;
}
- public void setHarvestResult(String harvestResult) {
+ public void setResult(String harvestResult) {
this.harvestResult = harvestResult;
}
@@ -198,7 +316,7 @@ public void setLastHarvestTime(Date lastHarvestTime) {
this.lastHarvestTime = lastHarvestTime;
}
- // This the last "successful harvest" - i.e., the last time we
+ // This is the last "successful harvest" - i.e., the last time we
// tried to harvest, and got a response from the remote server.
// We may not have necessarily harvested any useful content though;
// the result may have been a "no content" or "no changes since the last harvest"
@@ -235,7 +353,7 @@ public void setLastNonEmptyHarvestTime(Date lastNonEmptyHarvestTime) {
private Long failedDatasetCount;
private Long deletedDatasetCount;
- public Long getHarvestedDatasetCount() {
+ public Long getLastHarvestedDatasetCount() {
return harvestedDatasetCount;
}
@@ -243,7 +361,7 @@ public void setHarvestedDatasetCount(Long harvestedDatasetCount) {
this.harvestedDatasetCount = harvestedDatasetCount;
}
- public Long getFailedDatasetCount() {
+ public Long getLastFailedDatasetCount() {
return failedDatasetCount;
}
@@ -251,14 +369,14 @@ public void setFailedDatasetCount(Long failedDatasetCount) {
this.failedDatasetCount = failedDatasetCount;
}
- public Long getDeletedDatasetCount() {
+ public Long getLastDeletedDatasetCount() {
return deletedDatasetCount;
}
public void setDeletedDatasetCount(Long deletedDatasetCount) {
this.deletedDatasetCount = deletedDatasetCount;
}
- /**/
+ */
private boolean scheduled;
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
index 7783e64e8ff..df97463172e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java
@@ -1,11 +1,18 @@
package edu.harvard.iq.dataverse.harvest.client;
+import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseServiceBean;
+import java.util.ArrayList;
+import java.util.Date;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.Stateless;
+import javax.ejb.TransactionAttribute;
+import javax.ejb.TransactionAttributeType;
import javax.faces.bean.ManagedBean;
import javax.inject.Named;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
/**
*
@@ -20,5 +27,110 @@ public class HarvestingClientServiceBean {
@EJB
DataverseServiceBean dataverseService;
- private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvestingClinetServiceBean");
+ @PersistenceContext(unitName = "VDCNet-ejbPU")
+ private EntityManager em;
+
+ private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvestingClinetServiceBean");
+
+ /* let's try and live without this method:
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestResult(Long hdId, String result) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd); // ??
+ if (hd.isHarvested()) {
+ hd.getHarvestingClientConfig().setHarvestResult(result);
+ }
+ }
+ */
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void resetHarvestInProgress(Long hdId) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (!hd.isHarvested()) {
+ return;
+ }
+ hd.getHarvestingClientConfig().setHarvestingNow(false);
+
+ // And if there is an unfinished RunResult object, we'll
+ // just mark it as a failure:
+ if (hd.getHarvestingClientConfig().getLastRun() != null
+ && hd.getHarvestingClientConfig().getLastRun().isInProgress()) {
+ hd.getHarvestingClientConfig().getLastRun().setFailed();
+ }
+
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestInProgress(Long hdId, Date startTime) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ HarvestingClient harvestingClient = hd.getHarvestingClientConfig();
+ if (harvestingClient == null) {
+ return;
+ }
+ harvestingClient.setHarvestingNow(false);
+ if (harvestingClient.getRunHistory() == null) {
+ harvestingClient.setRunHistory(new ArrayList());
+ }
+ ClientHarvestRun currentRun = new ClientHarvestRun();
+ currentRun.setHarvestingClient(harvestingClient);
+ currentRun.setStartTime(startTime);
+ currentRun.setInProgress();
+ harvestingClient.getRunHistory().add(currentRun);
+ }
+
+ /*
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setLastHarvestTime(Long hdId, Date lastHarvestTime) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ if (hd.isHarvested()) {
+ hd.getHarvestingClientConfig().setLastHarvestTime(lastHarvestTime);
+ }
+ }*/
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestSuccess(Long hdId, Date currentTime, int harvestedCount, int failedCount) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ HarvestingClient harvestingClient = hd.getHarvestingClientConfig();
+ if (harvestingClient == null) {
+ return;
+ }
+
+ ClientHarvestRun currentRun = harvestingClient.getLastRun();
+
+ if (currentRun != null && currentRun.isInProgress()) {
+ // TODO: what if there's no current run in progress? should we just
+ // give up quietly, or should we make a noise of some kind? -- L.A. 4.4
+
+ currentRun.setSuccess();
+ currentRun.setFinishTime(currentTime);
+ currentRun.setHarvestedDatasetCount(new Long(harvestedCount));
+ currentRun.setFailedDatasetCount(new Long(failedCount));
+
+ /*TODO: still need to record the number of deleted datasets! */
+ }
+ }
+
+ @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+ public void setHarvestFailure(Long hdId, Date currentTime) {
+ Dataverse hd = em.find(Dataverse.class, hdId);
+ em.refresh(hd);
+ HarvestingClient harvestingClient = hd.getHarvestingClientConfig();
+ if (harvestingClient == null) {
+ return;
+ }
+
+ ClientHarvestRun currentRun = harvestingClient.getLastRun();
+
+ if (currentRun != null && currentRun.isInProgress()) {
+ // TODO: what if there's no current run in progress? should we just
+ // give up quietly, or should we make a noise of some kind? -- L.A. 4.4
+
+ currentRun.setFailed();
+ currentRun.setFinishTime(currentTime);
+ }
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
index f438bd10c14..b24be18264a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/timer/DataverseTimerServiceBean.java
@@ -10,6 +10,7 @@
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.harvest.client.HarvestTimerInfo;
import edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean;
+import edu.harvard.iq.dataverse.harvest.client.HarvestingClientServiceBean;
import java.io.Serializable;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -43,6 +44,8 @@ public class DataverseTimerServiceBean implements Serializable {
HarvesterServiceBean harvesterService;
@EJB
DataverseServiceBean dataverseService;
+ @EJB
+ HarvestingClientServiceBean harvestingClientService;
/*@EJB
StudyServiceLocal studyService;*/
@@ -80,11 +83,18 @@ public void handleTimeout(javax.ejb.Timer timer) {
HarvestTimerInfo info = (HarvestTimerInfo) timer.getInfo();
try {
- logger.log(Level.INFO, "DO HARVESTING of dataverse " + info.getHarvestingDataverseId());
+ logger.log(Level.INFO, "running a harvester client configured for dataverse " + info.getHarvestingDataverseId());
harvesterService.doHarvest(info.getHarvestingDataverseId());
} catch (Throwable e) {
- dataverseService.setHarvestResult(info.getHarvestingDataverseId(), harvesterService.HARVEST_RESULT_FAILED);
+ // Harvester Service should be handling any error notifications,
+ // if/when things go wrong.
+ // (TODO: -- verify this logic; harvesterService may still be able
+ // to throw an IOException, if it could not run the harvest at all,
+ // or could not for whatever reason modify the database record...
+ // in this case we should, probably, log the error and try to send
+ // a mail notification. -- L.A. 4.4)
+ //dataverseService.setHarvestResult(info.getHarvestingDataverseId(), harvesterService.HARVEST_RESULT_FAILED);
//mailService.sendHarvestErrorNotification(dataverseService.find().getSystemEmail(), dataverseService.find().getName());
logException(e, logger);
}
From 54b5cea00420366ae521f2bfce8d6fbf9feb7a0b Mon Sep 17 00:00:00 2001
From: Leonid Andreev
Date: Fri, 29 Apr 2016 17:17:54 -0400
Subject: [PATCH 10/37] API further reorganized; proper Commands added for
retrieving, creating, modifying and deleting Harvesting Clients.
---
.../iq/dataverse/DataverseServiceBean.java | 4 +-
.../harvard/iq/dataverse/api/Datasets.java | 2 +-
.../harvard/iq/dataverse/api/Harvesting.java | 213 +++++++++++++++---
.../impl/CreateHarvestingClientCommand.java | 14 +-
.../impl/DeleteHarvestingClientCommand.java | 11 +-
.../impl/GetHarvestingClientCommand.java | 6 +-
.../impl/UpdateHarvestingClientCommand.java | 17 +-
.../harvest/client/HarvestingClient.java | 13 ++
.../client/HarvestingClientServiceBean.java | 34 ++-
.../iq/dataverse/util/json/JsonParser.java | 17 +-
10 files changed, 260 insertions(+), 71 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 003df01f87b..8b2b438ac2a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -419,7 +419,7 @@ public List findDataversesThatLinkToThisDatasetId(long datasetId) {
*/
public Map getAllHarvestedDataverseDescriptions(){
- String qstr = "SELECT dataverse_id, archiveDescription FROM harvestingDataverseConfig;";
+ String qstr = "SELECT dataverse_id, archiveDescription FROM harvestingClient;";
List
-
+
#{bundle['dataset.publishBoth.tip']}
@@ -1340,7 +1340,7 @@
#{bundle['dataset.republish.tip']}
-
+
diff --git a/src/main/webapp/manage-guestbooks.xhtml b/src/main/webapp/manage-guestbooks.xhtml
index 3348ecdf55f..54e4f35d3dd 100644
--- a/src/main/webapp/manage-guestbooks.xhtml
+++ b/src/main/webapp/manage-guestbooks.xhtml
@@ -4,7 +4,8 @@
xmlns:f="http://java.sun.com/jsf/core"
xmlns:ui="http://java.sun.com/jsf/facelets"
xmlns:p="http://primefaces.org/ui"
- xmlns:jsf="http://xmlns.jcp.org/jsf">
+ xmlns:jsf="http://xmlns.jcp.org/jsf"
+ xmlns:o="http://omnifaces.org/ui">
@@ -18,6 +19,7 @@
+
@@ -182,12 +184,14 @@
From a7fdea3959b03656f0176c5b59adf9f1e0a936b1 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller
Date: Tue, 10 May 2016 17:17:09 -0400
Subject: [PATCH 29/37] fix import
---
src/main/webapp/dataset.xhtml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 572cf4b944a..733bec67f58 100755
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -15,7 +15,6 @@
-
@@ -24,6 +23,7 @@
+
From bd9c1a16237d758250c7332da2703552d6382290 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller
Date: Wed, 11 May 2016 08:47:38 -0400
Subject: [PATCH 30/37] Fix DV tag line and Permissions Success Message
---
.../edu/harvard/iq/dataverse/ManagePermissionsPage.java | 3 ++-
.../edu/harvard/iq/dataverse/RolePermissionFragment.java | 3 ++-
src/main/webapp/dataverse_header.xhtml | 8 +++++---
3 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
index 36902669816..b0125cce040 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
@@ -38,6 +38,7 @@
import javax.inject.Named;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
/**
@@ -396,7 +397,7 @@ private void notifyRoleChange(RoleAssignee ra, UserNotification.Type type) {
private void assignRole(RoleAssignee ra, DataverseRole r) {
try {
commandEngine.submit(new AssignRoleCommand(ra, r, dvObject, dvRequestService.getDataverseRequest()));
- JsfHelper.addSuccessMessage(r.getName() + " role assigned to " + ra.getDisplayInfo().getTitle() + " for " + dvObject.getDisplayName() + ".");
+ JsfHelper.addSuccessMessage(r.getName() + " role assigned to " + ra.getDisplayInfo().getTitle() + " for " + StringEscapeUtils.escapeHtml(dvObject.getDisplayName()) + ".");
// don't notify if role = file downloader and object is not released
if (!(r.getAlias().equals(DataverseRole.FILE_DOWNLOADER) && !dvObject.isReleased()) ){
notifyRoleChange(ra, UserNotification.Type.ASSIGNROLE);
diff --git a/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java b/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java
index c420c4692f2..99e7cded743 100644
--- a/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/RolePermissionFragment.java
@@ -34,6 +34,7 @@
import javax.inject.Named;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
/**
@@ -187,7 +188,7 @@ public void assignRole(ActionEvent evt) {
private void assignRole(RoleAssignee ra, DataverseRole r) {
try {
commandEngine.submit(new AssignRoleCommand(ra, r, dvObject, dvRequestService.getDataverseRequest()));
- JH.addMessage(FacesMessage.SEVERITY_INFO, "Role " + r.getName() + " assigned to " + ra.getDisplayInfo().getTitle() + " on " + dvObject.getDisplayName());
+ JH.addMessage(FacesMessage.SEVERITY_INFO, "Role " + r.getName() + " assigned to " + ra.getDisplayInfo().getTitle() + " on " + StringEscapeUtils.escapeHtml(dvObject.getDisplayName()));
} catch (CommandException ex) {
JH.addMessage(FacesMessage.SEVERITY_ERROR, "Can't assign role: " + ex.getMessage());
}
diff --git a/src/main/webapp/dataverse_header.xhtml b/src/main/webapp/dataverse_header.xhtml
index e38a66e4c60..2ba378390df 100644
--- a/src/main/webapp/dataverse_header.xhtml
+++ b/src/main/webapp/dataverse_header.xhtml
@@ -5,7 +5,8 @@
xmlns:p="http://primefaces.org/ui"
xmlns:pt="http://xmlns.jcp.org/jsf/passthrough"
xmlns:jsf="http://xmlns.jcp.org/jsf"
- xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs">
+ xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"
+ xmlns:o="http://omnifaces.org/ui">