Merge pull request #5 from IQSS/master

Update to 5/7, 3:01pm

conf/solr/4.6.0/schema.xml

@@ -298,22 +298,22 @@
    <field name="coverage.Redshift.MinimumValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.RedshiftValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.SkyFraction" type="text_en" multiValued="false" stored="true" indexed="true"/>
-   <field name="coverage.Spatial" type="text_en" multiValued="false" stored="true" indexed="true"/>
+   <field name="coverage.Spatial" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.Spectral.Bandpass" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.Spectral.CentralWavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.Spectral.MaximumWavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.Spectral.MinimumWavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="coverage.Spectral.Wavelength" type="text_en" multiValued="true" stored="true" indexed="true"/>
-   <field name="coverage.Temporal" type="int" multiValued="false" stored="true" indexed="true"/>
-   <field name="coverage.Temporal.StartTime" type="int" multiValued="false" stored="true" indexed="true"/>
-   <field name="coverage.Temporal.StopTime" type="int" multiValued="false" stored="true" indexed="true"/>
+   <field name="coverage.Temporal" type="text_en" multiValued="true" stored="true" indexed="true"/>
+   <field name="coverage.Temporal.StartTime" type="text_en" multiValued="true" stored="true" indexed="true"/>
+   <field name="coverage.Temporal.StopTime" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="dataCollectionSituation" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="dataSources" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="datasetLevelErrorNotes" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="dateOfCollection" type="text_en" multiValued="true" stored="true" indexed="true"/>
-   <field name="dateOfCollectionEnd" type="int" multiValued="false" stored="true" indexed="true"/>
-   <field name="dateOfCollectionStart" type="int" multiValued="false" stored="true" indexed="true"/>
-   <field name="dateOfDeposit" type="int" multiValued="false" stored="true" indexed="true"/>
+   <field name="dateOfCollectionEnd" type="text_en" multiValued="true" stored="true" indexed="true"/>
+   <field name="dateOfCollectionStart" type="text_en" multiValued="true" stored="true" indexed="true"/>
+   <field name="dateOfDeposit" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="depositor" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="deviationsFromSampleDesign" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="distributorContact" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -336,7 +336,7 @@
    <field name="otherIdAgency" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="otherIdValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="otherReferences" type="text_en" multiValued="true" stored="true" indexed="true"/>
-   <field name="productionDate" type="int" multiValued="false" stored="true" indexed="true"/>
+   <field name="productionDate" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="productionPlace" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="publication" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="publicationCitation" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -350,7 +350,7 @@
    <field name="resolution.Redshift" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="resolution.Spatial" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="resolution.Spectral" type="text_en" multiValued="true" stored="true" indexed="true"/>
-   <field name="resolution.Temporal" type="text_en" multiValued="false" stored="true" indexed="true"/>
+   <field name="resolution.Temporal" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="responseRate" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="sampleFrame" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="samplingErrorEstimates" type="text_en" multiValued="false" stored="true" indexed="true"/>
@@ -370,9 +370,9 @@
    <field name="subject" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="targetSampleSize" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="timeMethod" type="text_en" multiValued="false" stored="true" indexed="true"/>
-   <field name="timePeriodCovered" type="int" multiValued="false" stored="true" indexed="true"/>
-   <field name="timePeriodCoveredEnd" type="int" multiValued="false" stored="true" indexed="true"/>
-   <field name="timePeriodCoveredStart" type="int" multiValued="false" stored="true" indexed="true"/>
+   <field name="timePeriodCovered" type="text_en" multiValued="true" stored="true" indexed="true"/>
+   <field name="timePeriodCoveredEnd" type="text_en" multiValued="true" stored="true" indexed="true"/>
+   <field name="timePeriodCoveredStart" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="title" type="text_en" multiValued="false" stored="true" indexed="true"/>
    <field name="topicClassValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
    <field name="topicClassVocab" type="text_en" multiValued="true" stored="true" indexed="true"/>

doc/Sphinx/source/User/account.rst

@@ -15,8 +15,8 @@ Create User Account
 
 Edit Your Account
 ==================
-#. To edit your account, click on your account name in the header on the right hand side.
-#. On the right of your account information, click on the "Edit Account" button and from there you can select to edit either your Account Information or your Account Password. 
+#. To edit your account, click on your account name in the header on the right hand side and click on either Notifications or Data Related to Me.
+#. On the top right of your account page, click on the "Edit Account" button and from there you can select to edit either your Account Information or your Account Password. 
 #. Select "Save Changes" when you are done.
 
 Notifications: Setup & Maintainance
@@ -25,7 +25,8 @@ Notifications appear in the notifications tab on your account page and are also
 
 You will receive a notification when:
 
-- You've published a dataverse or dataset
-- When another user has published a dataverse or dataset within your dataverse (only if your dataverse allows this)- A user has requested access to restricted dataverses, datasets, or files within your dataverse
+- You've created your account
+- You've created a dataverse or added a dataset
+- More notifications to come!
 
 Dataverse will email your unread notifications once a day. Notifications will only be emailed one time even if you haven't read the notification on the Dataverse site.

doc/Sphinx/source/User/dataset-management.rst

@@ -1,43 +1,47 @@
 Dataset & File Management
 +++++++++++++++++++++++++++++
 
-A Dataset in Dataverse is a container for your data, documentation, code, and the metadata describing this Dataset.
+A dataset in Dataverse is a container for your data, documentation, code, and the metadata describing this Dataset.
 
 |image1|
 
-Add Dataset
+New Dataset
 ====================
 
-#. Navigate to the dataverse in which you want to add a Dataset (or in the "root" dataverse). 
-#. Click on the "Add Data" button and select "Add Dataset" in the dropdown menu.
+#. Navigate to the dataverse in which you want to add a dataset (or in the "root" dataverse). 
+#. Click on the "Add Data" button and select "New Dataset" in the dropdown menu.
 #. To quickly get started, enter at minimum all the required fields with an asterisk (e.g., the Dataset Title, Author, 
    Description, etc)
 #. Scroll down to "Files" tab and click on "Select Files to Add" to add all the relevant files to your Dataset. 
    You can also upload your files directly from your Dropbox. **Tip:** You can drag and drop your files from your desktop,
    directly into the upload widget. Your file will appear below the "Select Files to Add" button where you can add a
    description of the file.
-#. Click the "Create Dataset" button when you are done. Your unpublished Dataset is now created. 
+#. Click the "Add Dataset" button when you are done. Your unpublished dataset is now created. 
 
-Note: You can add additional metadata once you have completed the initial Dataset creation by going to Edit Dataset. 
+Note: You can add additional metadata once you have completed the initial dataset creation by going to Edit Dataset. 
 
 
 Edit Dataset
 ==================
 
-Go to your Dataset page and click on the "Edit Dataset" button. There you will have two options where you can either edit:
+Go to your dataset page and click on the "Edit Dataset" button. There you will have two options where you can either edit:
 
-- Files (Upload or Edit Data): to add or edit files in this Dataset.
+- Files (Upload or Edit Data): to add or edit files in this dataset.
 - Metadata: to add/edit metadata including additional metadata than was not previously available during Dataset Creation.
 
 
 Publish Dataset
 ====================
 
-When you publish a Dataset, you make it available to the public. Users can
-browse it or search for it. Once your Dataset is ready to go public, go to your Dataset page, 
-click on the "Unpublished" button on the right hand side of the page which should indicate: 
-"This Dataset is unpublished. To publish it click 'Publish Dataset' link."
+When you publish a dataset, you make it available to the public so that other users can
+browse or search for it. Once your dataset is ready to go public, go to your dataset page and click on the 
+"Publish" button on the right hand side of the page. A pop-up will appear to confirm that you are ready to actually 
+Publish since once a dataset is made public it can no longer be unpublished. 
+
+Whenever you edit your dataset, you are able to publish a new version of the dataset. The publish dataset button will reappear whenever you edit the metadata of the dataset or add a file.
+
+Note: Prior to publishing your dataset the Data Citation will indicate that this is a draft but the "DRAFT VERSION" text
+will be removed as soon as you Publish.
 
-Important Note: Once a Dataset is published it **cannot be unpublished**; it can be archived instead.
 
 .. |image1| image:: ./img/DatasetDiagram.png

doc/Sphinx/source/User/dataverse-management.rst

@@ -44,12 +44,9 @@ where you will be presented with the following editing options.
 Publish Your Dataverse
 =================================================================
 
-Once your dataverse is ready to go public, go to your dataverse page, click on the "Unpublished" button on the right 
-hand side of the page which should indicate: 
-"This dataverse is Unpublished. To publish it click 'Publish dataverse' link." Once you click "Publish dataverse" it
-will be made public.
-
-**Important Note**: Once a dataverse is made public it can no longer be unpublished.
+Once your dataverse is ready to go public, go to your dataverse page, click on the "Publish" button on the right 
+hand side of the page. A pop-up will appear to confirm that you are ready to actually Publish since once a dataverse
+is made public it can no longer be unpublished.
 
 
 .. |image1| image:: ./img/Dataverse-Diagram.png

doc/Sphinx/source/User/find-use-data.rst

@@ -40,6 +40,12 @@ To perform an advanced search, click the Advanced Search link next to the search
 
 **Advanced Search fields** 
 
+*Dataverses:*
+
+- Name - The project, department, university, or professor this Dataverse will contain data for.
+- Affiliation - The organization with which this Dataverse is affiliated.
+- Description - A summary describing the purpose, nature, or scope of this Dataverse.
+
 *Citation Metadata:*
 
 - Title - Full title by which the dataset is known.
@@ -84,6 +90,11 @@ To perform an advanced search, click the Advanced Search link next to the search
 - Organism - The taxonomic name of the organism used in a study or from which the starting biological material derives.
 - Cell Type - The name of the cell line from which the source or sample derives.
 
+*Files:*
+
+- Name - Full name by which the file is known.
+- Description - A summary describing the file, variables, or type.
+- File Type - The extension for the file, for example, JPEG, PNG, dta, etc
 
 Browsing Dataverse
 --------------------

scripts/api/data/metadatablocks/astrophysics.tsv

@@ -7,16 +7,16 @@
 	astroObject	Object	Astronomical Objects represented in the data (Given as SIMBAD recognizable names preferred).		text	3	TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		astrophysics
 	resolution.Spatial	Spatial Resolution	The spatial (angular) resolution that is typical of the observations, in decimal degrees.		float	4	TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		astrophysics
 	resolution.Spectral	Spectral Resolution	The spectral resolution that is typical of the observations, given as the ratio λ/Δλ.		float	5	TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		astrophysics
-	resolution.Temporal	Time Resolution	The temporal resolution that is typical of the observations, given in seconds.		float	6	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		astrophysics
+	resolution.Temporal	Time Resolution	The temporal resolution that is typical of the observations, given in seconds.		float	6	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		astrophysics
 	coverage.Spectral.Bandpass	Bandpass	Conventional bandpass name		text	7	TRUE	TRUE	TRUE	TRUE	FALSE	FALSE		astrophysics
 	coverage.Spectral.CentralWavelength	Central Wavelength (m)	The central wavelength of the spectral bandpass, in meters.		float	8	TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		astrophysics
 	coverage.Spectral.Wavelength	Wavelength Range	The minimum and maximum wavelength of the spectral bandpass.		float	9	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		astrophysics
 	coverage.Spectral.MinimumWavelength	Minimum (m)	The minimum wavelength of the spectral bandpass, in meters.		float	10	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	coverage.Spectral.Wavelength	astrophysics
 	coverage.Spectral.MaximumWavelength	Maximum (m)	The maximum wavelength of the spectral bandpass, in meters.		float	11	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	coverage.Spectral.Wavelength	astrophysics
-	coverage.Temporal	Dataset Date Range	 Time period covered by the data.		date	12	TRUE	FALSE	TRUE	FALSE	FALSE	FALSE		astrophysics
-	coverage.Temporal.StartTime	Start	Dataset Start Date	YYYY-MM-DD	date	13	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	coverage.Temporal	astrophysics
-	coverage.Temporal.StopTime	End	Dataset End Date	YYYY-MM-DD	date	14	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	coverage.Temporal	astrophysics
-	coverage.Spatial	Sky Coverage	The sky coverage of the data object.		text	15	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		astrophysics
+	coverage.Temporal	Dataset Date Range	 Time period covered by the data.		date	12	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		astrophysics
+	coverage.Temporal.StartTime	Start	Dataset Start Date	YYYY-MM-DD	date	13	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	coverage.Temporal	astrophysics
+	coverage.Temporal.StopTime	End	Dataset End Date	YYYY-MM-DD	date	14	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	coverage.Temporal	astrophysics
+	coverage.Spatial	Sky Coverage	The sky coverage of the data object.		text	15	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		astrophysics
 	coverage.Depth	Depth Coverage	The (typical) depth coverage, or sensitivity, of the data object in Jy.		float	16	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		astrophysics
 	coverage.ObjectDensity	Object Density	The (typical) density of objects, catalog entries, telescope pointings, etc., on the sky, in number per square degree.		float	17	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		astrophysics
 	coverage.ObjectCount	Object Count	The total number of objects, catalog entries, etc., in the data object.		int	18	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		astrophysics
@@ -51,4 +51,4 @@
 	astroType	Object	20											
 	astroType	Value	21											
 	astroType	ValuePair	22											
-	astroType	Survey	23											
+	astroType	Survey	23											

scripts/api/data/metadatablocks/citation.tsv

@@ -5,7 +5,7 @@
 	author	Author	The person(s), corporate body(ies), or agency(ies) responsible for creating the work.		text	1	FALSE	FALSE	TRUE	FALSE	TRUE	FALSE		citation
 	authorName	Name	The author's Family Name, Given Name or the name of the organization responsible for this dataset.	FamilyName, GivenName or Organization	text	2	TRUE	FALSE	FALSE	TRUE	TRUE	TRUE	author	citation
 	authorAffiliation	Affiliation	The organization with which the author is affiliated.		text	3	TRUE	TRUE	FALSE	TRUE	TRUE	FALSE	author	citation
-	distributorContact	Contact E-mail	The e-mail address(es) of the contact(s) for the Dataset. This will not be displayed.		text	4	FALSE	FALSE	TRUE	FALSE	TRUE	TRUE		citation
+	distributorContact	Contact E-mail	The e-mail address(es) of the contact(s) for the Dataset. This will not be displayed.		email	4	FALSE	FALSE	TRUE	FALSE	TRUE	TRUE		citation
 	dsDescription	Description	A summary describing the purpose, nature, and scope of the Dataset.		textbox	5	TRUE	FALSE	FALSE	FALSE	TRUE	TRUE		citation
 	keyword	Keyword	Key terms that describe important aspects of the Dataset.		text	6	TRUE	FALSE	TRUE	TRUE	TRUE	FALSE		citation
 	subject	Subject	Domain-specific Subjects that are topically relevant to the Dataset.		text	7	TRUE	TRUE	TRUE	TRUE	TRUE	TRUE		citation
@@ -82,4 +82,4 @@
 	contributorType	Sponsor	14											
 	contributorType	Supervisor	15											
 	contributorType	Work Package Leader	16											
-	contributorType	Other	17											
+	contributorType	Other	17											

scripts/api/setup-dvs.sh

@@ -30,4 +30,6 @@ curl -H "Content-type:application/json" -X POST -d @data/role-guest.json "http:/
 curl -H "Content-type:application/json" -X POST -d"{\"userName\":\"__GUEST__\",\"roleAlias\":\"guest-role\"}" http://localhost:8080/api/dvs/root/assignments/?key=pete
 
 echo Set the metadata block for Root
-curl -X POST -H "Content-type:application/json" -d "[\"citation\"]" http://localhost:8080/api/dvs/:root/metadatablocks/?key=pete
+curl -X POST -H "Content-type:application/json" -d "[\"citation\"]" http://localhost:8080/api/dvs/:root/metadatablocks/?key=pete
+
+

scripts/database/homebrew/rebuild-and-test

@@ -12,3 +12,4 @@ sleep 15
 scripts/database/homebrew/run-post-create-post-deploy
 scripts/search/tests/permissions
 scripts/search/tests/delete-dataverse
+scripts/search/tests/query-unparseable

scripts/search/tests/dataset-versioning02

@@ -7,6 +7,21 @@
 #
 # To do this, you must also release the Trees and Root dataverses
 # raw version
+#
+# the expected output in server.log is something like this:
+#
+#deleteDraftDatasetVersionResult: Attempted to delete draft dataset_17_draft from Solr index. updateReponse was: {responseHeader={status=0,QTime=1}}, deleteDraftFilesResults: , indexReleasedVersionResult:indexed dataset 17 as dataset_17
+#indexFilesResults for dataset_17, filesIndexed: [datafile_18], 
+#rationale:
+#version found with database id 1
+#- title: Rings of Trees and Other Observations
+#- semanticVersion-STATE: 1.0-RELEASED
+#- isWorkingCopy: false
+#- isReleased: true
+#- files: 1 [18:trees.png]
+#The latest version is not a working copy (latestVersionState: RELEASED) and will be indexed as dataset_17 (visible by anonymous) and we will be deleting dataset_17_draft and its files (if any, num:1): [datafile_18_draft]
+#The released version is 1.0 (releasedVersionState: RELEASED) and will be (again) indexed as dataset_17 (visible by anonymous)]]
+#
 #diff <(curl -s 'http://localhost:8080/api/datasets/17/versions/1?key=pete') scripts/search/tests/expected/dataset-versioning02raw
 # anon can now see the dataset and file and parents
 diff -u <(curl -s 'http://localhost:8080/api/search?q=trees&showrelevance=true') scripts/search/tests/expected/dataset-versioning02anon