Skip to content

Commit

Permalink
Merge pull request #6333 from IQSS/6301-search-unpublished
Browse files Browse the repository at this point in the history 
Search API no longer limited to published data #6301
  • Loading branch information
@kcondon
kcondon authored Nov 1, 2019
2 parents b18f706 + 7ebcc3f commit b46243d
Show file tree
Hide file tree
Showing 6 changed files with 30 additions and 119 deletions.
2 changes: 1 addition & 1 deletion doc/sphinx-guides/source/api/search.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Search API

The Search API supports the same searching, sorting, and faceting operations as the Dataverse web interface.

Unlike the web interface, this new API is limited to *published* data.
To search unpublished content, you must pass in an API token as described in the :doc:`auth` section.

The parameters and JSON response are partly inspired by the `GitHub Search API <https://developer.github.com/v3/search/>`_.

Expand Down
28 changes: 9 additions & 19 deletions src/main/java/edu/harvard/iq/dataverse/api/Search.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,14 @@ public Response search(
try {
if (!types.isEmpty()) {
filterQueries.add(getFilterQueryFromTypes(types));
} else {
/**
* Added to prevent a NullPointerException for superusers
* (who don't use our permission JOIN) when
* SearchServiceBean tries to get SearchFields.TYPE. The GUI
* always seems to add SearchFields.TYPE, even for superusers.
*/
filterQueries.add(SearchFields.TYPE + ":(" + SearchConstants.DATAVERSES + " OR " + SearchConstants.DATASETS + " OR " + SearchConstants.FILES + ")");
}
sortBy = SearchUtil.getSortBy(sortField, sortOrder);
numResultsPerPage = getNumberOfResultsPerPage(numResultsPerPageRequested);
Expand Down Expand Up @@ -202,15 +210,6 @@ public Response search(
}

private User getUser() throws WrappedResponse {
/**
* @todo support searching as non-guest:
* https://github.com/IQSS/dataverse/issues/1299
*
* Note that superusers can't currently use the Search API because they
* see permission documents (all Solr documents, really) and we get a
* NPE when trying to determine the DvObject type if their query matches
* a permission document.
*/
User userToExecuteSearchAs = GuestUser.get();
try {
AuthenticatedUser authenticatedUser = findAuthenticatedUserOrDie();
Expand All @@ -222,16 +221,7 @@ private User getUser() throws WrappedResponse {
throw ex;
}
}
if (nonPublicSearchAllowed()) {
return userToExecuteSearchAs;
} else {
return GuestUser.get();
}
}

public boolean nonPublicSearchAllowed() {
boolean safeDefaultIfKeyNotFound = false;
return settingsSvc.isTrueForKey(SettingsServiceBean.Key.SearchApiNonPublicAllowed, safeDefaultIfKeyNotFound);
return userToExecuteSearchAs;
}

public boolean tokenLessSearchAllowed() {
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -883,8 +883,9 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
// Yes, give back everything
// ----------------------------------------------------
if (au.isSuperuser()) {
// dangerous because this user will be able to see
// EVERYTHING in Solr with no regard to permissions!
// Somewhat dangerous because this user (a superuser) will be able
// to see everything in Solr with no regard to permissions. But it's
// been this way since Dataverse 4.0. So relax. :)

return dangerZoneNoSolrJoin;
}
Expand Down
5 changes: 0 additions & 5 deletions src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,6 @@ public enum Key {
* shouldn't.
*/
MyDataDoesNotUseSolrPermissionDocs,
/**
* Experimental: Allow non-public search with a key/token using the
* Search API. See also https://github.com/IQSS/dataverse/issues/1299
*/
SearchApiNonPublicAllowed,
/**
* In Dataverse 4.7 and earlier, an API token was required to use the
* Search API. Tokens are no longer required but you can revert to the
Expand Down
18 changes: 1 addition & 17 deletions src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,6 @@ public class FilesIT {
@BeforeClass
public static void setUpClass() {
RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();

Response removeSearchApiNonPublicAllowed = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
removeSearchApiNonPublicAllowed.prettyPrint();
removeSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);

}

/**
Expand Down Expand Up @@ -1098,12 +1092,7 @@ public void testAccessFacet() {
Response searchShouldFindNothingBecauseUnpublished = UtilIT.search("id:datafile_" + fileId + "_draft", apiToken);
searchShouldFindNothingBecauseUnpublished.prettyPrint();
searchShouldFindNothingBecauseUnpublished.then().assertThat()
// This is normal. "limited to published data" http://guides.dataverse.org/en/4.7/api/search.html
.body("data.total_count", equalTo(0))
.statusCode(OK.getStatusCode());
// Let's temporarily allow searching of drafts.
Response enableNonPublicSearch = UtilIT.enableSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
enableNonPublicSearch.then().assertThat()
.body("data.total_count", equalTo(1))
.statusCode(OK.getStatusCode());

Response searchResponse = UtilIT.searchAndShowFacets("id:datafile_" + fileId + "_draft", apiToken);
Expand All @@ -1119,11 +1108,6 @@ public void testAccessFacet() {
.body("data.facets[0].publicationStatus", CoreMatchers.not(equalTo(null)))
.statusCode(OK.getStatusCode());

Response removeSearchApiNonPublicAllowed = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
removeSearchApiNonPublicAllowed.prettyPrint();
removeSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);

//reset public install
UtilIT.setSetting(SettingsServiceBean.Key.PublicInstall, "false");

Expand Down
91 changes: 16 additions & 75 deletions src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,6 @@ public static void setUpClass() {
makeSureTokenlessSearchIsEnabled.then().assertThat()
.statusCode(OK.getStatusCode());

Response removeSearchApiNonPublicAllowed = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
removeSearchApiNonPublicAllowed.prettyPrint();
removeSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);

Response remove = UtilIT.deleteSetting(SettingsServiceBean.Key.ThumbnailSizeLimitImage);
remove.then().assertThat()
.statusCode(200);
Expand All @@ -74,10 +69,6 @@ public void testSearchPermisions() throws InterruptedException {

Integer datasetId1 = UtilIT.getDatasetIdFromResponse(createDataset1Response);

Response enableNonPublicSearch = UtilIT.enableSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
enableNonPublicSearch.then().assertThat()
.statusCode(OK.getStatusCode());

Response shouldBeVisibleToUser1 = UtilIT.search("id:dataset_" + datasetId1 + "_draft", apiToken1);
shouldBeVisibleToUser1.prettyPrint();
shouldBeVisibleToUser1.then().assertThat()
Expand Down Expand Up @@ -131,9 +122,6 @@ public void testSearchPermisions() throws InterruptedException {
publishDataset.then().assertThat()
.statusCode(OK.getStatusCode());

Response disableNonPublicSearch = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
disableNonPublicSearch.then().assertThat()
.statusCode(OK.getStatusCode());

Response makeSureTokenlessSearchIsEnabled = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiRequiresToken);
makeSureTokenlessSearchIsEnabled.then().assertThat()
Expand Down Expand Up @@ -188,10 +176,6 @@ public void testSearchCitation() {
createDatasetResponse.prettyPrint();
Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);

Response enableNonPublicSearch = UtilIT.enableSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
enableNonPublicSearch.then().assertThat()
.statusCode(OK.getStatusCode());

Response searchResponse = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken);
searchResponse.prettyPrint();
assertFalse(searchResponse.body().jsonPath().getString("data.items[0].citation").contains("href"));
Expand All @@ -211,10 +195,6 @@ public void testSearchCitation() {
deleteDataverseResponse.then().assertThat()
.statusCode(OK.getStatusCode());

Response disableNonPublicSearch = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
disableNonPublicSearch.then().assertThat()
.statusCode(OK.getStatusCode());

Response deleteUserResponse = UtilIT.deleteUser(username);
deleteUserResponse.prettyPrint();
assertEquals(200, deleteUserResponse.getStatusCode());
Expand All @@ -232,14 +212,6 @@ public void testSearchCitation() {
public void testDatasetThumbnail() {
logger.info("BEGIN testDatasetThumbnail");

// Response setSearchApiNonPublicAllowed = UtilIT.setSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed, "true");
// setSearchApiNonPublicAllowed.prettyPrint();
//
// assertEquals("foo", "foo");
// if (true) {
// return;
// }
//
Response createUser = UtilIT.createRandomUser();
createUser.prettyPrint();
String username = UtilIT.getUsernameFromResponse(createUser);
Expand All @@ -253,11 +225,6 @@ public void testDatasetThumbnail() {
createDatasetResponse.prettyPrint();
Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse);

Response setSearchApiNonPublicAllowed = UtilIT.setSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed, "true");
setSearchApiNonPublicAllowed.prettyPrint();
setSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);

Response search1 = UtilIT.search("id:dataset_" + datasetId + "_draft", apiToken);
search1.prettyPrint();
search1.then().assertThat()
Expand Down Expand Up @@ -612,10 +579,6 @@ public void testDatasetThumbnail() {
searchResponse.then().assertThat()
.statusCode(OK.getStatusCode());

Response removeSearchApiNonPublicAllowed = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
removeSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);

/**
* @todo What happens when you delete a dataset? Does the thumbnail
* created based on the logo get deleted too? Should it?
Expand Down Expand Up @@ -653,8 +616,7 @@ public void testIdentifier() {
searchUnpublished.prettyPrint();
searchUnpublished.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
.body("data.total_count", CoreMatchers.equalTo(0));
.body("data.total_count", CoreMatchers.equalTo(1));

Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
publishDataverse.then().assertThat()
Expand Down Expand Up @@ -701,16 +663,15 @@ public void testNestedSubtree() {
searchUnpublishedSubtree.prettyPrint();
searchUnpublishedSubtree.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
.body("data.total_count", CoreMatchers.equalTo(0));
.body("data.total_count", CoreMatchers.equalTo(1));

Response searchUnpublishedSubtree2 = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias2);
searchUnpublishedSubtree2.prettyPrint();
searchUnpublishedSubtree2.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
// TODO: investigate if this is a bug that nothing was found.
.body("data.total_count", CoreMatchers.equalTo(0));

Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
publishDataverse.then().assertThat()
.statusCode(OK.getStatusCode());
Expand Down Expand Up @@ -751,6 +712,7 @@ public void testNestedSubtree() {
searchPublishedSubtree2.prettyPrint();
searchPublishedSubtree2.then().assertThat()
.statusCode(OK.getStatusCode())
// TODO: investigate if this is a bug that nothing was found.
.body("data.total_count", CoreMatchers.equalTo(0));

Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias2, apiToken);
Expand Down Expand Up @@ -782,11 +744,7 @@ public void testNestedSubtree() {
//Hopefully it will not fail as we fixed the issue in https://github.com/IQSS/dataverse/issues/3471
@Test
public void testCuratorCardDataversePopulation() throws InterruptedException {
Response setSearchApiNonPublicAllowed = UtilIT.setSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed, "true");
setSearchApiNonPublicAllowed.prettyPrint();
setSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);


Response createSuperUser = UtilIT.createRandomUser();
createSuperUser.prettyPrint();
assertEquals(200, createSuperUser.getStatusCode());
Expand Down Expand Up @@ -886,63 +844,59 @@ public void testSubtreePermissions() {
searchFakeSubtreeNoAPI.prettyPrint();
searchFakeSubtreeNoAPI.then().assertThat()
.statusCode(400);

Response searchUnpublishedSubtree = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias);
searchUnpublishedSubtree.prettyPrint();
searchUnpublishedSubtree.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
.body("data.total_count", CoreMatchers.equalTo(0));
.body("data.total_count", CoreMatchers.equalTo(1));

Response searchUnpublishedSubtreeNoAPI = UtilIT.search(searchPart, null, "&subtree="+dataverseAlias);
searchUnpublishedSubtreeNoAPI.prettyPrint();
searchUnpublishedSubtreeNoAPI.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
// TODO: investigate if this is a bug that nothing was found.
.body("data.total_count", CoreMatchers.equalTo(0));

Response searchUnpublishedSubtrees = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias +"&subtree="+dataverseAlias2);
searchUnpublishedSubtrees.prettyPrint();
searchUnpublishedSubtrees.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find them because they haven't been published.
.body("data.total_count", CoreMatchers.equalTo(0));
.body("data.total_count", CoreMatchers.equalTo(2));

Response searchUnpublishedSubtreesNoAPI = UtilIT.search(searchPart, null, "&subtree="+dataverseAlias +"&subtree="+dataverseAlias2);
searchUnpublishedSubtreesNoAPI.prettyPrint();
searchUnpublishedSubtreesNoAPI.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find them because they haven't been published.
// TODO: investigate if this is a bug that nothing was found.
.body("data.total_count", CoreMatchers.equalTo(0));

Response searchUnpublishedRootSubtreeForDataset = UtilIT.search(identifier.replace("FK2/", ""), apiToken, "&subtree=root");
searchUnpublishedRootSubtreeForDataset.prettyPrint();
searchUnpublishedRootSubtreeForDataset.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
.body("data.total_count", CoreMatchers.equalTo(0));
.body("data.total_count", CoreMatchers.equalTo(1));

Response searchUnpublishedRootSubtreeForDatasetNoAPI = UtilIT.search(identifier.replace("FK2/", ""), null, "&subtree=root");
searchUnpublishedRootSubtreeForDatasetNoAPI.prettyPrint();
searchUnpublishedRootSubtreeForDatasetNoAPI.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
// TODO: investigate if this is a bug that nothing was found.
.body("data.total_count", CoreMatchers.equalTo(0));

Response searchUnpublishedNoSubtreeForDataset = UtilIT.search(identifier.replace("FK2/", ""), apiToken, "");
searchUnpublishedNoSubtreeForDataset.prettyPrint();
searchUnpublishedNoSubtreeForDataset.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
.body("data.total_count", CoreMatchers.equalTo(0));
.body("data.total_count", CoreMatchers.equalTo(1));

Response searchUnpublishedNoSubtreeForDatasetNoAPI = UtilIT.search(identifier.replace("FK2/", ""), null, "");
searchUnpublishedNoSubtreeForDatasetNoAPI.prettyPrint();
searchUnpublishedNoSubtreeForDatasetNoAPI.then().assertThat()
.statusCode(OK.getStatusCode())
// It's expected that you can't find it because it hasn't been published.
// TODO: investigate if this is a bug that nothing was found.
.body("data.total_count", CoreMatchers.equalTo(0));

//PUBLISH

Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
Expand Down Expand Up @@ -1010,19 +964,6 @@ public void tearDownDataverse() {

@AfterClass
public static void cleanup() {

Response enableNonPublicSearch = UtilIT.enableSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
assertEquals(200, enableNonPublicSearch.getStatusCode());

Response deleteSearchApiNonPublicAllowed = UtilIT.deleteSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
deleteSearchApiNonPublicAllowed.then().assertThat()
.statusCode(200);

Response getSearchApiNonPublicAllowed = UtilIT.getSetting(SettingsServiceBean.Key.SearchApiNonPublicAllowed);
// getSearchApiNonPublicAllowed.prettyPrint();
getSearchApiNonPublicAllowed.then().assertThat()
.body("message", CoreMatchers.equalTo("Setting " + SettingsServiceBean.Key.SearchApiNonPublicAllowed + " not found"))
.statusCode(404);
}

}

0 comments on commit b46243d

Please sign in to comment.