Skip to content

Commit

Permalink
[RW-2704][risk=low] Split up listCluster API and move initialization …
Browse files Browse the repository at this point in the history 
…logic to the client-side. (#3135)
  • Loading branch information
@gjuggler
gjuggler authored Feb 27, 2020
1 parent d505c38 commit efc5fa2
Show file tree
Hide file tree
Showing 16 changed files with 1,116 additions and 483 deletions.
136 changes: 59 additions & 77 deletions api/src/main/java/org/pmiops/workbench/api/ClusterController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,11 @@
import com.google.gson.Gson;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Timestamp;
import java.time.Clock;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand All @@ -28,6 +26,7 @@
import org.pmiops.workbench.db.model.DbCdrVersion;
import org.pmiops.workbench.db.model.DbUser;
import org.pmiops.workbench.db.model.DbUser.ClusterConfig;
import org.pmiops.workbench.db.model.DbWorkspace;
import org.pmiops.workbench.exceptions.BadRequestException;
import org.pmiops.workbench.exceptions.NotFoundException;
import org.pmiops.workbench.exceptions.ServerErrorException;
Expand All @@ -38,14 +37,12 @@
import org.pmiops.workbench.model.ClusterLocalizeRequest;
import org.pmiops.workbench.model.ClusterLocalizeResponse;
import org.pmiops.workbench.model.ClusterStatus;
import org.pmiops.workbench.model.DefaultClusterResponse;
import org.pmiops.workbench.model.EmptyResponse;
import org.pmiops.workbench.model.ListClusterDeleteRequest;
import org.pmiops.workbench.model.ListClusterResponse;
import org.pmiops.workbench.model.UpdateClusterConfigRequest;
import org.pmiops.workbench.model.WorkspaceAccessLevel;
import org.pmiops.workbench.notebooks.LeonardoNotebooksClient;
import org.pmiops.workbench.notebooks.model.ClusterError;
import org.pmiops.workbench.notebooks.model.StorageLink;
import org.pmiops.workbench.workspaces.WorkspaceService;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -187,115 +184,99 @@ public ResponseEntity<List<ListClusterResponse>> deleteClustersInProject(
return ResponseEntity.ok(clustersInProjectAffected);
}

private DbWorkspace lookupWorkspace(String workspaceNamespace) throws NotFoundException {
return workspaceService
.getByNamespace(workspaceNamespace)
.orElseThrow(() -> new NotFoundException("Workspace not found: " + workspaceNamespace));
}

@Override
public ResponseEntity<DefaultClusterResponse> listClusters(
String billingProjectId, String workspaceFirecloudName) {
if (billingProjectId == null) {
throw new BadRequestException("Must specify billing project");
}
public ResponseEntity<Cluster> getCluster(String workspaceNamespace) {
String firecloudWorkspaceName = lookupWorkspace(workspaceNamespace).getFirecloudName();
workspaceService.enforceWorkspaceAccessLevel(
billingProjectId, workspaceFirecloudName, WorkspaceAccessLevel.READER);
workspaceService.validateActiveBilling(billingProjectId, workspaceFirecloudName);
workspaceNamespace, firecloudWorkspaceName, WorkspaceAccessLevel.WRITER);
workspaceService.validateActiveBilling(workspaceNamespace, firecloudWorkspaceName);

DbUser user = this.userProvider.get();
org.pmiops.workbench.notebooks.model.Cluster firecloudCluster =
leonardoNotebooksClient.getCluster(
workspaceNamespace, clusterNameForUser(userProvider.get()));

String clusterName = clusterNameForUser(user);
return ResponseEntity.ok(TO_ALL_OF_US_CLUSTER.apply(firecloudCluster));
}

org.pmiops.workbench.notebooks.model.Cluster fcCluster;
try {
fcCluster = this.leonardoNotebooksClient.getCluster(billingProjectId, clusterName);
} catch (NotFoundException e) {
fcCluster =
this.leonardoNotebooksClient.createCluster(
billingProjectId, clusterName, workspaceFirecloudName);
}
@Override
public ResponseEntity<Cluster> createCluster(String workspaceNamespace) {
String firecloudWorkspaceName = lookupWorkspace(workspaceNamespace).getFirecloudName();
workspaceService.enforceWorkspaceAccessLevel(
workspaceNamespace, firecloudWorkspaceName, WorkspaceAccessLevel.WRITER);
workspaceService.validateActiveBilling(workspaceNamespace, firecloudWorkspaceName);

int retries = Optional.ofNullable(user.getClusterCreateRetries()).orElse(0);
if (org.pmiops.workbench.notebooks.model.ClusterStatus.ERROR.equals(fcCluster.getStatus())) {
if (retries <= 2) {
this.userService.setClusterRetryCount(retries + 1);
log.warning("Cluster has errored with logs: ");
if (fcCluster.getErrors() != null) {
for (ClusterError e : fcCluster.getErrors()) {
log.warning(e.getErrorMessage());
}
}
log.warning("Retrying cluster creation.");

this.leonardoNotebooksClient.deleteCluster(billingProjectId, clusterName);
}
} else if (org.pmiops.workbench.notebooks.model.ClusterStatus.RUNNING.equals(
fcCluster.getStatus())
&& retries != 0) {
this.userService.setClusterRetryCount(0);
}
DefaultClusterResponse resp = new DefaultClusterResponse();
resp.setDefaultCluster(TO_ALL_OF_US_CLUSTER.apply(fcCluster));
return ResponseEntity.ok(resp);
org.pmiops.workbench.notebooks.model.Cluster firecloudCluster =
leonardoNotebooksClient.createCluster(
workspaceNamespace, clusterNameForUser(userProvider.get()), firecloudWorkspaceName);

return ResponseEntity.ok(TO_ALL_OF_US_CLUSTER.apply(firecloudCluster));
}

@Override
public ResponseEntity<EmptyResponse> deleteCluster(String projectName, String clusterName) {
this.userService.setClusterRetryCount(0);
this.leonardoNotebooksClient.deleteCluster(projectName, clusterName);
public ResponseEntity<EmptyResponse> deleteCluster(String workspaceNamespace) {
String firecloudWorkspaceName = lookupWorkspace(workspaceNamespace).getFirecloudName();
workspaceService.enforceWorkspaceAccessLevel(
workspaceNamespace, firecloudWorkspaceName, WorkspaceAccessLevel.WRITER);

leonardoNotebooksClient.deleteCluster(
workspaceNamespace, clusterNameForUser(userProvider.get()));
return ResponseEntity.ok(new EmptyResponse());
}

@Override
public ResponseEntity<ClusterLocalizeResponse> localize(
String projectName, String clusterName, ClusterLocalizeRequest body) {
String workspaceNamespace, ClusterLocalizeRequest body) {
DbWorkspace dbWorkspace = lookupWorkspace(workspaceNamespace);
workspaceService.enforceWorkspaceAccessLevel(
body.getWorkspaceNamespace(), body.getWorkspaceId(), WorkspaceAccessLevel.READER);
workspaceService.validateActiveBilling(body.getWorkspaceNamespace(), body.getWorkspaceId());
dbWorkspace.getWorkspaceNamespace(),
dbWorkspace.getFirecloudName(),
WorkspaceAccessLevel.WRITER);
workspaceService.validateActiveBilling(
dbWorkspace.getWorkspaceNamespace(), dbWorkspace.getFirecloudName());

FirecloudWorkspace fcWorkspace;
final FirecloudWorkspace firecloudWorkspace;
try {
fcWorkspace =
firecloudWorkspace =
fireCloudService
.getWorkspace(body.getWorkspaceNamespace(), body.getWorkspaceId())
.getWorkspace(dbWorkspace.getWorkspaceNamespace(), dbWorkspace.getFirecloudName())
.getWorkspace();
} catch (NotFoundException e) {
throw new NotFoundException(
String.format(
"workspace %s/%s not found or not accessible",
body.getWorkspaceNamespace(), body.getWorkspaceId()));
dbWorkspace.getWorkspaceNamespace(), dbWorkspace.getFirecloudName()));
}
DbCdrVersion cdrVersion =
workspaceService
.getRequired(body.getWorkspaceNamespace(), body.getWorkspaceId())
.getCdrVersion();
DbCdrVersion cdrVersion = dbWorkspace.getCdrVersion();

// For the common case where the notebook cluster matches the workspace
// namespace, simply name the directory as the workspace ID; else we
// include the namespace in the directory name to avoid possible conflicts
// in workspace IDs.
String gcsNotebooksDir = "gs://" + fcWorkspace.getBucketName() + "/notebooks";
Timestamp now = new Timestamp(clock.instant().toEpochMilli());
long workspaceId =
workspaceService
.getRequired(body.getWorkspaceNamespace(), body.getWorkspaceId())
.getWorkspaceId();
String gcsNotebooksDir = "gs://" + firecloudWorkspace.getBucketName() + "/notebooks";
long workspaceId = dbWorkspace.getWorkspaceId();

body.getNotebookNames()
.forEach(
notebook ->
notebookName ->
userRecentResourceService.updateNotebookEntry(
workspaceId, userProvider.get().getUserId(), gcsNotebooksDir + "/" + notebook));
String workspacePath = body.getWorkspaceId();
if (!projectName.equals(body.getWorkspaceNamespace())) {
workspacePath =
body.getWorkspaceNamespace()
+ FireCloudService.WORKSPACE_DELIMITER
+ body.getWorkspaceId();
}
workspaceId,
userProvider.get().getUserId(),
gcsNotebooksDir + "/" + notebookName));

String workspacePath = dbWorkspace.getFirecloudName();
String editDir = "workspaces/" + workspacePath;
String playgroundDir = "workspaces_playground/" + workspacePath;
String targetDir = body.getPlaygroundMode() ? playgroundDir : editDir;

leonardoNotebooksClient.createStorageLink(
projectName,
clusterName,
workspaceNamespace,
clusterNameForUser(userProvider.get()),
new StorageLink()
.cloudStorageDirectory(gcsNotebooksDir)
.localBaseDirectory(editDir)
Expand All @@ -307,7 +288,7 @@ public ResponseEntity<ClusterLocalizeResponse> localize(

// The Welder extension offers direct links to/from playground mode; write the AoU config file
// to both locations so notebooks will work in either directory.
String aouConfigUri = aouConfigDataUri(fcWorkspace, cdrVersion, projectName);
String aouConfigUri = aouConfigDataUri(firecloudWorkspace, cdrVersion, workspaceNamespace);
localizeMap.put(editDir + "/" + AOU_CONFIG_FILENAME, aouConfigUri);
localizeMap.put(playgroundDir + "/" + AOU_CONFIG_FILENAME, aouConfigUri);

Expand All @@ -319,8 +300,9 @@ public ResponseEntity<ClusterLocalizeResponse> localize(
Collectors.toMap(
name -> targetDir + "/" + name, name -> gcsNotebooksDir + "/" + name)));
}

leonardoNotebooksClient.localize(projectName, clusterName, localizeMap);
log.info(localizeMap.toString());
leonardoNotebooksClient.localize(
workspaceNamespace, clusterNameForUser(userProvider.get()), localizeMap);

// This is the Jupyer-server-root-relative path, the style used by the Jupyter REST API.
return ResponseEntity.ok(new ClusterLocalizeResponse().clusterLocalDirectory(targetDir));
Expand Down
8 changes: 7 additions & 1 deletion api/src/main/java/org/pmiops/workbench/notebooks/LeonardoNotebooksClientImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.pmiops.workbench.db.model.DbUser.ClusterConfig;
import org.pmiops.workbench.db.model.DbWorkspace;
import org.pmiops.workbench.db.model.DbWorkspace.BillingMigrationStatus;
import org.pmiops.workbench.exceptions.ExceptionUtils;
import org.pmiops.workbench.notebooks.api.ClusterApi;
import org.pmiops.workbench.notebooks.api.NotebooksApi;
import org.pmiops.workbench.notebooks.api.StatusApi;
Expand Down Expand Up @@ -171,7 +172,12 @@ public void deleteCluster(String googleProject, String clusterName) {
@Override
public Cluster getCluster(String googleProject, String clusterName) {
ClusterApi clusterApi = clusterApiProvider.get();
return retryHandler.run((context) -> clusterApi.getCluster(googleProject, clusterName));
try {
return retryHandler.runAndThrowChecked(
(context) -> clusterApi.getCluster(googleProject, clusterName));
} catch (ApiException e) {
throw ExceptionUtils.convertNotebookException(e);
}
}

@Override
Expand Down
3 changes: 3 additions & 0 deletions api/src/main/java/org/pmiops/workbench/workspaces/WorkspaceService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.pmiops.workbench.db.model.DbUserRecentWorkspace;
import org.pmiops.workbench.db.model.DbWorkspace;
import org.pmiops.workbench.exceptions.ForbiddenException;
import org.pmiops.workbench.exceptions.NotFoundException;
import org.pmiops.workbench.firecloud.FireCloudService;
import org.pmiops.workbench.firecloud.model.FirecloudWorkspaceACLUpdate;
import org.pmiops.workbench.firecloud.model.FirecloudWorkspaceAccessEntry;
Expand All @@ -33,6 +34,8 @@ public interface WorkspaceService {

List<WorkspaceResponse> getWorkspacesAndPublicWorkspaces();

WorkspaceResponse getWorkspace(String workspaceNamespace) throws NotFoundException;

WorkspaceResponse getWorkspace(String workspaceNamespace, String workspaceId);

List<WorkspaceResponse> getWorkspaces();
Expand Down
16 changes: 15 additions & 1 deletion api/src/main/java/org/pmiops/workbench/workspaces/WorkspaceServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,18 +204,32 @@ public List<WorkspaceResponse> getWorkspacesAndPublicWorkspaces() {
.collect(Collectors.toList());
}

@Transactional
@Override
public WorkspaceResponse getWorkspace(String workspaceNamespace) throws NotFoundException {
DbWorkspace dbWorkspace =
getByNamespace(workspaceNamespace)
.orElseThrow(() -> new NotFoundException("Workspace not found: " + workspaceNamespace));
return getWorkspaceImpl(dbWorkspace);
}

@Transactional
@Override
public WorkspaceResponse getWorkspace(String workspaceNamespace, String workspaceId) {
DbWorkspace dbWorkspace = getRequired(workspaceNamespace, workspaceId);
return getWorkspaceImpl(dbWorkspace);
}

private WorkspaceResponse getWorkspaceImpl(DbWorkspace dbWorkspace) {
FirecloudWorkspaceResponse fcResponse;
FirecloudWorkspace fcWorkspace;

WorkspaceResponse workspaceResponse = new WorkspaceResponse();

// This enforces access controls.
fcResponse = fireCloudService.getWorkspace(workspaceNamespace, workspaceId);
fcResponse =
fireCloudService.getWorkspace(
dbWorkspace.getWorkspaceNamespace(), dbWorkspace.getFirecloudName());
fcWorkspace = fcResponse.getWorkspace();

if (fcResponse.getAccessLevel().equals(WorkspaceService.PROJECT_OWNER_ACCESS_LEVEL)) {
Expand Down
Loading

0 comments on commit efc5fa2

Please sign in to comment.