Admin ui for live view of queries

backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java

@@ -1,11 +1,13 @@
 package com.bakdata.conquery.apiv1;
 
 import java.net.URL;
+import java.time.LocalDateTime;
 import java.time.ZonedDateTime;
 import java.util.Collections;
 import java.util.List;
 
 import com.bakdata.conquery.models.execution.ExecutionState;
+import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId;
 import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
@@ -38,6 +40,9 @@ public abstract class ExecutionStatus {
 	private Long numberOfResults;
 	private Long requiredTime;
 
+	private LocalDateTime startTime;
+	private LocalDateTime finishTime;
+
 	private String queryType;
 	private SecondaryIdDescriptionId secondaryId;
 

backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java

@@ -16,6 +16,9 @@
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldNameConstants;
+import com.bakdata.conquery.models.i18n.I18n;
+
+import static org.apache.shiro.util.StringUtils.hasText;
 
 /**
  * This status holds extensive information about the query description and meta data that is computational heavy

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -200,8 +200,7 @@ public Stream<ExecutionStatus> getQueriesFiltered(Dataset datasetId, UriBuilder
 						 .map(mq -> {
 							 OverviewExecutionStatus status = mq.buildStatusOverview(
 									 uriBuilder.clone(),
-									 user,
-									 datasetAbilities
+									 user
 							 );
 							 if (mq.isReadyToDownload(datasetAbilities)) {
 								 setDownloadUrls(status, config.getResultProviders(), mq, uriBuilder, allProviders);
@@ -361,7 +360,7 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution<?> query, User us
 		query.initExecutable(datasetRegistry, config);
 
 		Map<DatasetId, Set<Ability>> datasetAbilities = buildDatasetAbilityMap(user, datasetRegistry);
-		final FullExecutionStatus status = query.buildStatusFull(storage, url, user, datasetRegistry, datasetAbilities);
+		final FullExecutionStatus status = query.buildStatusFull(storage, user, datasetRegistry);
 
 		if (query.isReadyToDownload(datasetAbilities)) {
 			setDownloadUrls(status, config.getResultProviders(), query, url, allProviders);

backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java

@@ -55,15 +55,13 @@
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.Uninterruptibles;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.NonNull;
-import lombok.Setter;
-import lombok.ToString;
+import lombok.*;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.shiro.authz.Permission;
 
+import static org.apache.shiro.util.StringUtils.hasText;
+
 @Getter
 @Setter
 @ToString
@@ -238,7 +236,7 @@ public void awaitDone(int time, TimeUnit unit) {
 		Uninterruptibles.awaitUninterruptibly(execution, time, unit);
 	}
 
-	protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status, UriBuilder url, Map<DatasetId, Set<Ability>> datasetAbilities) {
+	protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status) {
 		status.setLabel(label == null ? queryId.toString() : getLabelWithoutAutoLabelSuffix());
 		status.setPristineLabel(label == null || queryId.toString().equals(label) || isAutoLabeled());
 		status.setId(getId());
@@ -247,6 +245,8 @@ protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status
 		status.setOwn(user.isOwner(this));
 		status.setCreatedAt(getCreationTime().atZone(ZoneId.systemDefault()));
 		status.setRequiredTime((startTime != null && finishTime != null) ? ChronoUnit.MILLIS.between(startTime, finishTime) : null);
+		status.setStartTime(startTime);
+		status.setFinishTime(finishTime);
 		status.setStatus(getState());
 		if (owner != null) {
 			status.setOwner(owner.getId());
@@ -257,9 +257,9 @@ protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status
 	/**
 	 * Renders a lightweight status with meta information about this query. Computation an size should be small for this.
 	 */
-	public OverviewExecutionStatus buildStatusOverview(UriBuilder url, User user, Map<DatasetId, Set<Ability>> datasetAbilities) {
+	public OverviewExecutionStatus buildStatusOverview(UriBuilder url, User user) {
 		OverviewExecutionStatus status = new OverviewExecutionStatus();
-		setStatusBase(user, status, url, datasetAbilities);
+		setStatusBase(user, status);
 
 		return status;
 	}
@@ -268,12 +268,12 @@ public OverviewExecutionStatus buildStatusOverview(UriBuilder url, User user, Ma
 	 * Renders an extensive status of this query (see {@link FullExecutionStatus}. The rendering can be computation intensive and can produce a large
 	 * object. The use  of the full status is only intended if a client requested specific information about this execution.
 	 */
-	public FullExecutionStatus buildStatusFull(@NonNull MetaStorage storage, UriBuilder url, User user, DatasetRegistry datasetRegistry, Map<DatasetId, Set<Ability>> datasetAbilities) {
+	public FullExecutionStatus buildStatusFull(@NonNull MetaStorage storage, User user, DatasetRegistry datasetRegistry) {
 		Preconditions.checkArgument(isInitialized(), "The execution must have been initialized first");
 		FullExecutionStatus status = new FullExecutionStatus();
-		setStatusBase(user, status, url, datasetAbilities);
+		setStatusBase(user, status);
 
-		setAdditionalFieldsForStatusWithColumnDescription(storage, url, user, status, datasetRegistry);
+		setAdditionalFieldsForStatusWithColumnDescription(storage, user, status, datasetRegistry);
 		setAdditionalFieldsForStatusWithSource(user, status);
 		setAdditionalFieldsForStatusWithGroups(storage, status);
 		setAvailableSecondaryIds(status);
@@ -313,7 +313,7 @@ private void setAdditionalFieldsForStatusWithGroups(@NonNull MetaStorage storage
 		status.setGroups(permittedGroups);
 	}
 
-	protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaStorage storage, UriBuilder url, User user, FullExecutionStatus status, DatasetRegistry datasetRegistry) {
+	protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaStorage storage, User user, FullExecutionStatus status, DatasetRegistry datasetRegistry) {
 		// Implementation specific
 	}
 

backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java

@@ -8,8 +8,6 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
-import javax.ws.rs.core.UriBuilder;
-
 import com.bakdata.conquery.apiv1.FullExecutionStatus;
 import com.bakdata.conquery.apiv1.forms.Form;
 import com.bakdata.conquery.apiv1.query.QueryDescription;
@@ -179,8 +177,8 @@ public QueryDescription getSubmitted() {
 
 
 	@Override
-	protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaStorage storage, UriBuilder url, User user, FullExecutionStatus status, DatasetRegistry datasetRegistry) {
-		super.setAdditionalFieldsForStatusWithColumnDescription(storage, url, user, status, datasetRegistry);
+	protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaStorage storage, User user, FullExecutionStatus status, DatasetRegistry datasetRegistry) {
+		super.setAdditionalFieldsForStatusWithColumnDescription(storage, user, status, datasetRegistry);
 		// Set the ColumnDescription if the Form only consits of a single subquery
 		if (subQueries == null) {
 			// If subqueries was not set the Execution was not initialized, do it manually

backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java

@@ -12,8 +12,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import javax.ws.rs.core.UriBuilder;
-
 import c10n.C10N;
 import com.bakdata.conquery.apiv1.ExecutionStatus;
 import com.bakdata.conquery.apiv1.FullExecutionStatus;
@@ -27,15 +25,13 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
-import com.bakdata.conquery.models.auth.permissions.Ability;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.execution.ExecutionState;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.externalservice.ResultType;
 import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable;
-import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.messages.namespaces.WorkerMessage;
 import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteQuery;
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
@@ -136,8 +132,8 @@ public void start() {
 	}
 
 	@Override
-	protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status, UriBuilder url, Map<DatasetId, Set<Ability>> datasetAbilities) {
-		super.setStatusBase(user, status, url, datasetAbilities);
+	protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status) {
+		super.setStatusBase(user, status);
 		status.setNumberOfResults(lastResultCount);
 
 		status.setQueryType(query.getClass().getAnnotation(CPSType.class).id());
@@ -148,8 +144,8 @@ protected void setStatusBase(@NonNull User user, @NonNull ExecutionStatus status
 	}
 
 	@Override
-	protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaStorage storage, UriBuilder url, User user, FullExecutionStatus status, DatasetRegistry datasetRegistry) {
-		super.setAdditionalFieldsForStatusWithColumnDescription(storage, url, user, status, datasetRegistry);
+	protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaStorage storage, User user, FullExecutionStatus status, DatasetRegistry datasetRegistry) {
+		super.setAdditionalFieldsForStatusWithColumnDescription(storage, user, status, datasetRegistry);
 		if (columnDescriptions == null) {
 			columnDescriptions = generateColumnDescriptions(datasetRegistry);
 		}

backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java

@@ -12,6 +12,7 @@ public class ResourceConstants {
 	public static final String TABLE	=	"table";
 	public static final String FILTER	=	"filter";
 	public static final String QUERY	=	"query";
+	public static final String QUERIES_MONITOR_PATH_ELEMENT	=	"queries";
 	public static final String FORM_CONFIG	=	"form-config";
 	public static final String FILENAME	=	"filename";
 	public static final String API		=	"api";

backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java

@@ -1,18 +1,25 @@
 package com.bakdata.conquery.resources.admin.rest;
 
-import com.bakdata.conquery.io.jackson.Jackson;
+import com.bakdata.conquery.apiv1.FullExecutionStatus;
+import com.bakdata.conquery.apiv1.query.ConceptQuery;
+import com.bakdata.conquery.apiv1.query.Query;
+import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd;
+import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.io.jersey.ExtraMimeTypes;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
-import com.bakdata.conquery.models.common.Range;
+import com.bakdata.conquery.models.error.ConqueryError;
+import com.bakdata.conquery.models.execution.ExecutionState;
+import com.bakdata.conquery.models.execution.ManagedExecution;
+import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
-import com.bakdata.conquery.models.jobs.Job;
+import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
 import com.bakdata.conquery.models.jobs.JobManagerStatus;
 import com.bakdata.conquery.models.messages.network.specific.CancelJobMessage;
+import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.ShardNodeInformation;
 import com.bakdata.conquery.resources.admin.ui.AdminUIResource;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.Uninterruptibles;
 import io.dropwizard.auth.Auth;
 
 import javax.inject.Inject;
@@ -21,12 +28,12 @@
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 import java.time.LocalDate;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.util.*;
 
 import static com.bakdata.conquery.resources.ResourceConstants.JOB_ID;
+import static org.apache.shiro.util.StringUtils.hasText;
 
 @Consumes({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING})
 @Produces(ExtraMimeTypes.JSON_STRING)
@@ -52,7 +59,6 @@ public String executeScript(@Auth User user, String script) {
      * Execute script and serialize return value as Json.
      * Useful for configuration and verification scripts.
      */
-    @Produces(MediaType.APPLICATION_JSON)
     @Consumes(MediaType.TEXT_PLAIN)
     @POST
     @Path("/script")
@@ -79,6 +85,19 @@ public Response cancelJob(@PathParam(JOB_ID) UUID jobId) {
     @GET
     @Path("/jobs/")
     public ImmutableMap<String, JobManagerStatus> getJobs() {
-        return  processor.getJobs();
+        return processor.getJobs();
     }
-}
+
+
+    @GET
+    @Path("/queries")
+    public FullExecutionStatus[] getQueries(@Auth User currentUser, @QueryParam("limit") OptionalLong limit, @QueryParam("since") Optional<String> since) {
+        final MetaStorage storage = processor.getStorage();
+        final DatasetRegistry datasetRegistry = processor.getDatasetRegistry();
+        return storage.getAllExecutions().stream()
+                .map(t -> t.buildStatusFull(storage, currentUser, datasetRegistry))
+                .filter(t -> t.getCreatedAt().toLocalDate().isEqual(since.map(LocalDate::parse).orElse(LocalDate.now())))
+                .limit(limit.orElse(100))
+                .toArray(FullExecutionStatus[]::new);
+    }
+}

backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java

@@ -179,7 +179,6 @@ public ImportStatistics getImportStatistics(Import imp) {
 		return new ImportStatistics(imp, cBlockSize);
 	}
 
-
 	public static long calculateCBlocksSizeBytes(Import imp, Collection<? extends Concept<?>> concepts) {
 
 		// CBlocks are created per (per Bucket) Import per Connector targeting this table

backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java

@@ -73,4 +73,10 @@ public View getJobs() {
 		return new UIView<>("jobs.html.ftl", uiProcessor.getUIContext(), uiProcessor.getAdminProcessor().getJobs());
 	}
 
+	@GET
+	@Path("queries")
+	public View getQueries() {
+		return new UIView<>("queries.html.ftl", uiProcessor.getUIContext());
+	}
+
 }