diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 445a1a8a9b..b3f4404b2a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -212,7 +212,7 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder subject ); if (mq.isReadyToDownload(datasetAbilities)) { - setDownloadUrls(status, config.getResultProviders(), mq, uriBuilder, allProviders); + status.setResultUrls(getDownloadUrls(config.getResultProviders(), mq, uriBuilder, allProviders)); } return status; }); @@ -223,23 +223,18 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder * Sets the result urls for the given result renderer. Result urls are only rendered for providers that match the * result type of the execution. * - * @param status The status that is edited. * @param renderer The renderer that are requested for a result url generation. * @param exec The execution that is used for generating the url * @param uriBuilder The Uribuilder with the base configuration to generate the urls * @param allProviders If true, forces {@link ResultRendererProvider} to return an URL if possible. - * @param The type of the provided and returned status * @return The modified status */ - public static S setDownloadUrls(S status, List renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public static List getDownloadUrls(List renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { - List resultUrls = renderer.stream() - .map(r -> r.generateResultURLs(exec, uriBuilder.clone(), allProviders)) - .flatMap(Collection::stream).collect(Collectors.toList()); + return renderer.stream() + .map(r -> r.generateResultURLs(exec, uriBuilder.clone(), allProviders)) + .flatMap(Collection::stream).collect(Collectors.toList()); - status.setResultUrls(resultUrls); - - return status; } @@ -346,7 +341,7 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject final FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); if (query.isReadyToDownload(datasetAbilities)) { - setDownloadUrls(status, config.getResultProviders(), query, url, allProviders); + status.setResultUrls(getDownloadUrls(config.getResultProviders(), query, url, allProviders)); } return status; } @@ -425,11 +420,9 @@ public List getSingleEntityExport(Subject subject, UriBuilder uriBuilder, S throw ConqueryError.ContextError.fromErrorInfo(execution.getError()); } + // Use the provided format name to find the respective provider. - return config.getResultProviders().stream() - .map(resultRendererProvider -> resultRendererProvider.generateResultURLs(execution, uriBuilder.clone(), true)) - .flatMap(Collection::stream) - .collect(Collectors.toList()); + return getDownloadUrls(config.getResultProviders(), execution, uriBuilder, true); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java index 8fb639f9d0..81b28c9145 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java @@ -1,16 +1,12 @@ package com.bakdata.conquery.io.result.ResultRender; import java.net.URL; -import java.nio.charset.Charset; import java.util.Collection; -import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSBase; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ManagedExecution; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.dropwizard.jersey.setup.JerseyEnvironment; @@ -30,17 +26,5 @@ public interface ResultRendererProvider { */ Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders); - /** - * TODO write a bit more to this - * @param subject Subject trying to download the result - * @param exec execution for which the result should be provided - * @param dataset dataset of the execution - * @param pretty if true, use pretty pringting/human readable formats - * @param charset charset to use for encoding strings - * @param onClose Hook to run right after the result is finished downloading - * @return Response object containing data of the executions result. - */ - Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty, Charset charset, Runnable onClose); - void registerResultResource(JerseyEnvironment environment, ManagerNode manager); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 596fdebf9e..73a4037c42 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -49,7 +49,7 @@ public static & SingleTableResult> Response getAr boolean pretty, String fileExtension, MediaType mediaType, - ConqueryConfig config, Runnable onClose) { + ConqueryConfig config) { final Namespace namespace = datasetRegistry.get(dataset.getId()); @@ -99,7 +99,6 @@ public static & SingleTableResult> Response getAr } finally { log.trace("DONE downloading data for `{}`", exec.getId()); - onClose.run(); } }; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowFileResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowFileResultProvider.java index da5dead67c..b4d2b56db0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowFileResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowFileResultProvider.java @@ -6,7 +6,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.channels.Channels; -import java.nio.charset.Charset; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -58,8 +57,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u return List.of(ResultArrowFileResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec)); } - @Override - public Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty, Charset charset, Runnable onClose) { + public Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty) { return getArrowResult( (output) -> (root) -> new ArrowFileWriter(root, new DictionaryProvider.MapDictionaryProvider(), Channels.newChannel(output)), subject, @@ -69,7 +67,7 @@ public Response createResult(Subject subject, ManagedExecution exec, Dataset pretty, FILE_EXTENTION_ARROW_FILE, MEDIA_TYPE, - config, onClose + config ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowStreamResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowStreamResultProvider.java index 4729dddd13..3fe0f23fdd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowStreamResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowStreamResultProvider.java @@ -5,7 +5,6 @@ import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.Charset; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -56,8 +55,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u return List.of(ResultArrowStreamResource.getDownloadURL(uriBuilder, exec)); } - @Override - public Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty, Charset charset, Runnable onClose) { + public Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty) { return getArrowResult( (output) -> (root) -> new ArrowStreamWriter(root, new DictionaryProvider.MapDictionaryProvider(), output), subject, @@ -67,8 +65,7 @@ public Response createResult(Subject subject, ManagedExecution exec, Dataset pretty, FILE_EXTENTION_ARROW_STREAM, MEDIA_TYPE, - config, - onClose + config ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultRenderer.java b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultRenderer.java index c49065e00f..db70777381 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultRenderer.java @@ -72,8 +72,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u return List.of(ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec)); } - @Override - public Response createResult(Subject subject, ManagedExecution execRaw, Dataset dataset, boolean pretty, Charset charset, Runnable onClose) { + public Response createResult(Subject subject, ManagedExecution execRaw, Dataset dataset, boolean pretty, Charset charset) { final ManagedQuery exec = (ManagedQuery) execRaw; @@ -114,7 +113,7 @@ public Response createResult(Subject subject, ManagedExecution execRaw, Datas throw new WebApplicationException("Failed to load result", e); } finally { - onClose.run(); + log.trace("FINISHED downloading {}", exec.getId()); } }; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 8912a4d00a..6d1e512343 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -4,7 +4,6 @@ import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.Charset; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -36,9 +35,11 @@ import io.dropwizard.jersey.setup.JerseyEnvironment; import lombok.Data; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; @Data @CPSType(base = ResultRendererProvider.class, id = "XLSX") +@Slf4j public class ExcelResultProvider implements ResultRendererProvider { // Media type according to https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types @@ -77,8 +78,7 @@ public void registerResultResource(JerseyEnvironment environment, ManagerNode ma environment.register(ResultExcelResource.class); } - @Override - public Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty, Charset charset, Runnable onClose) { + public Response createResult(Subject subject, ManagedExecution exec, Dataset dataset, boolean pretty) { ConqueryMDC.setLocation(subject.getName()); final Namespace namespace = datasetRegistry.get(dataset.getId()); @@ -100,16 +100,12 @@ public Response createResult(Subject subject, ManagedExecution exec, Dataset ExcelRenderer excelRenderer = new ExcelRenderer(config.getExcel(), settings); StreamingOutput out = output -> { - try { - excelRenderer.renderToStream( - config.getFrontend().getQueryUpload().getIdResultInfos(), - (ManagedExecution & SingleTableResult) exec, - output - ); - } - finally { - onClose.run(); - } + excelRenderer.renderToStream( + config.getFrontend().getQueryUpload().getIdResultInfos(), + (ManagedExecution & SingleTableResult) exec, + output + ); + log.trace("FINISHED downloading {}", exec.getId()); }; return makeResponseWithFileName(out, exec.getLabelWithoutAutoLabelSuffix(), "xlsx", MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java index 14b26ca3f1..e7d4b654b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.resources.api; import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; -import static com.bakdata.conquery.io.result.ResultUtil.determineCharset; import static com.bakdata.conquery.resources.ResourceConstants.*; import java.net.MalformedURLException; @@ -43,12 +42,11 @@ public Response get( @PathParam(DATASET) Dataset dataset, @PathParam(QUERY) ManagedExecution query, @HeaderParam("subject-agent") String userAgent, - @QueryParam("charset") String queryCharset, @QueryParam("pretty") Optional pretty) { checkSingleTableResult(query); log.info("Result for {} download on dataset {} by subject {} ({}).", query.getId(), dataset.getId(), subject.getId(), subject.getName()); - return processor.createResult(subject, query, dataset, pretty.orElse(false), determineCharset(userAgent, queryCharset), () -> {}); + return processor.createResult(subject, query, dataset, pretty.orElse(false)); } public static & SingleTableResult> URL getDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java index 218e9bb44c..de28dcd730 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.resources.api; import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; -import static com.bakdata.conquery.io.result.ResultUtil.determineCharset; import static com.bakdata.conquery.resources.ResourceConstants.*; import java.net.MalformedURLException; @@ -50,15 +49,14 @@ public static URL getDownloadURL(UriBuilder uriBuilder, ManagedExecution exec @Path("{" + QUERY + "}." + FILE_EXTENTION_ARROW_STREAM) @Produces(AdditionalMediaTypes.ARROW_STREAM) public Response get( - @Auth Subject subject, - @PathParam(DATASET) Dataset dataset, - @PathParam(QUERY) ManagedExecution execution, - @HeaderParam("subject-agent") String userAgent, - @QueryParam("charset") String queryCharset, - @QueryParam("pretty") Optional pretty) + @Auth Subject subject, + @PathParam(DATASET) Dataset dataset, + @PathParam(QUERY) ManagedExecution execution, + @HeaderParam("subject-agent") String userAgent, + @QueryParam("pretty") Optional pretty) { checkSingleTableResult(execution); log.info("Result for {} download on dataset {} by subject {} ({}).", execution, dataset, subject.getId(), subject.getName()); - return processor.createResult(subject, execution, dataset, pretty.orElse(false), determineCharset(userAgent, queryCharset), () -> {}); + return processor.createResult(subject, execution, dataset, pretty.orElse(false)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java index 32f3f02af8..50c3052fba 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java @@ -62,7 +62,6 @@ public Response getAsCsv( @QueryParam("pretty") Optional pretty) { checkSingleTableResult(execution); log.info("Result for {} download on dataset {} by subject {} ({}).", execution, dataset.getId(), subject.getId(), subject.getName()); - return processor.createResult(subject, execution, dataset, pretty.orElse(Boolean.TRUE), determineCharset(userAgent, queryCharset), () -> { - }); + return processor.createResult(subject, execution, dataset, pretty.orElse(Boolean.TRUE), determineCharset(userAgent, queryCharset)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java index 2de8bd6568..ebd540525d 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.resources.api; import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; -import static com.bakdata.conquery.io.result.ResultUtil.determineCharset; import static com.bakdata.conquery.resources.ResourceConstants.DATASET; import static com.bakdata.conquery.resources.ResourceConstants.QUERY; @@ -37,21 +36,19 @@ public class ResultExcelResource { @Inject private ExcelResultProvider processor; - + @GET @Path("{" + QUERY + "}.xlsx") @Produces(AdditionalMediaTypes.EXCEL) public Response get( - @Auth Subject subject, - @PathParam(DATASET) Dataset dataset, - @PathParam(QUERY) ManagedExecution execution, - @HeaderParam("subject-agent") String userAgent, - @QueryParam("charset") String queryCharset, - @QueryParam("pretty") Optional pretty) { + @Auth Subject subject, + @PathParam(DATASET) Dataset dataset, + @PathParam(QUERY) ManagedExecution execution, + @HeaderParam("subject-agent") String userAgent, + @QueryParam("pretty") Optional pretty) { checkSingleTableResult(execution); log.info("Result for {} download on dataset {} by subject {} ({}).", execution.getId(), dataset, subject.getId(), subject.getName()); - return processor.createResult(subject, execution, dataset, pretty.orElse(true), determineCharset(userAgent, queryCharset), () -> { - }); + return processor.createResult(subject, execution, dataset, pretty.orElse(true)); } public static & SingleTableResult> URL getDownloadURL(UriBuilder uriBuilder, E exec) throws MalformedURLException {