don't double count tabular files in size #6118

IQSS · Jul 7, 2020 · 1957776 · 1957776
1 parent 427c883
commit 1957776
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 19 deletions.
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -2901,7 +2901,21 @@ public void setSelectedNonDownloadallableFiles(List<FileMetadata> selectedNonDow
     }
 
     public String getSizeOfDataset() {
-        GetDatasetStorageSizeCommand cmd = new GetDatasetStorageSizeCommand(dvRequestService.getDataverseRequest(), dataset, false, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, workingVersion);
+        boolean countCachedFiles = false;
+        boolean useOrigFileSize = false;
+        GetDatasetStorageSizeCommand cmd = new GetDatasetStorageSizeCommand(dvRequestService.getDataverseRequest(), dataset, countCachedFiles, useOrigFileSize, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, workingVersion);
+        try {
+            long bytes = commandEngine.submit(cmd);
+            return FileSizeChecker.bytesToHumanReadable(bytes);
+        } catch (CommandException ex) {
+            return "";
+        }
+    }
+
+    public String getSizeOfDatasetOrig() {
+        boolean countCachedFiles = false;
+        boolean useOrigFileSize = true;
+        GetDatasetStorageSizeCommand cmd = new GetDatasetStorageSizeCommand(dvRequestService.getDataverseRequest(), dataset, countCachedFiles, useOrigFileSize, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, workingVersion);
         try {
             long bytes = commandEngine.submit(cmd);
             return FileSizeChecker.bytesToHumanReadable(bytes);

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
@@ -880,11 +880,16 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
         return findStorageSize(dataset, countCachedExtras, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
     }
 
+    public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
+        boolean useOrigFileSize = false;
+        return findStorageSize(dataset, countCachedExtras, useOrigFileSize, mode, version);
+    }
     /**
      * Returns the total byte size of the files in this dataset 
      * 
      * @param dataset
      * @param countCachedExtras boolean indicating if the cached disposable extras should also be counted
+     * @param useOrigFileSize allows original tabular file size to be used instead of derived archival file
      * @param mode String indicating whether we are getting the result for storage (entire dataset) or download version based
      * @param version optional param for dataset version
      * @return total size 
@@ -893,7 +898,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
      * default mode, the method doesn't need to access the storage system, as the 
      * sizes of the main files are recorded in the database)
      */
-    public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
+    public long findStorageSize(Dataset dataset, boolean countCachedExtras, boolean useOrigFileSize, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
         long total = 0L; 
 
         if (dataset.isHarvested()) {
@@ -913,27 +918,39 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatas
 
 
         //CACHED EXTRAS FOR DOWNLOAD?
-
-
-        for (DataFile datafile : filesToTally) {
-                total += datafile.getFilesize();
 
-                if (!countCachedExtras) {
-                    if (datafile.isTabularData()) {
-                        // count the size of the stored original, in addition to the main tab-delimited file:
-                        Long originalFileSize = datafile.getDataTable().getOriginalFileSize();
-                        if (originalFileSize != null) {
-                            total += originalFileSize;
-                        }
+
+        for (DataFile datafile : filesToTally) {
+            if (datafile.isTabularData()) {
+                if (useOrigFileSize) {
+                    // count the size of the stored original, rather than the main tab-delimited file
+                    Long originalFileSize = datafile.getDataTable().getOriginalFileSize();
+                    if (originalFileSize != null) {
+                        total += originalFileSize;
                     }
                 } else {
-                    StorageIO<DataFile> storageIO = datafile.getStorageIO();
-                    for (String cachedFileTag : storageIO.listAuxObjects()) {
-                        total += storageIO.getAuxObjectSize(cachedFileTag);
+                    total += datafile.getFilesize();
+                }
+            } else {
+                total += datafile.getFilesize();
+            }
+
+            if (!countCachedExtras) {
+                if (!useOrigFileSize && datafile.isTabularData()) {
+                    // count the size of the stored original, in addition to the main tab-delimited file:
+                    Long originalFileSize = datafile.getDataTable().getOriginalFileSize();
+                    if (originalFileSize != null) {
+                        total += originalFileSize;
                     }
                 }
+            } else {
+                StorageIO<DataFile> storageIO = datafile.getStorageIO();
+                for (String cachedFileTag : storageIO.listAuxObjects()) {
+                    total += storageIO.getAuxObjectSize(cachedFileTag);
+                }
             }
-
+        }
+
         // and finally,
         if (countCachedExtras) {
             // count the sizes of the files cached for the dataset itself

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
@@ -32,6 +32,7 @@ public class GetDatasetStorageSizeCommand extends AbstractCommand<Long> {
 
     private final Dataset dataset;
     private final Boolean countCachedFiles;
+    private final boolean useOrigFileSize;
     private final Mode mode;
     private final DatasetVersion version;
 
@@ -44,6 +45,7 @@ public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target) {
         super(aRequest, target);
         dataset = target;
         countCachedFiles = false;
+        this.useOrigFileSize = false;
         mode = Mode.DOWNLOAD;
         version = null;
     }
@@ -52,6 +54,16 @@ public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, b
         super(aRequest, target);
         dataset = target;
         this.countCachedFiles = countCachedFiles;
+        this.useOrigFileSize = false;
+        this.mode = mode;
+        this.version = version;
+    }
+
+    public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles, boolean useOrigFileSize, Mode mode, DatasetVersion version) {
+        super(aRequest, target);
+        dataset = target;
+        this.countCachedFiles = countCachedFiles;
+        this.useOrigFileSize = useOrigFileSize;
         this.mode = mode;
         this.version = version;
     }
@@ -66,7 +78,7 @@ public Long execute(CommandContext ctxt) throws CommandException {
         }
 
         try {
-            return ctxt.datasets().findStorageSize(dataset, countCachedFiles, mode, version);
+            return ctxt.datasets().findStorageSize(dataset, countCachedFiles, useOrigFileSize, mode, version);
         } catch (IOException ex) {
             throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.datasize.ioerror"), this);
         }

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
@@ -156,7 +156,7 @@
                                                             <p:commandLink update="@form" actionListener="#{DatasetPage.validateAllFilesForDownloadOriginal()}" styleClass="btn-download">
                                                                 #{bundle.downloadOriginal}
                                                                 <h:outputFormat value="#{bundle['dataset.accessBtn.download.size']}">
-                                                                    <f:param value="#{DatasetPage.sizeOfDataset}" />
+                                                                    <f:param value="#{DatasetPage.sizeOfDatasetOrig}" />
                                                                 </h:outputFormat>
                                                             </p:commandLink>
                                                         </li>