Cuppa: Improved progress logging

hartwigmedical · Dec 12, 2024 · bb089ad · bb089ad
1 parent 4db7126
commit bb089ad
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 36 deletions.
diff --git a/cuppa/src/main/java/com/hartwig/hmftools/cup/prep/CuppaDataPrep.java b/cuppa/src/main/java/com/hartwig/hmftools/cup/prep/CuppaDataPrep.java
@@ -97,6 +97,8 @@ public void extractSingleSample(boolean keepDataItems)
 
         for(CategoryType categoryType : mConfig.Categories)
         {
+            CUP_LOGGER.info("Extracting category({})", categoryType);
+
             CategoryPrep categoryPrep = createCategoryPrep(categoryType);
             SamplePrepTask sampleTask = new SamplePrepTask(0, mConfig, categoryPrep, null);
             sampleTask.run();
@@ -164,6 +166,8 @@ public void extractMultiSample(boolean keepDataItems)
 
     public void run(boolean keepDataItems)
     {
+        CUP_LOGGER.info("Starting Cuppa feature extraction");
+
         if(mConfig.SampleIds.isEmpty())
         {
             CUP_LOGGER.error("No sample ID(s) loaded");

diff --git a/cuppa/src/main/java/com/hartwig/hmftools/cup/prep/PrepConfig.java b/cuppa/src/main/java/com/hartwig/hmftools/cup/prep/PrepConfig.java
@@ -65,8 +65,6 @@ public class PrepConfig
     public final boolean WriteByCategory;
     public final int Threads;
 
-    public final int ProgressInterval;
-
     public static final String CATEGORIES = "categories";
     public static final String CATEGORIES_DESC = "Categories to build ref data for";
     public static final String ALL_CATEGORIES = "ALL";
@@ -81,9 +79,6 @@ public class PrepConfig
 
     public static final String THREADS_DESC = "Number of threads to use in multi sample mode";
 
-    public static final String  PROGRESS_INTERVAL = "progress_interval";
-    public static final String  PROGRESS_INTERVAL_DESC = "Print progress per this number of samples";
-
     public static final String SUBSET_DELIM = ";";
 
     public PrepConfig(final ConfigBuilder configBuilder)
@@ -107,8 +102,6 @@ public PrepConfig(final ConfigBuilder configBuilder)
         WriteByCategory = configBuilder.hasFlag(WRITE_FILE_BY_CATEGORY);
 
         Threads = TaskExecutor.parseThreads(configBuilder);
-
-        ProgressInterval = configBuilder.getInteger(PROGRESS_INTERVAL);
     }
 
     public static void registerConfig(final ConfigBuilder configBuilder)
@@ -133,8 +126,6 @@ public static void registerConfig(final ConfigBuilder configBuilder)
         configBuilder.addFlag(WRITE_FILE_BY_CATEGORY, WRITE_FILE_BY_CATEGORY_DESC);
         configBuilder.addConfigItem(THREADS, false, THREADS_DESC, "1");
 
-        configBuilder.addInteger(PROGRESS_INTERVAL, PROGRESS_INTERVAL_DESC, 100);
-
         ConfigUtils.addLoggingOptions(configBuilder);
     }
 
@@ -209,8 +200,7 @@ public PrepConfig(
             final String virusDir,
             final String isofoxDir,
             final String somaticVariantsDir,
-            final String altSpliceJunctionSites,
-            final int progressInterval
+            final String altSpliceJunctionSites
     )
     {
         SampleIds = sampleIds;
@@ -227,6 +217,5 @@ public PrepConfig(
         IsofoxDir = isofoxDir;
         SomaticVariantsDir = somaticVariantsDir;
         AltSpliceJunctionSites = altSpliceJunctionSites;
-        ProgressInterval = progressInterval;
     }
 }
diff --git a/cuppa/src/main/java/com/hartwig/hmftools/cup/prep/SamplePrepTask.java b/cuppa/src/main/java/com/hartwig/hmftools/cup/prep/SamplePrepTask.java
@@ -19,6 +19,9 @@ public class SamplePrepTask implements Callable
     @Nullable private List<DataItem> mDataItems;
     @Nullable private ConcurrentHashMap<DataItem.Index, String[]> FeatureBySampleMatrix;
 
+    private static final int FEW_SAMPLES_THRESHOLD = 10;
+    private static final int PROGRESS_INTERVAL = 100;
+
     public SamplePrepTask(
             final int sampleIndex,
             final PrepConfig prepConfig,
@@ -39,19 +42,6 @@ public SamplePrepTask(
         FeatureBySampleMatrix = featureBySampleMatrix;
     }
 
-    public void processSample()
-    {
-        int sampleNum = mSampleIndex + 1;
-        int totalSamples = mConfig.SampleIds.size();
-
-        if(mConfig.isMultiSample() & sampleNum % mConfig.ProgressInterval == 0)
-        {
-            CUP_LOGGER.info("{}/{}: sample({})", sampleNum, totalSamples, mSampleName);
-        }
-
-        mDataItems = mCategoryPrep.extractSampleData(mSampleName);
-    }
-
     public synchronized void addDataItemsToMatrix()
     {
         int nSamples = mConfig.SampleIds.size();
@@ -71,7 +61,14 @@ public synchronized void addDataItemsToMatrix()
 
     public void run()
     {
-        processSample();
+        if(mConfig.isMultiSample() & (mSampleIndex < FEW_SAMPLES_THRESHOLD || mSampleIndex % PROGRESS_INTERVAL == 0))
+        {
+            int sampleNum = mSampleIndex + 1;
+            int totalSamples = mConfig.SampleIds.size();
+            CUP_LOGGER.debug("{}/{}: sample({})", sampleNum, totalSamples, mSampleName);
+        }
+
+        mDataItems = mCategoryPrep.extractSampleData(mSampleName);
 
         if(mConfig.isMultiSample())
         {

diff --git a/cuppa/src/test/java/com/hartwig/hmftools/cup/CuppaDataPrepTest.java b/cuppa/src/test/java/com/hartwig/hmftools/cup/CuppaDataPrepTest.java
@@ -1,5 +1,7 @@
 package com.hartwig.hmftools.cup;
 
+import static com.hartwig.hmftools.cup.common.CupConstants.CUP_LOGGER;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
@@ -22,12 +24,19 @@
 
 import org.apache.commons.io.FileUtils;
 
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
 import org.junit.Test;
 
 public class CuppaDataPrepTest
 {
     File TMP_DIR = new File(System.getProperty("java.io.tmpdir") + "/CuppaDataPrepTest/");
 
+    public CuppaDataPrepTest()
+    {
+        Configurator.setLevel(CUP_LOGGER.getName(), Level.DEBUG);
+    }
+
     @Test
     public void canRunSingleSamplePrep() throws IOException
     {

diff --git a/cuppa/src/test/java/com/hartwig/hmftools/cup/TestPrepConfigBuilder.java b/cuppa/src/test/java/com/hartwig/hmftools/cup/TestPrepConfigBuilder.java
@@ -41,8 +41,6 @@ public class TestPrepConfigBuilder
     private String SomaticVariantsDir = TEST_SOMATIC_VARIANTS_DIR;
 
     private String AltSpliceJunctionSites = TEST_ALT_SPLICE_JUNCTION_SITES;
-    private int ProgressInterval = 100;
-
 
     public TestPrepConfigBuilder sampleIds(List<String> sampleIds)
     {
@@ -139,12 +137,6 @@ public TestPrepConfigBuilder altSpliceJunctionSites(String altSpliceJunctionSite
         return this;
     }
 
-    public TestPrepConfigBuilder progressInterval(int progressInterval)
-    {
-        ProgressInterval = progressInterval;
-        return this;
-    }
-
     public PrepConfig build()
     {
         return new PrepConfig(
@@ -161,8 +153,7 @@ public PrepConfig build()
                 VirusDir,
                 IsofoxDir,
                 SomaticVariantsDir,
-                AltSpliceJunctionSites,
-                ProgressInterval
+                AltSpliceJunctionSites
         );
     };