[MINDEXER-176] Clean up silos (#273)

The MT updater did not clean up silos after done. --- https://issues.apache.org/jira/browse/MINDEXER-176
apache · Nov 28, 2022 · c2a438a · c2a438a
1 parent ee4b9bb
commit c2a438a
Showing 1 changed file with 28 additions and 10 deletions.
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
@@ -23,6 +23,7 @@
 import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.EOFException;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UTFDataFormatException;
@@ -46,6 +47,7 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.maven.index.ArtifactInfo;
 import org.apache.maven.index.context.IndexUtils;
@@ -171,11 +173,15 @@ private IndexDataReadResult readIndexMT( IndexWriter w, IndexingContext context
 
         ExecutorService executorService = Executors.newFixedThreadPool( threads );
         ArrayList<Exception> errors = new ArrayList<>();
-        ArrayList<IndexWriter> silos = new ArrayList<>( threads );
+        ArrayList<FSDirectory> siloDirectories = new ArrayList<>( threads );
+        ArrayList<IndexWriter> siloWriters = new ArrayList<>( threads );
+        LOGGER.debug( "Creating {} silo writer threads...", threads );
         for ( int i = 0; i < threads; i++ )
         {
             final int silo = i;
-            silos.add( tempWriter( "silo" + i ) );
+            FSDirectory siloDirectory = tempDirectory( "silo" + i );
+            siloDirectories.add( siloDirectory );
+            siloWriters.add( tempWriter( siloDirectory ) );
             executorService.execute( () ->
             {
                 LOGGER.debug( "Starting thread {}", Thread.currentThread().getName() );
@@ -190,7 +196,7 @@ private IndexDataReadResult readIndexMT( IndexWriter w, IndexingContext context
                             {
                                 break;
                             }
-                            addToIndex( doc, context, silos.get( silo ), rootGroups, allGroups );
+                            addToIndex( doc, context, siloWriters.get( silo ), rootGroups, allGroups );
                         }
                         catch ( InterruptedException | IOException e )
                         {
@@ -206,6 +212,7 @@ private IndexDataReadResult readIndexMT( IndexWriter w, IndexingContext context
             } );
         }
 
+        LOGGER.debug( "Loading up documents into silos" );
         try
         {
             Document doc;
@@ -244,14 +251,25 @@ private IndexDataReadResult readIndexMT( IndexWriter w, IndexingContext context
             IndexUtils.updateTimestamp( w.getDirectory(), date );
         }
 
-        LOGGER.debug( "Merging silos..." );
-        for ( IndexWriter silo : silos )
+        LOGGER.debug( "Closing silo writers..." );
+        for ( IndexWriter siloWriter : siloWriters )
         {
-            IndexUtils.close( silo );
-            w.addIndexes( silo.getDirectory() );
+            siloWriter.commit();
+            siloWriter.close();
         }
 
-        LOGGER.debug( "Merged silos..." );
+        LOGGER.debug( "Merging silo directories..." );
+        w.addIndexes( siloDirectories.toArray( new Directory[0] ) );
+
+        LOGGER.debug( "Cleanup of silo directories..." );
+        for ( FSDirectory siloDirectory : siloDirectories )
+        {
+            File dir = siloDirectory.getDirectory().toFile();
+            siloDirectory.close();
+            IndexUtils.delete( dir );
+        }
+
+        LOGGER.debug( "Finalizing..." );
         w.commit();
 
         IndexDataReadResult result = new IndexDataReadResult();
@@ -269,11 +287,11 @@ private FSDirectory tempDirectory( final String name ) throws IOException
         return FSDirectory.open( Files.createTempDirectory( name + ".dir" ) );
     }
 
-    private IndexWriter tempWriter( final String name ) throws IOException
+    private IndexWriter tempWriter( final FSDirectory directory ) throws IOException
     {
         IndexWriterConfig config = new IndexWriterConfig( new NexusAnalyzer() );
         config.setUseCompoundFile( false );
-        return new NexusIndexWriter( tempDirectory( name ), config );
+        return new NexusIndexWriter( directory, config );
     }
 
     private void addToIndex( final Document doc, final IndexingContext context, final IndexWriter indexWriter,