Merge pull request #32 from mocobeta/pivot-luke

Merge recent changes in master branch

.gitignore

@@ -0,0 +1,2 @@
+target
+.DS_Store

luke.sh

@@ -1,4 +1,10 @@
-java -XX:MaxPermSize=512m -jar target/luke-with-deps.jar
+if [[ -d `echo $LUKE_PATH` ]]; then
+  java -XX:MaxPermSize=512m -jar $LUKE_PATH/target/luke-with-deps.jar
+else
+  echo "Unable to find the LUKE_PATH environnement variable..."
+  echo "Assuming you're running from the root folder of luke..."
+  java -XX:MaxPermSize=512m -jar target/luke-with-deps.jar
+fi
 #
 # In order to start luke with your custom analyzer class extending org.apache.lucene.analysis.Analyzer run:
 # java -XX:MaxPermSize=512m -cp target/luke-with-deps.jar:/path/to/custom_analyzer.jar org.getopt.luke.Luke

luke_solr.sh

@@ -1,4 +1,10 @@
-java -XX:MaxPermSize=512m -Dluke.ext.decoder.loader=org.apache.lucene.luke.ext.SolrDecoderLoader -jar target/luke-with-deps.jar
+if [[ -d `echo $LUKE_PATH` ]]; then
+  java -XX:MaxPermSize=512m -Dluke.ext.decoder.loader=org.apache.lucene.luke.ext.SolrDecoderLoader -jar target/luke-with-deps.jar
+else
+  echo "Unable to find the LUKE_PATH environnement variable..."
+  echo "Assuming you're running from the root folder of luke..."
+  java -XX:MaxPermSize=512m -Dluke.ext.decoder.loader=org.apache.lucene.luke.ext.SolrDecoderLoader -jar target/luke-with-deps.jar
+fi
 #
 # In order to start luke with your custom analyzer class extending org.apache.lucene.analysis.Analyzer run:
 # java -XX:MaxPermSize=512m -cp target/luke-with-deps.jar:/path/to/custom_analyzer.jar org.getopt.luke.Luke

pom.xml

@@ -10,7 +10,7 @@
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <lucene.version>4.10.4</lucene.version>
+        <lucene.version>5.2.1</lucene.version>
         <ehcache.version>2.9.0</ehcache.version>
         <hadoop.version>1.2.1</hadoop.version>
         <log4j.version>1.2.17</log4j.version>

src/main/java/org/apache/lucene/index/IndexGate.java

@@ -72,9 +72,6 @@ public class IndexGate {
   public static final int FORMAT_PRE_4 = -12;
 
   static {
-    knownExtensions.put(IndexFileNames.COMPOUND_FILE_EXTENSION, "compound file with various index data");
-    knownExtensions.put(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION, "compound file entries list");
-    knownExtensions.put(IndexFileNames.GEN_EXTENSION, "generation number - global file");
     knownExtensions.put(IndexFileNames.SEGMENTS, "per-commit list of segments and user data");
   }
 
@@ -186,7 +183,7 @@ public static String getLastModified(final Directory dir) throws Exception {
       @Override
       protected Object doBody(String segmentsFile) throws IOException {
         if (dir instanceof FSDirectory) {
-          File file = new File(((FSDirectory)dir).getDirectory(), segmentsFile);
+          File file = new File(((FSDirectory)dir).getDirectory().toFile(), segmentsFile);
           long lastModified = file.lastModified();
           return lastModified;
         }
@@ -211,11 +208,11 @@ protected Object doBody(String segmentsFile) throws IOException {
           if (indexFormat == CodecUtil.CODEC_MAGIC) {
             res.genericName = "Lucene 4.x";
             res.capabilities = "flexible, codec-specific";
-            int actualVersion = SegmentInfos.VERSION_40;
+            int actualVersion = SegmentInfos.VERSION_CURRENT;
             try {
               actualVersion = CodecUtil.checkHeaderNoMagic(in, "segments", SegmentInfos.VERSION_40, Integer.MAX_VALUE);
               res.version = Integer.toString(actualVersion);
-              if (actualVersion > SegmentInfos.VERSION_49) {
+              if (actualVersion > SegmentInfos.VERSION_CURRENT) {
                 res.capabilities += " (WARNING: newer version of Lucene than this tool)";
               }
             } catch (Exception e) {
@@ -235,6 +232,12 @@ protected Object doBody(String segmentsFile) throws IOException {
               case SegmentInfos.VERSION_49:
                 res.genericName = "Lucene 4.9+";
                 break;
+              case SegmentInfos.VERSION_50:
+                res.genericName = "Lucene 5.0+";
+                break;
+              case SegmentInfos.VERSION_51:
+                res.genericName = "Lucene 5.1+";
+                break;
               default:
                 res.capabilities = "unknown";
                 res.genericName = "Lucene 3.x or prior";
@@ -273,7 +276,7 @@ protected Object doBody(String segmentsFile) throws CorruptIndexException,
         final int actualFormat;
         try {
           if (format == CodecUtil.CODEC_MAGIC) {
-            actualFormat = CodecUtil.checkHeaderNoMagic(input, "segments", SegmentInfos.VERSION_40, SegmentInfos.VERSION_49);
+            actualFormat = CodecUtil.checkHeaderNoMagic(input, "segments", SegmentInfos.VERSION_40, SegmentInfos.VERSION_CURRENT);
           } else {
             actualFormat = -1;
           }
@@ -294,8 +297,7 @@ protected Object doBody(String segmentsFile) throws CorruptIndexException,
 
 
   public static boolean preferCompoundFormat(Directory dir) throws Exception {
-    SegmentInfos infos = new SegmentInfos();
-    infos.read(dir);
+    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
     int compound = 0, nonCompound = 0;
     for (int i = 0; i < infos.size(); i++) {
       if (infos.info(i).info.getUseCompoundFile()) {
@@ -336,9 +338,11 @@ public static List<String> getIndexFiles(Directory dir) throws Exception {
     for (IndexCommit ic : commits) {
       known.addAll(ic.getFileNames());
     }
+    /* FIXME: Directory#fileExists was removed since lucene 5.1.0
     if (dir.fileExists(IndexFileNames.SEGMENTS_GEN)) {
       known.add(IndexFileNames.SEGMENTS_GEN);
     }
+    */
     List<String> names = new ArrayList<String>(known);
     Collections.sort(names);
     return names;
@@ -347,8 +351,7 @@ public static List<String> getIndexFiles(Directory dir) throws Exception {
   public static Codec getCodecOfFirstSegment(Directory dir)
     throws CorruptIndexException, IOException {
 
-    SegmentInfos infos = new SegmentInfos();
-    infos.read(dir);
+    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
 
     SegmentInfo info = (SegmentInfo) infos.info(0).info;
     return info.getCodec();

src/main/java/org/apache/lucene/luke/core/AccessibleHitCollector.java

@@ -3,7 +3,7 @@
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.Scorer;
 
-public abstract class AccessibleHitCollector extends Collector {
+public abstract class AccessibleHitCollector implements Collector {
   protected Scorer scorer;
   protected boolean shouldScore;
   protected int docBase;

src/main/java/org/apache/lucene/luke/core/AccessibleTopHitCollector.java

@@ -1,7 +1,7 @@
 package org.apache.lucene.luke.core;
 
-import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopScoreDocCollector;
 
@@ -12,10 +12,8 @@ public class AccessibleTopHitCollector extends AccessibleHitCollector {
   private TopDocs topDocs = null;
   private int size;
 
-  public AccessibleTopHitCollector(int size, boolean outOfOrder, boolean shouldScore) {
-    tdc = TopScoreDocCollector.create(size, outOfOrder);
-    this.shouldScore = shouldScore;
-    this.outOfOrder = outOfOrder;
+  public AccessibleTopHitCollector(int size) {
+    tdc = TopScoreDocCollector.create(size);
     this.size = size;
   }
 
@@ -41,34 +39,18 @@ public int getTotalHits() {
   }
 
   @Override
-  public boolean acceptsDocsOutOfOrder() {
-    return tdc.acceptsDocsOutOfOrder();
-  }
-
-  @Override
-  public void collect(int doc) throws IOException {
-    tdc.collect(doc);
-  }
-
-  @Override
-  public void setNextReader(AtomicReaderContext context) throws IOException {
-    this.docBase = context.docBase;
-    tdc.setNextReader(context);
+  public void reset() {
+    tdc = TopScoreDocCollector.create(size);
+    topDocs = null;
   }
 
   @Override
-  public void setScorer(Scorer scorer) throws IOException {
-    if (shouldScore) {
-      tdc.setScorer(scorer);
-    } else {
-      tdc.setScorer(NoScoringScorer.INSTANCE);
-    }
+  public LeafCollector getLeafCollector(LeafReaderContext leafReaderContext) throws IOException {
+    return tdc.getLeafCollector(leafReaderContext);
   }
 
   @Override
-  public void reset() {
-    tdc = TopScoreDocCollector.create(size, outOfOrder);
-    topDocs = null;
+  public boolean needsScores() {
+    return tdc.needsScores();
   }
-
 }

src/main/java/org/apache/lucene/luke/core/AllHitsCollector.java

@@ -3,7 +3,8 @@
  */
 package org.apache.lucene.luke.core;
 
-import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Scorer;
 
 import java.io.IOException;
@@ -42,6 +43,37 @@ public float getScore(int i) {
     return (hits.get(i)).score;
   }
 
+  @Override
+  public LeafCollector getLeafCollector(LeafReaderContext leafReaderContext) throws IOException {
+    this.docBase = leafReaderContext.docBase;
+    return new LeafCollector() {
+      private Scorer scorer;
+      @Override
+      public void setScorer(Scorer scorer) throws IOException {
+        this.scorer = scorer;
+      }
+      @Override
+      public void collect(int doc) throws IOException {
+        float score = 1.0f;
+        if (shouldScore) {
+          try {
+            score = scorer.score();
+          } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+          }
+        }
+        hits.add(new AllHit(docBase + doc, score));
+      }
+    };
+  }
+
+
+  @Override
+  public boolean needsScores() {
+    return false;
+  }
+
   private static class AllHit {
     public int docId;
     public float score;
@@ -52,21 +84,6 @@ public AllHit(int docId, float score) {
     }
   }
 
-  @Override
-  public boolean acceptsDocsOutOfOrder() {
-    return outOfOrder;
-  }
-
-  @Override
-  public void setNextReader(AtomicReaderContext context) throws IOException {
-    this.docBase = context.docBase;
-  }
-
-  @Override
-  public void setScorer(Scorer scorer) throws IOException {
-    this.scorer = scorer;
-  }
-
   @Override
   public void reset() {
     hits.clear();

src/main/java/org/apache/lucene/luke/core/CountLimitedHitCollector.java

@@ -1,6 +1,7 @@
 package org.apache.lucene.luke.core;
 
-import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopScoreDocCollector;
@@ -14,12 +15,10 @@ public class CountLimitedHitCollector extends LimitedHitCollector {
   private TopScoreDocCollector tdc;
   private TopDocs topDocs = null;
 
-  public CountLimitedHitCollector(int maxSize, boolean outOfOrder, boolean shouldScore) {
+  public CountLimitedHitCollector(int maxSize) {
     this.maxSize = maxSize;
-    this.outOfOrder = outOfOrder;
-    this.shouldScore = shouldScore;
     count = 0;
-    tdc = TopScoreDocCollector.create(maxSize, outOfOrder);
+    tdc = TopScoreDocCollector.create(maxSize);
   }
 
   @Override
@@ -32,21 +31,6 @@ public int limitType() {
     return TYPE_SIZE;
   }
 
-  /* (non-Javadoc)
-   * @see AllHitsCollector#collect(int, float)
-   */
-  @Override
-  public void collect(int doc) throws IOException {
-    count++;
-    if (count > maxSize) {
-      count--;
-      throw new LimitedException(TYPE_SIZE, maxSize, count, lastDoc);
-    }
-    lastDoc = docBase + doc;
-
-    tdc.collect(doc);
-  }
-
   /* (non-Javadoc)
    * @see AccessibleHitCollector#getDocId(int)
    */
@@ -78,30 +62,40 @@ public int getTotalHits() {
   }
 
   @Override
-  public boolean acceptsDocsOutOfOrder() {
-    return tdc.acceptsDocsOutOfOrder();
-  }
-
-  @Override
-  public void setNextReader(AtomicReaderContext context) throws IOException {
-    this.docBase = context.docBase;
-    tdc.setNextReader(context);
+  public void reset() {
+    count = 0;
+    lastDoc = 0;
+    topDocs = null;
+    tdc = TopScoreDocCollector.create(maxSize);
   }
 
   @Override
-  public void setScorer(Scorer scorer) throws IOException {
-    if (shouldScore) {
-      tdc.setScorer(scorer);
-    } else {
-      tdc.setScorer(NoScoringScorer.INSTANCE);
-    }
+  public LeafCollector getLeafCollector(final LeafReaderContext leafReaderContext) throws IOException {
+    this.docBase = leafReaderContext.docBase;
+    return new LeafCollector() {
+      @Override
+      public void setScorer(Scorer scorer) throws IOException {
+        if (shouldScore) {
+          tdc.getLeafCollector(leafReaderContext).setScorer(scorer);
+        } else {
+          tdc.getLeafCollector(leafReaderContext).setScorer(NoScoringScorer.INSTANCE);
+        }
+      }
+      @Override
+      public void collect(int doc) throws IOException {
+        count++;
+        if (count > maxSize) {
+          count--;
+          throw new LimitedException(TYPE_SIZE, maxSize, count, lastDoc);
+        }
+        lastDoc = docBase + doc;
+        tdc.getLeafCollector(leafReaderContext).collect(doc);
+      }
+    };
   }
 
   @Override
-  public void reset() {
-    count = 0;
-    lastDoc = 0;
-    topDocs = null;
-    tdc = TopScoreDocCollector.create(maxSize, outOfOrder);
+  public boolean needsScores() {
+    return tdc.needsScores();
   }
 }