From 6c41b1ac7bb4c5ccdae038882659b603416e2499 Mon Sep 17 00:00:00 2001 From: vincent Date: Tue, 16 Aug 2022 14:33:28 +0200 Subject: [PATCH 1/2] Use Lucene BitSet --- .../multicriteria/McStopArrivals.java | 13 +++--- .../standard/StdRangeRaptorWorkerState.java | 4 +- .../standard/besttimes/BestTimes.java | 31 +++++++------- .../raptor/util/LuceneBitSetIterator.java | 32 +++++++++++++++ .../util/lang/ToStringBuilder.java | 9 +++++ .../raptor/RaptorArchitectureTest.java | 8 ++-- .../raptor/util/LuceneBitSetIteratorTest.java | 40 +++++++++++++++++++ .../util/UtilArchitectureTest.java | 4 +- .../util/lang/ToStringBuilderTest.java | 5 ++- 9 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIterator.java create mode 100644 src/test/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIteratorTest.java diff --git a/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/multicriteria/McStopArrivals.java b/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/multicriteria/McStopArrivals.java index 2e00960353a..3c543280b70 100644 --- a/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/multicriteria/McStopArrivals.java +++ b/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/multicriteria/McStopArrivals.java @@ -1,7 +1,8 @@ package org.opentripplanner.transit.raptor.rangeraptor.multicriteria; -import java.util.BitSet; import java.util.List; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.FixedBitSet; import org.opentripplanner.transit.raptor.api.response.StopArrivals; import org.opentripplanner.transit.raptor.api.transit.IntIterator; import org.opentripplanner.transit.raptor.api.transit.RaptorTripSchedule; @@ -10,7 +11,7 @@ import org.opentripplanner.transit.raptor.rangeraptor.multicriteria.arrivals.AbstractStopArrival; import org.opentripplanner.transit.raptor.rangeraptor.path.DestinationArrivalPaths; import org.opentripplanner.transit.raptor.rangeraptor.transit.EgressPaths; -import org.opentripplanner.transit.raptor.util.BitSetIterator; +import org.opentripplanner.transit.raptor.util.LuceneBitSetIterator; /** * This class serve as a wrapper for all stop arrival pareto set, one set for each stop. It also @@ -38,7 +39,7 @@ public McStopArrivals( ) { //noinspection unchecked this.arrivals = (StopArrivalParetoSet[]) new StopArrivalParetoSet[nStops]; - this.touchedStops = new BitSet(nStops); + this.touchedStops = new FixedBitSet(nStops); this.debugHandlerFactory = debugHandlerFactory; this.debugStats = new DebugStopArrivalsStatistics(debugHandlerFactory.debugLogger()); @@ -85,11 +86,11 @@ public int smallestNumberOfTransfers(int stopIndex) { } boolean updateExist() { - return !touchedStops.isEmpty(); + return touchedStops.cardinality() > 0; } IntIterator stopsTouchedIterator() { - return new BitSetIterator(touchedStops); + return new LuceneBitSetIterator(touchedStops); } void addStopArrival(AbstractStopArrival arrival) { @@ -117,7 +118,7 @@ void clearTouchedStopsAndSetStopMarkers() { while (it.hasNext()) { arrivals[it.next()].markAtEndOfSet(); } - touchedStops.clear(); + touchedStops.clear(0, touchedStops.length()); } /* private methods */ diff --git a/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/StdRangeRaptorWorkerState.java b/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/StdRangeRaptorWorkerState.java index 6113911abee..47f84c5fff4 100644 --- a/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/StdRangeRaptorWorkerState.java +++ b/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/StdRangeRaptorWorkerState.java @@ -13,7 +13,7 @@ import org.opentripplanner.transit.raptor.rangeraptor.standard.internalapi.ArrivedAtDestinationCheck; import org.opentripplanner.transit.raptor.rangeraptor.standard.internalapi.StopArrivalsState; import org.opentripplanner.transit.raptor.rangeraptor.transit.TransitCalculator; -import org.opentripplanner.transit.raptor.util.BitSetIterator; +import org.opentripplanner.transit.raptor.util.LuceneBitSetIterator; /** * Tracks the state of a standard Range Raptor search, specifically the best arrival times at each @@ -81,7 +81,7 @@ public IntIterator stopsTouchedPreviousRound() { } @Override - public BitSetIterator stopsTouchedByTransitCurrentRound() { + public LuceneBitSetIterator stopsTouchedByTransitCurrentRound() { return bestTimes.reachedByTransitCurrentRound(); } diff --git a/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/besttimes/BestTimes.java b/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/besttimes/BestTimes.java index d5d4691d195..c2907a83686 100644 --- a/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/besttimes/BestTimes.java +++ b/src/main/java/org/opentripplanner/transit/raptor/rangeraptor/standard/besttimes/BestTimes.java @@ -2,10 +2,11 @@ import static org.opentripplanner.transit.raptor.util.IntUtils.intArray; -import java.util.BitSet; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.FixedBitSet; import org.opentripplanner.transit.raptor.rangeraptor.internalapi.WorkerLifeCycle; import org.opentripplanner.transit.raptor.rangeraptor.transit.TransitCalculator; -import org.opentripplanner.transit.raptor.util.BitSetIterator; +import org.opentripplanner.transit.raptor.util.LuceneBitSetIterator; import org.opentripplanner.util.lang.ToStringBuilder; /** @@ -43,11 +44,11 @@ public final class BestTimes { public BestTimes(int nStops, TransitCalculator calculator, WorkerLifeCycle lifeCycle) { this.calculator = calculator; this.times = intArray(nStops, calculator.unreachedTime()); - this.reachedCurrentRound = new BitSet(nStops); - this.reachedLastRound = new BitSet(nStops); + this.reachedCurrentRound = new FixedBitSet(nStops); + this.reachedLastRound = new FixedBitSet(nStops); this.transitArrivalTimes = intArray(nStops, calculator.unreachedTime()); - this.reachedByTransitCurrentRound = new BitSet(nStops); + this.reachedByTransitCurrentRound = new FixedBitSet(nStops); // Attach to Worker life cycle lifeCycle.onSetupIteration(ignore -> setupIteration()); @@ -66,21 +67,21 @@ public int transitArrivalTime(int stop) { * @return true if at least one stop arrival was reached last round (best overall). */ public boolean isCurrentRoundUpdated() { - return !reachedCurrentRound.isEmpty(); + return reachedCurrentRound.cardinality() > 0; } /** * @return an iterator for all stops reached (overall best) in the last round. */ - public BitSetIterator stopsReachedLastRound() { - return new BitSetIterator(reachedLastRound); + public LuceneBitSetIterator stopsReachedLastRound() { + return new LuceneBitSetIterator(reachedLastRound); } /** * @return an iterator of all stops reached on-board in the current round. */ - public BitSetIterator reachedByTransitCurrentRound() { - return new BitSetIterator(reachedByTransitCurrentRound); + public LuceneBitSetIterator reachedByTransitCurrentRound() { + return new LuceneBitSetIterator(reachedByTransitCurrentRound); } /** @@ -137,7 +138,7 @@ public String toString() { .of(BestTimes.class) .addIntArraySize("times", times, unreachedTime) .addIntArraySize("transitArrivalTimes", transitArrivalTimes, unreachedTime) - .addNum("reachedCurrentRound", reachedCurrentRound.size()) + .addNum("reachedCurrentRound", reachedCurrentRound.length()) .addBitSetSize("reachedByTransitCurrentRound", reachedByTransitCurrentRound) .addBitSetSize("reachedLastRound", reachedLastRound) .toString(); @@ -156,8 +157,8 @@ boolean isStopReachedOnBoardInCurrentRound(int stop) { */ private void setupIteration() { // clear all touched stops to avoid constant reëxploration - reachedCurrentRound.clear(); - reachedByTransitCurrentRound.clear(); + reachedCurrentRound.clear(0, reachedCurrentRound.length()); + reachedByTransitCurrentRound.clear(0, reachedByTransitCurrentRound.length()); } /** @@ -165,8 +166,8 @@ private void setupIteration() { */ private void prepareForNextRound() { swapReachedCurrentAndLastRound(); - reachedCurrentRound.clear(); - reachedByTransitCurrentRound.clear(); + reachedCurrentRound.clear(0, reachedCurrentRound.length()); + reachedByTransitCurrentRound.clear(0, reachedCurrentRound.length()); } /* private methods */ diff --git a/src/main/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIterator.java b/src/main/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIterator.java new file mode 100644 index 00000000000..08c15e8afb5 --- /dev/null +++ b/src/main/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIterator.java @@ -0,0 +1,32 @@ +package org.opentripplanner.transit.raptor.util; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.opentripplanner.transit.raptor.api.transit.IntIterator; + +/** + * TODO TGR + */ +public final class LuceneBitSetIterator implements IntIterator { + + private final BitSet set; + private int nextIndex; + + public LuceneBitSetIterator(BitSet set) { + this.set = set; + this.nextIndex = set.nextSetBit(nextIndex); + } + + @Override + public int next() { + int index = nextIndex; + nextIndex = + index == set.length() - 1 ? DocIdSetIterator.NO_MORE_DOCS : set.nextSetBit(index + 1); + return index; + } + + @Override + public boolean hasNext() { + return nextIndex != DocIdSetIterator.NO_MORE_DOCS; + } +} diff --git a/src/main/java/org/opentripplanner/util/lang/ToStringBuilder.java b/src/main/java/org/opentripplanner/util/lang/ToStringBuilder.java index 00d99d9a268..e0ee578ad8c 100644 --- a/src/main/java/org/opentripplanner/util/lang/ToStringBuilder.java +++ b/src/main/java/org/opentripplanner/util/lang/ToStringBuilder.java @@ -188,6 +188,15 @@ public ToStringBuilder addBitSetSize(String name, BitSet bitSet) { return addIt(name, bitSet.cardinality() + "/" + bitSet.length()); } + /** Add the Lucene BitSet: name : {cardinality}/{logical size}/{size} */ + public ToStringBuilder addBitSetSize(String name, org.apache.lucene.util.BitSet bitSet) { + if (bitSet == null) { + return this; + } + // TODO Lucene BitSet length() has a different meaning + return addIt(name, bitSet.cardinality() + "/" + bitSet.length()); + } + /* Special purpose formatters */ /** Add a Coordinate location, longitude or latitude */ diff --git a/src/test/java/org/opentripplanner/transit/raptor/RaptorArchitectureTest.java b/src/test/java/org/opentripplanner/transit/raptor/RaptorArchitectureTest.java index f4b757c1f60..9d690b56aa5 100644 --- a/src/test/java/org/opentripplanner/transit/raptor/RaptorArchitectureTest.java +++ b/src/test/java/org/opentripplanner/transit/raptor/RaptorArchitectureTest.java @@ -29,6 +29,8 @@ public class RaptorArchitectureTest { private static final Package RR_STD_CONFIGURE = RR_STANDARD.subPackage("configure"); private static final Package RR_CONTEXT = RANGE_RAPTOR.subPackage("context"); + private static final Package LUCENE_BITSET = Package.of("org.apache.lucene.util"); + @Test void enforcePackageDependenciesRaptorAPI() { var api = RAPTOR.subPackage("api"); @@ -42,7 +44,7 @@ void enforcePackageDependenciesRaptorAPI() { @Test void enforcePackageDependenciesUtil() { - RAPTOR_UTIL.dependsOn(UTILS, RAPTOR_API).verify(); + RAPTOR_UTIL.dependsOn(UTILS, RAPTOR_API, LUCENE_BITSET).verify(); RAPTOR_UTIL_PARETO_SET.verify(); } @@ -70,7 +72,7 @@ void enforcePackageDependenciesInRaptorImplementation() { var stdInternalApi = RR_STANDARD.subPackage("internalapi").dependsOn(RAPTOR_API).verify(); var stdBestTimes = RR_STANDARD .subPackage("besttimes") - .dependsOn(rrCommon, stdInternalApi) + .dependsOn(rrCommon, stdInternalApi, LUCENE_BITSET) .verify(); var stdStopArrivals = RR_STANDARD .subPackage("stoparrivals") @@ -118,7 +120,7 @@ void enforcePackageDependenciesInRaptorImplementation() { .subPackage("heuristic") .dependsOn(rrCommon, mcArrivals) .verify(); - RR_MULTI_CRITERIA.dependsOn(rrCommon, mcArrivals, mcHeuristics).verify(); + RR_MULTI_CRITERIA.dependsOn(rrCommon, mcArrivals, mcHeuristics, LUCENE_BITSET).verify(); RR_MC_CONFIGURE .dependsOn(rrCommon, RR_CONTEXT, pathConfigure, mcHeuristics, RR_MULTI_CRITERIA) diff --git a/src/test/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIteratorTest.java b/src/test/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIteratorTest.java new file mode 100644 index 00000000000..8b2bf078404 --- /dev/null +++ b/src/test/java/org/opentripplanner/transit/raptor/util/LuceneBitSetIteratorTest.java @@ -0,0 +1,40 @@ +package org.opentripplanner.transit.raptor.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.FixedBitSet; +import org.junit.jupiter.api.Test; + +public class LuceneBitSetIteratorTest { + + @Test + public void test() { + LuceneBitSetIterator it; + BitSet set = new FixedBitSet(6); + + // Empty set does not have any elements + it = new LuceneBitSetIterator(set); + assertFalse(it.hasNext()); + + set.set(2); + it = new LuceneBitSetIterator(set); + assertTrue(it.hasNext()); + assertEquals(2, it.next()); + assertFalse(it.hasNext()); + + // set: [0, 2, 5], 2 is added above + set.set(0); + set.set(5); + it = new LuceneBitSetIterator(set); + assertTrue(it.hasNext()); + assertEquals(0, it.next()); + assertTrue(it.hasNext()); + assertEquals(2, it.next()); + assertTrue(it.hasNext()); + assertEquals(5, it.next()); + assertFalse(it.hasNext()); + } +} diff --git a/src/test/java/org/opentripplanner/util/UtilArchitectureTest.java b/src/test/java/org/opentripplanner/util/UtilArchitectureTest.java index 97e6e888a18..c71026655b8 100644 --- a/src/test/java/org/opentripplanner/util/UtilArchitectureTest.java +++ b/src/test/java/org/opentripplanner/util/UtilArchitectureTest.java @@ -10,6 +10,8 @@ public class UtilArchitectureTest { private static final Package LANG = UTIL.subPackage("lang"); private static final Package TIME = UTIL.subPackage("time"); + private static final Package LUCENE_BITSET = Package.of("org.apache.lucene.util"); + @Test void enforcePackageDependencies() { // The util packages needs cleanup, it contains model, framework and API classes. The strategy @@ -21,6 +23,6 @@ void enforcePackageDependencies() { // It might sound strange that lang depend on time, but we allow this to avoid creating another // util package where we can put the ToStringBuilder classes(witch depend on time). As long as // we do not get cyclic dependencies between lang and time there is not problem with this. - LANG.dependsOn(TIME).verify(); + LANG.dependsOn(TIME, LUCENE_BITSET).verify(); } } diff --git a/src/test/java/org/opentripplanner/util/lang/ToStringBuilderTest.java b/src/test/java/org/opentripplanner/util/lang/ToStringBuilderTest.java index 4ad532476bc..ff6286f7d19 100644 --- a/src/test/java/org/opentripplanner/util/lang/ToStringBuilderTest.java +++ b/src/test/java/org/opentripplanner/util/lang/ToStringBuilderTest.java @@ -202,7 +202,10 @@ public void addBitSetSize() { subject().addBitSetSize("bitSet", bset).toString() ); - assertEquals("ToStringBuilderTest{}", subject().addBitSetSize("bitSet", null).toString()); + assertEquals( + "ToStringBuilderTest{}", + subject().addBitSetSize("bitSet", (BitSet) null).toString() + ); } @Test From 372adcd6ca8f75af3dd75065fe66d9f51cdc72f4 Mon Sep 17 00:00:00 2001 From: vincent Date: Wed, 17 Aug 2022 15:02:06 +0200 Subject: [PATCH 2/2] Fix SpeedTest --- .../opentripplanner/transit/raptor/speed_test/SpeedTest.java | 5 ++--- .../transit/raptor/speed_test/model/testcase/CsvFileIO.java | 2 +- .../raptor/speed_test/model/timer/SpeedTestTimer.java | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/opentripplanner/transit/raptor/speed_test/SpeedTest.java b/src/test/java/org/opentripplanner/transit/raptor/speed_test/SpeedTest.java index 8cb682e0531..1b041e599ba 100644 --- a/src/test/java/org/opentripplanner/transit/raptor/speed_test/SpeedTest.java +++ b/src/test/java/org/opentripplanner/transit/raptor/speed_test/SpeedTest.java @@ -183,11 +183,10 @@ private void runSingleTest(int sample, int nSamples) { // We assume we are debugging and not measuring performance if we only run 1 test-case // one time; Hence skip JIT compiler warm-up. int samplesPrProfile = opts.numberOfTestsSamplesToRun() / opts.profiles().length; - if (testCases.size() > 1 || samplesPrProfile > 1) { + if (testCases.size() > 1 && samplesPrProfile > 1) { // Warm-up JIT compiler, run the second test-case if it exist to avoid the same // test case from being repeated. If there is just one case, then run it. - int index = testCases.size() == 1 ? 0 : 1; - runSingleTestCase(testCases.get(index), true); + runSingleTestCase(testCases.get(1), true); } ResultPrinter.logSingleTestHeader(routeProfile); diff --git a/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/testcase/CsvFileIO.java b/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/testcase/CsvFileIO.java index fc2d4030145..eae3cc36b0c 100644 --- a/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/testcase/CsvFileIO.java +++ b/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/testcase/CsvFileIO.java @@ -77,7 +77,7 @@ public void writeResultsToFile(List testCases) { if (!tcIds.isEmpty()) { LOG.warn( "No results file written, at least one test-case is not run or returned without any result!" + - " Test-Cases: " + + " Failed Test-Cases: " + tcIds ); return; diff --git a/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/timer/SpeedTestTimer.java b/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/timer/SpeedTestTimer.java index 9750558cb50..0a4d57a71e2 100644 --- a/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/timer/SpeedTestTimer.java +++ b/src/test/java/org/opentripplanner/transit/raptor/speed_test/model/timer/SpeedTestTimer.java @@ -129,6 +129,9 @@ public void finishUp() { public int totalTimerMean(String timerName) { long count = getTotalTimers(timerName).mapToLong(Timer::count).sum(); + if (count == 0) { + return 0; + } return (int) (testTotalTimeMs(timerName) / count); }