[Jepsen] Running tests in batch fails with OOM

[frozenspider]

When running

lein run test-all -o debian --url ... --concurrency 20 --only-workloads-expected-to-pass

at some point it starts failing with OutOfMemoryError:

WARN [2019-06-24 18:54:50,560] Service Thread - knossos.search Out of memory; aborting search
WARN [2019-06-24 18:55:00,664] clojure-agent-send-pool-0 - jepsen.checker Error while checking history:
java.util.concurrent.ExecutionException: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at java.util.concurrent.FutureTask.report(FutureTask.java:122) [na:1.8.0_171]
	at java.util.concurrent.FutureTask.get(FutureTask.java:192) [na:1.8.0_171]
	at clojure.core$deref_future.invokeStatic(core.clj:2300) ~[clojure-1.10.0.jar:na]
	at clojure.core$future_call$reify__8439.deref(core.clj:6974) ~[clojure-1.10.0.jar:na]
	at clojure.core$deref.invokeStatic(core.clj:2320) ~[clojure-1.10.0.jar:na]
	at clojure.core$deref.invoke(core.clj:2306) ~[clojure-1.10.0.jar:na]
	at clojure.core$map$fn__5851.invoke(core.clj:2755) ~[clojure-1.10.0.jar:na]
	at clojure.lang.LazySeq.sval(LazySeq.java:42) ~[clojure-1.10.0.jar:na]
	at clojure.lang.LazySeq.seq(LazySeq.java:51) ~[clojure-1.10.0.jar:na]
	at clojure.lang.Cons.next(Cons.java:39) ~[clojure-1.10.0.jar:na]
	at clojure.lang.RT.next(RT.java:709) ~[clojure-1.10.0.jar:na]
	at clojure.core$next__5371.invokeStatic(core.clj:64) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8144.invokeStatic(protocols.clj:169) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8144.invoke(protocols.clj:124) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8099$G__8094__8108.invoke(protocols.clj:19) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$seq_reduce.invokeStatic(protocols.clj:31) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8131.invokeStatic(protocols.clj:75) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8131.invoke(protocols.clj:75) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8073$G__8068__8086.invoke(protocols.clj:13) ~[clojure-1.10.0.jar:na]
	at clojure.core$reduce.invokeStatic(core.clj:6828) ~[clojure-1.10.0.jar:na]
	at clojure.core$into.invokeStatic(core.clj:6895) ~[clojure-1.10.0.jar:na]
	at clojure.core$into.invoke(core.clj:6887) ~[clojure-1.10.0.jar:na]
	at jepsen.checker$compose$reify__4748.check(checker.clj:101) ~[jepsen-0.1.14.jar:na]
	at jepsen.checker$check_safe.invokeStatic(checker.clj:84) [jepsen-0.1.14.jar:na]
	at jepsen.checker$check_safe.invoke(checker.clj:77) [jepsen-0.1.14.jar:na]
	at jepsen.independent$checker$reify__3661$fn__3662.invoke(independent.clj:272) [na:na]
	at dom_top.core$bounded_pmap$launcher__209$fn__210.invoke(core.clj:129) [jepsen-0.1.14.jar:na]
	at clojure.core$binding_conveyor_fn$fn__5739.invoke(core.clj:2030) [clojure-1.10.0.jar:na]
	at clojure.lang.AFn.call(AFn.java:18) [clojure-1.10.0.jar:na]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_171]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_171]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_171]
	at java.lang.Thread.run(Thread.java:748) [na:1.8.0_171]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at clojure.lang.PersistentHashMap$ArrayNode.assoc(PersistentHashMap.java:420) ~[clojure-1.10.0.jar:na]
	at clojure.lang.PersistentHashMap.assoc(PersistentHashMap.java:143) ~[clojure-1.10.0.jar:na]
	at clojure.lang.PersistentHashSet.cons(PersistentHashSet.java:99) ~[clojure-1.10.0.jar:na]
	at clojure.lang.PersistentHashSet.cons(PersistentHashSet.java:17) ~[clojure-1.10.0.jar:na]
	at clojure.lang.RT.conj(RT.java:673) ~[clojure-1.10.0.jar:na]
	at clojure.core$conj__5375.invokeStatic(core.clj:85) ~[clojure-1.10.0.jar:na]
	at clojure.core$conj__5375.invoke(core.clj:82) ~[clojure-1.10.0.jar:na]
	at clojure.core.reducers$map$fn__554$fn__555.invoke(reducers.clj:167) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$map$fn__554$fn__555.invoke(reducers.clj:167) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$mapcat$fn__561$f1__562.invoke(reducers.clj:180) ~[knossos-0.3.4.jar:na]
	at clojure.core.protocols$fn__8144.invokeStatic(protocols.clj:168) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8144.invoke(protocols.clj:124) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8099$G__8094__8108.invoke(protocols.clj:19) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$seq_reduce.invokeStatic(protocols.clj:31) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8131.invokeStatic(protocols.clj:75) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8131.invoke(protocols.clj:75) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8073$G__8068__8086.invoke(protocols.clj:13) ~[clojure-1.10.0.jar:na]
	at clojure.core.reducers$reduce.invokeStatic(reducers.clj:79) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$reduce.invoke(reducers.clj:71) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$mapcat$fn__561$fn__564.invoke(reducers.clj:183) ~[knossos-0.3.4.jar:na]
	at clojure.core.protocols$fn__8144.invokeStatic(protocols.clj:168) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8144.invoke(protocols.clj:124) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8099$G__8094__8108.invoke(protocols.clj:19) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$seq_reduce.invokeStatic(protocols.clj:31) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8131.invokeStatic(protocols.clj:75) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8131.invoke(protocols.clj:75) ~[clojure-1.10.0.jar:na]
	at clojure.core.protocols$fn__8073$G__8068__8086.invoke(protocols.clj:13) ~[clojure-1.10.0.jar:na]
	at clojure.core.reducers$reduce.invokeStatic(reducers.clj:79) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$reduce.invoke(reducers.clj:71) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$fn__633.invokeStatic(reducers.clj:357) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$fn__633.invoke(reducers.clj:347) ~[knossos-0.3.4.jar:na]
	at clojure.core.reducers$fn__516$G__511__527.invoke(reducers.clj:81) ~[knossos-0.3.4.jar:na]

[aphyr]

The linearizable checker in knossos is exponential in the number of concurrent processes, including crashed processes. You may want to see if you can categorize fewer responses as :info, reduce concurrency, or reduce the number of updates per key to improve performance.