From eb2cd9bb9610b9ba1ad20c249eb2adf17409e9d9 Mon Sep 17 00:00:00 2001 From: SolrBot Date: Thu, 4 Jan 2024 12:36:18 +0000 Subject: [PATCH 1/2] Update dependency com.tdunning:t-digest to v3.3 --- solr/licenses/t-digest-3.1.jar.sha1 | 1 - solr/licenses/t-digest-3.3.jar.sha1 | 1 + versions.lock | 2 +- versions.props | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 solr/licenses/t-digest-3.1.jar.sha1 create mode 100644 solr/licenses/t-digest-3.3.jar.sha1 diff --git a/solr/licenses/t-digest-3.1.jar.sha1 b/solr/licenses/t-digest-3.1.jar.sha1 deleted file mode 100644 index 1c4c89ce090..00000000000 --- a/solr/licenses/t-digest-3.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -451ed219688aed5821a789428fd5e10426d11312 diff --git a/solr/licenses/t-digest-3.3.jar.sha1 b/solr/licenses/t-digest-3.3.jar.sha1 new file mode 100644 index 00000000000..e9cd01f1c99 --- /dev/null +++ b/solr/licenses/t-digest-3.3.jar.sha1 @@ -0,0 +1 @@ +5e96c4fd7d63b05828cf5ef41da20649195b1b78 diff --git a/versions.lock b/versions.lock index 4b779e3cf05..3bfb75aeae0 100644 --- a/versions.lock +++ b/versions.lock @@ -75,7 +75,7 @@ com.squareup.okio:okio:3.6.0 (1 constraints: 530c38fd) com.squareup.okio:okio-jvm:3.6.0 (1 constraints: 500ad3b9) com.sun.activation:jakarta.activation:1.2.2 (1 constraints: ba0dac35) com.sun.istack:istack-commons-runtime:3.0.12 (1 constraints: eb0d9a43) -com.tdunning:t-digest:3.1 (1 constraints: a804212c) +com.tdunning:t-digest:3.3 (1 constraints: aa04232c) com.zaxxer:SparseBitSet:1.2 (1 constraints: 0d081e75) commons-cli:commons-cli:1.6.0 (1 constraints: 09050236) commons-codec:commons-codec:1.16.0 (11 constraints: a297a2db) diff --git a/versions.props b/versions.props index 63b6e77eb71..c5ef9c255e7 100644 --- a/versions.props +++ b/versions.props @@ -16,7 +16,7 @@ com.google.re2j:re2j=1.7 com.j256.simplemagic:simplemagic=1.17 com.jayway.jsonpath:json-path=2.9.0 com.lmax:disruptor=3.4.4 -com.tdunning:t-digest=3.1 +com.tdunning:t-digest=3.3 commons-cli:commons-cli=1.6.0 commons-codec:commons-codec=1.16.0 commons-collections:commons-collections=3.2.2 From e0d36967626d1e261c172739974d534dc81ce219 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Thu, 7 Mar 2024 22:36:50 -0500 Subject: [PATCH 2/2] Update percentile assertions based on bug fixes in t-digest --- ...utedFacetSimpleRefinementLongTailTest.java | 4 +- .../solr/search/facet/TestJsonFacets.java | 16 ++--- .../solrj/io/stream/StreamExpressionTest.java | 68 +++++++++---------- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java b/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java index 0be315afc21..017b651dd5b 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java +++ b/solr/core/src/test/org/apache/solr/search/facet/DistributedFacetSimpleRefinementLongTailTest.java @@ -511,7 +511,7 @@ private void checkSubFacetStats(String extraJson) throws Exception { assertEquals(300L, aaa0_Bucket.get("countvals")); assertEquals(0L, aaa0_Bucket.get("missing")); assertEquals(34650.0D, aaa0_Bucket.get("sum")); - assertEquals(483.70000000000016D, (double) aaa0_Bucket.get("percentile"), 0.1E-7); + assertEquals(486.5D, (double) aaa0_Bucket.get("percentile"), 0.1E-7); assertEquals(115.5D, (double) aaa0_Bucket.get("avg"), 0.1E-7); assertEquals(1.674585E7D, (double) aaa0_Bucket.get("sumsq"), 0.1E-7); assertEquals(206.4493184076D, (double) aaa0_Bucket.get("stddev"), 0.1E-7); @@ -548,7 +548,7 @@ private void checkSubFacetStats(String extraJson) throws Exception { assertEquals(35L, tailB_Bucket.get("min")); assertEquals(40L, tailB_Bucket.get("max")); assertEquals(12L, tailB_Bucket.get("countvals")); - assertEquals(39.9D, tailB_Bucket.get("percentile")); + assertEquals(40.0D, tailB_Bucket.get("percentile")); assertEquals(5L, tailB_Bucket.get("missing")); assertEquals(450.0D, tailB_Bucket.get("sum")); assertEquals(37.5D, (double) tailB_Bucket.get("avg"), 0.1E-7); diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java index 7f26565725e..66673b77f57 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java @@ -2327,9 +2327,9 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw + " , f3:{terms:{${terms} field:'${cat_s}', sort:'n1 desc', facet:{n1:'percentile(${sparse_num_d},50)'} }} " + "}"), "facets=={ 'count':6, " - + " f1:{ 'buckets':[{ val:'A', count:2, n1:3.0 }, { val:'B', count:3, n1:-5.0}]}" - + ", f2:{ 'buckets':[{ val:'B', count:3, n1:-5.0}, { val:'A', count:2, n1:3.0 }]}" - + ", f3:{ 'buckets':[{ val:'A', count:2, n1:1.0}, { val:'B', count:3}]}" + + " f1:{ 'buckets':[{ val:'A', count:2, n1:4.0 }, { val:'B', count:3, n1:-5.0}]}" + + ", f2:{ 'buckets':[{ val:'B', count:3, n1:-5.0}, { val:'A', count:2, n1:4.0 }]}" + + ", f3:{ 'buckets':[{ val:'A', count:2, n1:6.0}, { val:'B', count:3}]}" + "}"); // test sorting by multiple percentiles (sort is by first) @@ -2342,8 +2342,8 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw "{f1:{terms:{${terms} field:${cat_s}, sort:'n1 desc', facet:{n1:'percentile(${num_d},50,0,100)'} }}" + " , f2:{terms:{${terms} field:${cat_s}, sort:'n1 asc', facet:{n1:'percentile(${num_d},50,0,100)'} }} }"), "facets=={ 'count':6, " - + " f1:{ 'buckets':[{ val:'A', count:2, n1:[3.0,2.0,4.0] }, { val:'B', count:3, n1:[-5.0,-9.0,11.0] }]}" - + ", f2:{ 'buckets':[{ val:'B', count:3, n1:[-5.0,-9.0,11.0]}, { val:'A', count:2, n1:[3.0,2.0,4.0] }]} }"); + + " f1:{ 'buckets':[{ val:'A', count:2, n1:[4.0,2.0,4.0] }, { val:'B', count:3, n1:[-5.0,-9.0,11.0] }]}" + + ", f2:{ 'buckets':[{ val:'B', count:3, n1:[-5.0,-9.0,11.0]}, { val:'A', count:2, n1:[4.0,2.0,4.0] }]} }"); // test sorting by count/index order client.testJQ( @@ -2812,7 +2812,7 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw "facets=={ 'count':6, " + "sum1:0.0, sumsq1:51.5, avg1:0.0, mind:-5.0, maxd:3.0" + ", mini:-5, maxi:3, mins:'a', maxs:'b'" - + ", stddev:2.712405363721075, variance:7.3571428571, median:0.0, perc:[-5.0,2.25,3.0], maxss:'b'" + + ", stddev:2.712405363721075, variance:7.3571428571, median:0.0, perc:[-5.0,3.0,3.0], maxss:'b'" + "}"); // test sorting by multi-valued @@ -2840,8 +2840,8 @@ public static void doStatsTemplated(Client client, ModifiableSolrParams p) throw "{f1:{terms:{${terms} field:'${cat_s}', sort:'n1 asc', facet:{n1:'percentile(${num_is}, 50)'} }}" + " , f2:{terms:{${terms} field:'${cat_s}', sort:'n1 desc', facet:{n1:'percentile(${num_is}, 50)'} }} }"), "facets=={ 'count':6, " - + " f1:{ 'buckets':[{ val:'B', count:3, n1: -0.50}, { val:'A', count:2, n1:1.0}]}" - + ", f2:{ 'buckets':[{ val:'A', count:2, n1:1.0}, { val:'B', count:3, n1:-0.50 }]} }"); + + " f1:{ 'buckets':[{ val:'B', count:3, n1: 0.0}, { val:'A', count:2, n1:2.0}]}" + + ", f2:{ 'buckets':[{ val:'A', count:2, n1:2.0}, { val:'B', count:3, n1:0.0 }]} }"); // test sorting by multi-valued field with domain query client.testJQ( diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java index 22ce4a5eac3..050b5726ff2 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java @@ -939,8 +939,8 @@ public void testStatsStream() throws Exception { assertEquals(5.5D, avgf, 0.0); assertEquals(5.477225575051661D, stdi, 0.0); assertEquals(3.0276503540974917D, stdf, 0.0); - assertEquals(7.0D, peri, 0.0); - assertEquals(5.5D, perf, 0.0); + assertEquals(10.0D, peri, 0.0); + assertEquals(6.0D, perf, 0.0); assertEquals(10, count, 0.0); assertEquals(countDist.longValue(), 3L); @@ -986,8 +986,8 @@ public void testStatsStream() throws Exception { assertEquals(5.5D, avgf, 0.0); assertEquals(5.477225575051661D, stdi, 0.0); assertEquals(3.0276503540974917D, stdf, 0.0); - assertEquals(7.0D, peri, 0.0); - assertEquals(5.5D, perf, 0.0); + assertEquals(10.0D, peri, 0.0); + assertEquals(6.0D, perf, 0.0); assertEquals(10, count, 0.0); // Test with shards parameter @@ -1036,8 +1036,8 @@ public void testStatsStream() throws Exception { assertEquals(5.5D, avgf, 0.0); assertEquals(5.477225575051661D, stdi, 0.0); assertEquals(3.0276503540974917D, stdf, 0.0); - assertEquals(7.0D, peri, 0.0); - assertEquals(5.5D, perf, 0.0); + assertEquals(10.0D, peri, 0.0); + assertEquals(6.0D, perf, 0.0); assertEquals(10, count, 0.0); // Execersise the /stream hander @@ -1424,8 +1424,8 @@ public void testFacetStream() throws Exception { assertEquals(2, count, 0.0); assertEquals(4.949747468305833D, stdi, 0.0); assertEquals(2.1213203435596424D, stdf, 0.0); - assertEquals(7.5D, peri, 0.0); - assertEquals(5.5D, perf, 0.0); + assertEquals(11.0D, peri, 0.0); + assertEquals(7.0D, perf, 0.0); assertEquals(countDist.longValue(), 2); tuple = tuples.get(1); @@ -1457,8 +1457,8 @@ public void testFacetStream() throws Exception { assertEquals(4, count, 0.0); assertEquals(6.551081335677848D, stdi, 0.0); assertEquals(4.041451884327381D, stdf, 0.0); - assertEquals(1.5D, peri, 0.0); - assertEquals(3.5D, perf, 0.0); + assertEquals(2.0D, peri, 0.0); + assertEquals(5.0D, perf, 0.0); assertEquals(countDist.longValue(), 4); tuple = tuples.get(2); @@ -1490,8 +1490,8 @@ public void testFacetStream() throws Exception { assertEquals(4, count, 0.0); assertEquals(4.509249752822894D, stdi, 0.0); assertEquals(2.6457513110645907D, stdf, 0.0); - assertEquals(11.0D, peri, 0.0); - assertEquals(7.0D, perf, 0.0); + assertEquals(12.0D, peri, 0.0); + assertEquals(8.0D, perf, 0.0); assertEquals(countDist.longValue(), 4); // Reverse the Sort. @@ -1547,8 +1547,8 @@ public void testFacetStream() throws Exception { assertEquals(4, count, 0.0); assertEquals(4.509249752822894D, stdi, 0.0); assertEquals(2.6457513110645907D, stdf, 0.0); - assertEquals(11.0D, peri, 0.0); - assertEquals(7.0D, perf, 0.0); + assertEquals(12.0D, peri, 0.0); + assertEquals(8.0D, perf, 0.0); tuple = tuples.get(1); bucket = tuple.getString("a_s"); @@ -1578,8 +1578,8 @@ public void testFacetStream() throws Exception { assertEquals(4, count, 0.0); assertEquals(6.551081335677848D, stdi, 0.0); assertEquals(4.041451884327381D, stdf, 0.0); - assertEquals(1.5D, peri, 0.0); - assertEquals(3.5D, perf, 0.0); + assertEquals(2.0D, peri, 0.0); + assertEquals(5.0D, perf, 0.0); tuple = tuples.get(2); bucket = tuple.getString("a_s"); @@ -1609,8 +1609,8 @@ public void testFacetStream() throws Exception { assertEquals(2, count, 0.0); assertEquals(4.949747468305833D, stdi, 0.0); assertEquals(2.1213203435596424D, stdf, 0.0); - assertEquals(7.5D, peri, 0.0); - assertEquals(5.5D, perf, 0.0); + assertEquals(11.0D, peri, 0.0); + assertEquals(7.0D, perf, 0.0); clause = "facet(" @@ -1803,8 +1803,8 @@ public void testFacetStream() throws Exception { assertEquals(2, count, 0.0); assertEquals(4.949747468305833D, stdi, 0.0); assertEquals(2.1213203435596424D, stdf, 0.0); - assertEquals(7.5D, peri, 0.0); - assertEquals(5.5D, perf, 0.0); + assertEquals(11.0D, peri, 0.0); + assertEquals(7.0D, perf, 0.0); tuple = tuples.get(1); bucket = tuple.getString("a_s"); @@ -1834,8 +1834,8 @@ public void testFacetStream() throws Exception { assertEquals(4, count, 0.0); assertEquals(4.509249752822894D, stdi, 0.0); assertEquals(2.6457513110645907D, stdf, 0.0); - assertEquals(11.0D, peri, 0.0); - assertEquals(7.0D, perf, 0.0); + assertEquals(12.0D, peri, 0.0); + assertEquals(8.0D, perf, 0.0); tuple = tuples.get(2); bucket = tuple.getString("a_s"); @@ -1865,8 +1865,8 @@ public void testFacetStream() throws Exception { assertEquals(4, count, 0.0); assertEquals(6.551081335677848D, stdi, 0.0); assertEquals(4.041451884327381D, stdf, 0.0); - assertEquals(1.5D, peri, 0.0); - assertEquals(3.5D, perf, 0.0); + assertEquals(2.0D, peri, 0.0); + assertEquals(5.0D, perf, 0.0); // Test index sort @@ -2477,10 +2477,10 @@ public void testSubFacetStream() throws Exception { double stdi = tuple.getDouble("std(a_i)"); double peri = tuple.getDouble("per(a_i,50)"); - assertEquals("hello3", bucket1); + assertEquals("hello0", bucket1); assertEquals("b", bucket2); - assertEquals(1.5275252316519468D, stdi, 0.0); - assertEquals(12.0D, peri, 0.0); + assertEquals(9.192388155425117D, stdi, 0.0); + assertEquals(14.0D, peri, 0.0); tuple = tuples.get(1); bucket1 = tuple.getString("level1_s"); @@ -2488,10 +2488,10 @@ public void testSubFacetStream() throws Exception { stdi = tuple.getDouble("std(a_i)"); peri = tuple.getDouble("per(a_i,50)"); - assertEquals("hello4", bucket1); + assertEquals("hello3", bucket1); assertEquals("b", bucket2); - assertEquals(0.0D, stdi, 0.0); - assertEquals(11.0, peri, 0.0); + assertEquals(1.5275252316519468D, stdi, 0.0); + assertEquals(12.0, peri, 0.0); tuple = tuples.get(2); bucket1 = tuple.getString("level1_s"); @@ -2499,10 +2499,10 @@ public void testSubFacetStream() throws Exception { stdi = tuple.getDouble("std(a_i)"); peri = tuple.getDouble("per(a_i,50)"); - assertEquals("hello0", bucket1); + assertEquals("hello4", bucket1); assertEquals("b", bucket2); - assertEquals(9.192388155425117D, stdi, 0.0); - assertEquals(7.5D, peri, 0.0); + assertEquals(0.0D, stdi, 0.0); + assertEquals(11.0D, peri, 0.0); tuple = tuples.get(3); bucket1 = tuple.getString("level1_s"); @@ -2535,7 +2535,7 @@ public void testSubFacetStream() throws Exception { assertEquals("hello0", bucket1); assertEquals("a", bucket2); assertEquals(1.4142135623730951D, stdi, 0.0); - assertEquals(1.0D, peri, 0.0); + assertEquals(2.0D, peri, 0.0); } @Test