diff --git a/hail/src/main/scala/is/hail/rvd/AbstractRVDSpec.scala b/hail/src/main/scala/is/hail/rvd/AbstractRVDSpec.scala index b1bf4ad3395..e6d83d4333d 100644 --- a/hail/src/main/scala/is/hail/rvd/AbstractRVDSpec.scala +++ b/hail/src/main/scala/is/hail/rvd/AbstractRVDSpec.scala @@ -5,8 +5,8 @@ import is.hail.backend.{ExecuteContext, HailStateManager} import is.hail.compatibility import is.hail.expr.{ir, JSONAnnotationImpex} import is.hail.expr.ir.{ - IR, Literal, PartitionNativeReader, PartitionZippedIndexedNativeReader, - PartitionZippedNativeReader, ReadPartition, ToStream, + flatMapIR, IR, Literal, PartitionNativeReader, PartitionZippedIndexedNativeReader, + PartitionZippedNativeReader, ReadPartition, Ref, ToStream, } import is.hail.expr.ir.lowering.{TableStage, TableStageDependency} import is.hail.io._ @@ -484,48 +484,84 @@ case class IndexedRVDSpec2( ): IR => TableStage = newPartitioner match { case Some(np) => val part = partitioner(ctx.stateManager) + /* ensure the old and new partitioners have the same key, and ensure the new partitioner is + * strict */ val extendedNP = np.extendKey(part.kType) - val tmpPartitioner = part.intersect(extendedNP) assert(key.nonEmpty) - val rSpec = typedCodecSpec - val reader = - ir.PartitionNativeReaderIndexed(rSpec, indexSpec, part.kType.fieldNames, uidFieldName) - - val absPath = path - val partPaths = tmpPartitioner.rangeBounds.map(b => partFiles(part.lowerBoundInterval(b))) - - val kSize = part.kType.size - absolutePartPaths(path) - assert(tmpPartitioner.rangeBounds.size == partPaths.length) - val contextsValues: IndexedSeq[Row] = tmpPartitioner.rangeBounds.map { interval => - val partIdx = part.lowerBoundInterval(interval) - val partPath = partFiles(partIdx) + val reader = ir.PartitionNativeReaderIndexed( + typedCodecSpec, + indexSpec, + part.kType.fieldNames, + uidFieldName, + ) + + def makeCtx(oldPartIdx: Int, newPartIdx: Int): Row = { + val oldInterval = part.rangeBounds(oldPartIdx) + val partFile = partFiles(oldPartIdx) + val intersectionInterval = + extendedNP.rangeBounds(newPartIdx) + .intersect(extendedNP.kord, oldInterval).get Row( - partIdx.toLong, - s"$absPath/parts/$partPath", - s"$absPath/${indexSpec.relPath}/$partPath.idx", - RVDPartitioner.intervalToIRRepresentation(interval, kSize), + oldPartIdx.toLong, + s"$path/parts/$partFile", + s"$path/${indexSpec.relPath}/$partFile.idx", + RVDPartitioner.intervalToIRRepresentation(intersectionInterval, part.kType.size), ) } - assert(TArray(reader.contextType).typeCheck(contextsValues)) + val (nestedContexts, newPartitioner) = if (filterIntervals) { + /* We want to filter to intervals in newPartitioner, while preserving the old partitioning, + * but dropping any partitions we know would be empty. So we construct a map from old + * partitions to the range of overlapping new partitions, dropping any with an empty range. */ + val contextsAndBounds = for { + (oldInterval, oldPartIdx) <- part.rangeBounds.toFastSeq.zipWithIndex + overlapRange = extendedNP.queryInterval(oldInterval) + if overlapRange.nonEmpty + } yield { + val ctxs = overlapRange.map(newPartIdx => makeCtx(oldPartIdx, newPartIdx)) + // the interval spanning all overlapping filter intervals + val newInterval = Interval( + extendedNP.rangeBounds(overlapRange.head).left, + extendedNP.rangeBounds(overlapRange.last).right, + ) + ( + ctxs, + // Shrink oldInterval to the rows filtered to. + // By construction we know oldInterval and newInterval overlap + oldInterval.intersect(extendedNP.kord, newInterval).get, + ) + } + val (nestedContexts, newRangeBounds) = contextsAndBounds.unzip + + (nestedContexts, new RVDPartitioner(part.sm, part.kType, newRangeBounds)) + } else { + /* We want to use newPartitioner as the partitioner, dropping any rows not contained in any + * new partition. So we construct a map from new partitioner to the range of overlapping old + * partitions. */ + val nestedContexts = + extendedNP.rangeBounds.toFastSeq.zipWithIndex.map { case (newInterval, newPartIdx) => + val overlapRange = part.queryInterval(newInterval) + overlapRange.map(oldPartIdx => makeCtx(oldPartIdx, newPartIdx)) + } - val contexts = ir.ToStream(ir.Literal(TArray(reader.contextType), contextsValues)) + (nestedContexts, extendedNP) + } - val body = (ctx: IR) => ir.ReadPartition(ctx, requestedType.rowType, reader) + assert(TArray(TArray(reader.contextType)).typeCheck(nestedContexts)) { (globals: IR) => - val ts = TableStage( + TableStage( globals, - tmpPartitioner, + newPartitioner, TableStageDependency.none, - contexts, - body, + contexts = ir.ToStream(ir.Literal(TArray(TArray(reader.contextType)), nestedContexts)), + body = (ctxs: Ref) => + flatMapIR(ToStream(ctxs, true)) { ctx => + ir.ReadPartition(ctx, requestedType.rowType, reader) + }, ) - if (filterIntervals) ts.repartitionNoShuffle(ctx, part, dropEmptyPartitions = true) - else ts.repartitionNoShuffle(ctx, extendedNP) } case None => diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/README.txt b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/README.txt new file mode 100644 index 00000000000..62f13d3fe96 --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/README.txt @@ -0,0 +1,3 @@ +This folder comprises a Hail (www.hail.is) native Table or MatrixTable. + Written with version 0.2.128-705d4033e0c9 + Created at 2024/03/27 12:03:10 \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/_SUCCESS b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/_SUCCESS new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/README.txt b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/README.txt new file mode 100644 index 00000000000..62f13d3fe96 --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/README.txt @@ -0,0 +1,3 @@ +This folder comprises a Hail (www.hail.is) native Table or MatrixTable. + Written with version 0.2.128-705d4033e0c9 + Created at 2024/03/27 12:03:10 \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/_SUCCESS b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/_SUCCESS new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/metadata.json.gz new file mode 100644 index 00000000000..ba853f2bd33 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/rows/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/rows/metadata.json.gz new file mode 100644 index 00000000000..4b51c50a941 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/rows/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/rows/parts/part-0 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/rows/parts/part-0 new file mode 100644 index 00000000000..5c15140d10d Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/cols/rows/parts/part-0 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/README.txt b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/README.txt new file mode 100644 index 00000000000..62f13d3fe96 --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/README.txt @@ -0,0 +1,3 @@ +This folder comprises a Hail (www.hail.is) native Table or MatrixTable. + Written with version 0.2.128-705d4033e0c9 + Created at 2024/03/27 12:03:10 \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/_SUCCESS b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/_SUCCESS new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/metadata.json.gz new file mode 100644 index 00000000000..ca138832fdd Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/metadata.json.gz new file mode 100644 index 00000000000..5d5444d5a3a Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14 new file mode 100644 index 00000000000..46ce64cb28e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-01-06f6a507-61e2-4bd1-a917-e1809270144c b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-01-06f6a507-61e2-4bd1-a917-e1809270144c new file mode 100644 index 00000000000..9770ebd72d4 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-01-06f6a507-61e2-4bd1-a917-e1809270144c differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde new file mode 100644 index 00000000000..58aa83882f4 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-03-1e085a57-4dcb-4131-bc79-353324ffad47 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-03-1e085a57-4dcb-4131-bc79-353324ffad47 new file mode 100644 index 00000000000..557d34f510f Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-03-1e085a57-4dcb-4131-bc79-353324ffad47 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581 new file mode 100644 index 00000000000..d3ca64b40ba Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d new file mode 100644 index 00000000000..728043e4dd0 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658 new file mode 100644 index 00000000000..ea21ec1388a Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7 new file mode 100644 index 00000000000..850e93422b8 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63 new file mode 100644 index 00000000000..a2747db6ac5 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c new file mode 100644 index 00000000000..e20bed4dea8 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-10-17048169-a98b-49ee-ae4d-62641023b3ac b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-10-17048169-a98b-49ee-ae4d-62641023b3ac new file mode 100644 index 00000000000..5020435b81b Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-10-17048169-a98b-49ee-ae4d-62641023b3ac differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-11-c89858f5-4d78-4739-af31-308a1c257ff4 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-11-c89858f5-4d78-4739-af31-308a1c257ff4 new file mode 100644 index 00000000000..f2ddc1d151a Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-11-c89858f5-4d78-4739-af31-308a1c257ff4 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8 new file mode 100644 index 00000000000..329bf68b7bb Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-13-62566d28-e496-4538-a325-b567be66accf b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-13-62566d28-e496-4538-a325-b567be66accf new file mode 100644 index 00000000000..78c76c8d04e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-13-62566d28-e496-4538-a325-b567be66accf differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6 new file mode 100644 index 00000000000..c5756714bb1 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683 new file mode 100644 index 00000000000..b3a18a17ea0 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431 new file mode 100644 index 00000000000..325ba793d14 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df new file mode 100644 index 00000000000..621aa82fb97 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e new file mode 100644 index 00000000000..788d0cd093f Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200 new file mode 100644 index 00000000000..b58aaa00ba7 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/entries/rows/parts/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/README.txt b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/README.txt new file mode 100644 index 00000000000..62f13d3fe96 --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/README.txt @@ -0,0 +1,3 @@ +This folder comprises a Hail (www.hail.is) native Table or MatrixTable. + Written with version 0.2.128-705d4033e0c9 + Created at 2024/03/27 12:03:10 \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/_SUCCESS b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/_SUCCESS new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/globals/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/globals/metadata.json.gz new file mode 100644 index 00000000000..369b04d91ac Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/globals/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/globals/parts/part-0 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/globals/parts/part-0 new file mode 100644 index 00000000000..89e711531de Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/globals/parts/part-0 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/metadata.json.gz new file mode 100644 index 00000000000..89b81c893f6 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/rows/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/rows/metadata.json.gz new file mode 100644 index 00000000000..369b04d91ac Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/rows/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/rows/parts/part-0 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/rows/parts/part-0 new file mode 100644 index 00000000000..89e711531de Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/globals/rows/parts/part-0 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14.idx/index new file mode 100644 index 00000000000..627c0967eef Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14.idx/metadata.json.gz new file mode 100644 index 00000000000..7d3e3e8fc77 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-01-06f6a507-61e2-4bd1-a917-e1809270144c.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-01-06f6a507-61e2-4bd1-a917-e1809270144c.idx/index new file mode 100644 index 00000000000..a43c7360c42 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-01-06f6a507-61e2-4bd1-a917-e1809270144c.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-01-06f6a507-61e2-4bd1-a917-e1809270144c.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-01-06f6a507-61e2-4bd1-a917-e1809270144c.idx/metadata.json.gz new file mode 100644 index 00000000000..685b6946a41 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-01-06f6a507-61e2-4bd1-a917-e1809270144c.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde.idx/index new file mode 100644 index 00000000000..2d6ef5bfb89 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde.idx/metadata.json.gz new file mode 100644 index 00000000000..6441fab8db8 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-03-1e085a57-4dcb-4131-bc79-353324ffad47.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-03-1e085a57-4dcb-4131-bc79-353324ffad47.idx/index new file mode 100644 index 00000000000..d42ae3f6d26 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-03-1e085a57-4dcb-4131-bc79-353324ffad47.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-03-1e085a57-4dcb-4131-bc79-353324ffad47.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-03-1e085a57-4dcb-4131-bc79-353324ffad47.idx/metadata.json.gz new file mode 100644 index 00000000000..025d6ff3782 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-03-1e085a57-4dcb-4131-bc79-353324ffad47.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581.idx/index new file mode 100644 index 00000000000..e77f2844cd2 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581.idx/metadata.json.gz new file mode 100644 index 00000000000..685b6946a41 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d.idx/index new file mode 100644 index 00000000000..4606b87a655 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d.idx/metadata.json.gz new file mode 100644 index 00000000000..dff25d235e5 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658.idx/index new file mode 100644 index 00000000000..84c9027fccb Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658.idx/metadata.json.gz new file mode 100644 index 00000000000..685b6946a41 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7.idx/index new file mode 100644 index 00000000000..eda3e9c9b38 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7.idx/metadata.json.gz new file mode 100644 index 00000000000..43a1c4fe7eb Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63.idx/index new file mode 100644 index 00000000000..5460a0ff2a7 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63.idx/metadata.json.gz new file mode 100644 index 00000000000..685b6946a41 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c.idx/index new file mode 100644 index 00000000000..08d2e54c744 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c.idx/metadata.json.gz new file mode 100644 index 00000000000..6e9d0a13d7e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-10-17048169-a98b-49ee-ae4d-62641023b3ac.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-10-17048169-a98b-49ee-ae4d-62641023b3ac.idx/index new file mode 100644 index 00000000000..eb46615541c Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-10-17048169-a98b-49ee-ae4d-62641023b3ac.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-10-17048169-a98b-49ee-ae4d-62641023b3ac.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-10-17048169-a98b-49ee-ae4d-62641023b3ac.idx/metadata.json.gz new file mode 100644 index 00000000000..8141d4d180b Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-10-17048169-a98b-49ee-ae4d-62641023b3ac.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-11-c89858f5-4d78-4739-af31-308a1c257ff4.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-11-c89858f5-4d78-4739-af31-308a1c257ff4.idx/index new file mode 100644 index 00000000000..075b284619d Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-11-c89858f5-4d78-4739-af31-308a1c257ff4.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-11-c89858f5-4d78-4739-af31-308a1c257ff4.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-11-c89858f5-4d78-4739-af31-308a1c257ff4.idx/metadata.json.gz new file mode 100644 index 00000000000..43a1c4fe7eb Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-11-c89858f5-4d78-4739-af31-308a1c257ff4.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8.idx/index new file mode 100644 index 00000000000..32dde649c5f Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8.idx/metadata.json.gz new file mode 100644 index 00000000000..685b6946a41 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-13-62566d28-e496-4538-a325-b567be66accf.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-13-62566d28-e496-4538-a325-b567be66accf.idx/index new file mode 100644 index 00000000000..74a6b5222be Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-13-62566d28-e496-4538-a325-b567be66accf.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-13-62566d28-e496-4538-a325-b567be66accf.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-13-62566d28-e496-4538-a325-b567be66accf.idx/metadata.json.gz new file mode 100644 index 00000000000..6e9d0a13d7e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-13-62566d28-e496-4538-a325-b567be66accf.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6.idx/index new file mode 100644 index 00000000000..de143b10175 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6.idx/metadata.json.gz new file mode 100644 index 00000000000..685b6946a41 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683.idx/index new file mode 100644 index 00000000000..443c522300a Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683.idx/metadata.json.gz new file mode 100644 index 00000000000..025d6ff3782 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431.idx/index new file mode 100644 index 00000000000..163abab71e0 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431.idx/metadata.json.gz new file mode 100644 index 00000000000..6e9d0a13d7e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df.idx/index new file mode 100644 index 00000000000..e592bce7110 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df.idx/metadata.json.gz new file mode 100644 index 00000000000..6e9d0a13d7e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e.idx/index new file mode 100644 index 00000000000..c67d666d1a5 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e.idx/metadata.json.gz new file mode 100644 index 00000000000..6e9d0a13d7e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200.idx/index b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200.idx/index new file mode 100644 index 00000000000..a3b1b24d5f7 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200.idx/index differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200.idx/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200.idx/metadata.json.gz new file mode 100644 index 00000000000..025d6ff3782 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/index/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200.idx/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/metadata.json.gz new file mode 100644 index 00000000000..ab3049a6bcf Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/README.txt b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/README.txt new file mode 100644 index 00000000000..62f13d3fe96 --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/README.txt @@ -0,0 +1,3 @@ +This folder comprises a Hail (www.hail.is) native Table or MatrixTable. + Written with version 0.2.128-705d4033e0c9 + Created at 2024/03/27 12:03:10 \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/_SUCCESS b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/_SUCCESS new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/metadata.json b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/metadata.json new file mode 100644 index 00000000000..93b22d27737 --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/metadata.json @@ -0,0 +1 @@ +{"file_version":67328,"hail_version":"0.2.128-705d4033e0c9","references_rel_path":"../references","table_type":"Table{global:Struct{},key:[locus],row:Struct{locus:Locus(GRCh37),alleles:Array[String],rsid:String,qual:Float64,filters:Set[String],info:Struct{NEGATIVE_TRAIN_SITE:Boolean,HWP:Float64,AC:Array[Int32],culprit:String,MQ0:Int32,ReadPosRankSum:Float64,AN:Int32,InbreedingCoeff:Float64,AF:Array[Float64],GQ_STDDEV:Float64,FS:Float64,DP:Int32,GQ_MEAN:Float64,POSITIVE_TRAIN_SITE:Boolean,VQSLOD:Float64,ClippingRankSum:Float64,BaseQRankSum:Float64,MLEAF:Array[Float64],MLEAC:Array[Int32],MQ:Float64,QD:Float64,END:Int32,DB:Boolean,HaplotypeScore:Float64,MQRankSum:Float64,CCC:Int32,NCC:Int32,DS:Boolean}}}","components":{"globals":{"name":"RVDComponentSpec","rel_path":"../globals/rows"},"rows":{"name":"RVDComponentSpec","rel_path":"rows"},"partition_counts":{"name":"PartitionCountsComponentSpec","counts":[18,17,17,18,17,17,17,18,17,17,17,18,17,17,17,18,17,17,17,18]},"properties":{"name":"PropertiesSpec","properties":{"distinctlyKeyed":false}}},"name":"TableSpec"} \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/metadata.json.gz new file mode 100644 index 00000000000..08dc5527ba2 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/metadata.json b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/metadata.json new file mode 100644 index 00000000000..ac31fd099cf --- /dev/null +++ b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/metadata.json @@ -0,0 +1 @@ +{"name":"IndexedRVDSpec2","_key":["locus"],"_codecSpec":{"name":"TypedCodecSpec","_eType":"+EBaseStruct{locus:EBaseStruct{contig:+EBinary,position:+EInt32},alleles:EArray[EBinary],rsid:EBinary,qual:EFloat64,filters:EArray[EBinary],info:EBaseStruct{NEGATIVE_TRAIN_SITE:EBoolean,HWP:EFloat64,AC:EArray[EInt32],culprit:EBinary,MQ0:EInt32,ReadPosRankSum:EFloat64,AN:EInt32,InbreedingCoeff:EFloat64,AF:EArray[EFloat64],GQ_STDDEV:EFloat64,FS:EFloat64,DP:EInt32,GQ_MEAN:EFloat64,POSITIVE_TRAIN_SITE:EBoolean,VQSLOD:EFloat64,ClippingRankSum:EFloat64,BaseQRankSum:EFloat64,MLEAF:EArray[EFloat64],MLEAC:EArray[EInt32],MQ:EFloat64,QD:EFloat64,END:EInt32,DB:EBoolean,HaplotypeScore:EFloat64,MQRankSum:EFloat64,CCC:EInt32,NCC:EInt32,DS:EBoolean}}","_vType":"Struct{locus:Locus(GRCh37),alleles:Array[String],rsid:String,qual:Float64,filters:Set[String],info:Struct{NEGATIVE_TRAIN_SITE:Boolean,HWP:Float64,AC:Array[Int32],culprit:String,MQ0:Int32,ReadPosRankSum:Float64,AN:Int32,InbreedingCoeff:Float64,AF:Array[Float64],GQ_STDDEV:Float64,FS:Float64,DP:Int32,GQ_MEAN:Float64,POSITIVE_TRAIN_SITE:Boolean,VQSLOD:Float64,ClippingRankSum:Float64,BaseQRankSum:Float64,MLEAF:Array[Float64],MLEAC:Array[Int32],MQ:Float64,QD:Float64,END:Int32,DB:Boolean,HaplotypeScore:Float64,MQRankSum:Float64,CCC:Int32,NCC:Int32,DS:Boolean}}","_bufferSpec":{"name":"LEB128BufferSpec","child":{"name":"BlockingBufferSpec","blockSize":65536,"child":{"name":"ZstdBlockBufferSpec","blockSize":65536,"child":{"name":"StreamBlockBufferSpec"}}}}},"_indexSpec":{"name":"IndexSpec2","_relPath":"../../index","_leafCodec":{"name":"TypedCodecSpec","_eType":"EBaseStruct{first_idx:+EInt64,keys:+EArray[+EBaseStruct{key:+EBaseStruct{locus:EBaseStruct{contig:+EBinary,position:+EInt32}},offset:+EInt64,annotation:+EBaseStruct{entries_offset:EInt64}}]}","_vType":"Struct{first_idx:Int64,keys:Array[Struct{key:Struct{locus:Locus(GRCh37)},offset:Int64,annotation:Struct{entries_offset:Int64}}]}","_bufferSpec":{"name":"LEB128BufferSpec","child":{"name":"BlockingBufferSpec","blockSize":65536,"child":{"name":"ZstdBlockBufferSpec","blockSize":65536,"child":{"name":"StreamBlockBufferSpec"}}}}},"_internalNodeCodec":{"name":"TypedCodecSpec","_eType":"EBaseStruct{children:+EArray[+EBaseStruct{index_file_offset:+EInt64,first_idx:+EInt64,first_key:+EBaseStruct{locus:EBaseStruct{contig:+EBinary,position:+EInt32}},first_record_offset:+EInt64,first_annotation:+EBaseStruct{entries_offset:EInt64}}]}","_vType":"Struct{children:Array[Struct{index_file_offset:Int64,first_idx:Int64,first_key:Struct{locus:Locus(GRCh37)},first_record_offset:Int64,first_annotation:Struct{entries_offset:Int64}}]}","_bufferSpec":{"name":"LEB128BufferSpec","child":{"name":"BlockingBufferSpec","blockSize":65536,"child":{"name":"ZstdBlockBufferSpec","blockSize":65536,"child":{"name":"StreamBlockBufferSpec"}}}}},"_keyType":"Struct{locus:Locus(GRCh37)}","_annotationType":"Struct{entries_offset:Int64}"},"_partFiles":["part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14","part-01-06f6a507-61e2-4bd1-a917-e1809270144c","part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde","part-03-1e085a57-4dcb-4131-bc79-353324ffad47","part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581","part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d","part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658","part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7","part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63","part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c","part-10-17048169-a98b-49ee-ae4d-62641023b3ac","part-11-c89858f5-4d78-4739-af31-308a1c257ff4","part-12-3e391e78-782d-495d-a29c-cacc56e1baf8","part-13-62566d28-e496-4538-a325-b567be66accf","part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6","part-15-c4301966-4fd8-4ea0-b439-b49a693bf683","part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431","part-17-0c739863-b5fe-4e33-8f47-3e2751b599df","part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e","part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200"],"_jRangeBounds":[{"start":{"locus":{"contig":"20","position":10019093}},"end":{"locus":{"contig":"20","position":10286773}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":10286773}},"end":{"locus":{"contig":"20","position":10603326}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":10603326}},"end":{"locus":{"contig":"20","position":10625804}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":10625804}},"end":{"locus":{"contig":"20","position":10653469}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":10653469}},"end":{"locus":{"contig":"20","position":13071871}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":13071871}},"end":{"locus":{"contig":"20","position":13260252}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":13260252}},"end":{"locus":{"contig":"20","position":13561632}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":13561632}},"end":{"locus":{"contig":"20","position":13709115}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":13709115}},"end":{"locus":{"contig":"20","position":13798776}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":13798776}},"end":{"locus":{"contig":"20","position":14032627}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":14032627}},"end":{"locus":{"contig":"20","position":15948325}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":15948325}},"end":{"locus":{"contig":"20","position":16347823}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":16347823}},"end":{"locus":{"contig":"20","position":16410559}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":16410559}},"end":{"locus":{"contig":"20","position":17410116}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":17410116}},"end":{"locus":{"contig":"20","position":17475217}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":17475217}},"end":{"locus":{"contig":"20","position":17595540}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":17595540}},"end":{"locus":{"contig":"20","position":17600357}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":17600357}},"end":{"locus":{"contig":"20","position":17608348}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":17608348}},"end":{"locus":{"contig":"20","position":17705709}},"includeStart":true,"includeEnd":true},{"start":{"locus":{"contig":"20","position":17705709}},"end":{"locus":{"contig":"20","position":17970876}},"includeStart":true,"includeEnd":true}],"_attrs":{}} \ No newline at end of file diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/metadata.json.gz b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/metadata.json.gz new file mode 100644 index 00000000000..94fa69a4b5e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/metadata.json.gz differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14 new file mode 100644 index 00000000000..bd9fde1e1a7 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-00-cdb826da-6c5c-47b6-945b-3190a87a6a14 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-01-06f6a507-61e2-4bd1-a917-e1809270144c b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-01-06f6a507-61e2-4bd1-a917-e1809270144c new file mode 100644 index 00000000000..a1cd5b98f8a Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-01-06f6a507-61e2-4bd1-a917-e1809270144c differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde new file mode 100644 index 00000000000..afb8ce8cc61 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-02-881d024c-5baf-4fe6-bc8f-53eda3845bde differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-03-1e085a57-4dcb-4131-bc79-353324ffad47 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-03-1e085a57-4dcb-4131-bc79-353324ffad47 new file mode 100644 index 00000000000..768145d1894 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-03-1e085a57-4dcb-4131-bc79-353324ffad47 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581 new file mode 100644 index 00000000000..6239f2a11bc Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-04-d17ed9aa-6b33-4b0b-85d5-578da32f7581 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d new file mode 100644 index 00000000000..7e2b979ea0a Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-05-40d512f8-23ba-485e-aefa-47eced2bfe6d differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658 new file mode 100644 index 00000000000..b45a4a1edac Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-06-9b2dc9c7-c8b1-4ed4-9056-20142b5f6658 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7 new file mode 100644 index 00000000000..ed86b73273c Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-07-b9a32d97-cb10-4158-aeaa-645dcea68ca7 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63 new file mode 100644 index 00000000000..18f8b835340 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-08-c2a0123f-a3d4-4b80-9c21-73cb2bed0b63 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c new file mode 100644 index 00000000000..970968a799c Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-09-ca197aee-6bfd-4068-b771-e9ca63551a7c differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-10-17048169-a98b-49ee-ae4d-62641023b3ac b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-10-17048169-a98b-49ee-ae4d-62641023b3ac new file mode 100644 index 00000000000..370ced208ec Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-10-17048169-a98b-49ee-ae4d-62641023b3ac differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-11-c89858f5-4d78-4739-af31-308a1c257ff4 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-11-c89858f5-4d78-4739-af31-308a1c257ff4 new file mode 100644 index 00000000000..1977621af1c Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-11-c89858f5-4d78-4739-af31-308a1c257ff4 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8 new file mode 100644 index 00000000000..433d5edfe3f Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-12-3e391e78-782d-495d-a29c-cacc56e1baf8 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-13-62566d28-e496-4538-a325-b567be66accf b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-13-62566d28-e496-4538-a325-b567be66accf new file mode 100644 index 00000000000..a993f23c317 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-13-62566d28-e496-4538-a325-b567be66accf differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6 new file mode 100644 index 00000000000..19c21783ca7 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-14-8ab32ab7-15cd-4302-bb45-6b3dc02db5b6 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683 new file mode 100644 index 00000000000..b72a0a3b17e Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-15-c4301966-4fd8-4ea0-b439-b49a693bf683 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431 new file mode 100644 index 00000000000..807a56fbfad Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-16-8d638c2e-b1a5-4507-ba00-337a02e3f431 differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df new file mode 100644 index 00000000000..90e1b9a99d1 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-17-0c739863-b5fe-4e33-8f47-3e2751b599df differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e new file mode 100644 index 00000000000..77cc5d4ba42 Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-18-35d65ae7-5d1d-43f8-bb21-e6565874975e differ diff --git a/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200 b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200 new file mode 100644 index 00000000000..9b37a0670af Binary files /dev/null and b/hail/src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows/rows/parts/part-19-2fd81de2-5d34-43db-809d-2f1fe1e67200 differ diff --git a/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala b/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala index 6c1cd4a8bde..9a187a40590 100644 --- a/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala @@ -7,12 +7,12 @@ import is.hail.annotations.SafeNDArray import is.hail.expr.Nat import is.hail.expr.ir.TestUtils._ import is.hail.expr.ir.lowering.{DArrayLowering, LowerTableIR} -import is.hail.methods.ForceCountTable +import is.hail.methods.{ForceCountTable, NPartitionsTable} import is.hail.rvd.RVDPartitioner import is.hail.types._ import is.hail.types.virtual._ import is.hail.utils._ - +import is.hail.variant.Locus import org.apache.spark.sql.Row import org.scalatest.{Failed, Succeeded} import org.scalatest.Inspectors.forAll @@ -674,6 +674,25 @@ class TableIRSuite extends HailSuite { assertEvalsTo(TableCount(join), 346L) } + @Test def testNativeReaderWithOverlappingPartitions(): Unit = { + val path = "src/test/resources/sample.vcf-20-partitions-with-overlap.mt/rows" + // i1 overlaps the first two partitions + val i1 = Interval(Row(Locus("20", 10200000)), Row(Locus("20", 10500000)), true, true) + + def test(filterIntervals: Boolean, expectedNParts: Int): Unit = { + val opts = NativeReaderOptions(FastSeq(i1), TLocus("GRCh37"), filterIntervals) + val tr = TableNativeReader(fs, TableNativeReaderParameters(path, Some(opts))) + val tir = TableRead(tr.fullTypeWithoutUIDs, false, tr) + val nParts = TableToValueApply(tir, NPartitionsTable()) + val count = TableToValueApply(tir, ForceCountTable()) + assertEvalsTo(nParts, expectedNParts) + assertEvalsTo(count, 20L) + } + + test(false, 1) + test(true, 2) + } + @Test def testTableKeyBy(): Unit = { implicit val execStrats = ExecStrategy.interpretOnly val data = Array(Array("A", 1), Array("A", 2), Array("B", 1))