Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misc CRAM cleanup #1253

Merged
merged 19 commits into from
Jan 17, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/main/java/htsjdk/samtools/BAMSBIIndexer.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
import htsjdk.samtools.seekablestream.SeekableStream;
import htsjdk.samtools.util.BlockCompressedInputStream;
import htsjdk.samtools.util.IOUtil;
import htsjdk.samtools.util.RuntimeEOFException;

import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
Expand Down Expand Up @@ -80,7 +80,7 @@ public static void createIndex(final SeekableStream in, final OutputStream out,
// Process the record start position, then skip to the start of the next BAM record
indexWriter.processRecord(recordStart);
InputStreamUtils.skipFully(blockIn, blockSize);
} catch (EOFException e) {
} catch (RuntimeEOFException e) {
break;
}
}
Expand Down
5 changes: 0 additions & 5 deletions src/main/java/htsjdk/samtools/CRAMBAIIndexer.java
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ public CRAMBAIIndexer(final OutputStream output, final SAMFileHeader fileHeader)
* @param container container to be indexed
*/
public void processContainer(final Container container, final ValidationStringency validationStringency) {
try {
if (container == null || container.isEOF()) {
return;
}
Expand Down Expand Up @@ -162,10 +161,6 @@ public void processContainer(final Container container, final ValidationStringen
processSingleReferenceSlice(slice);
}
}

} catch (final IOException e) {
throw new RuntimeIOException("Failed to read cram container", e);
}
}

/**
Expand Down
11 changes: 3 additions & 8 deletions src/main/java/htsjdk/samtools/CRAMFileReader.java
Original file line number Diff line number Diff line change
Expand Up @@ -315,13 +315,10 @@ public SAMRecordIterator iterator(final SAMFileSpan fileSpan) {
// get the file coordinates for the span:
final long[] coordinateArray = ((BAMFileSpan) fileSpan).toCoordinateArray();
if (coordinateArray == null || coordinateArray.length == 0) return emptyIterator;
try {

// create an input stream that reads the source cram stream only within the coordinate pairs:
final SeekableStream seekableStream = getSeekableStreamOrFailWithRTE();
return new CRAMIterator(seekableStream, referenceSource, coordinateArray, validationStringency);
} catch (final IOException e) {
throw new RuntimeException(e);
}
}

@Override
Expand Down Expand Up @@ -508,16 +505,14 @@ public CRAMIntervalIterator(final QueryInterval[] queries, final boolean contain
super(queries, contained);

if (coordinates != null && coordinates.length != 0) {
try {

unfilteredIterator = new CRAMIterator(
getSeekableStreamOrFailWithRTE(),
referenceSource,
coordinates,
validationStringency
);
} catch (final IOException e) {
throw new RuntimeEOFException(e);
}

getNextRecord(); // advance to the first record that matches the filter criteria
}
}
Expand Down
42 changes: 20 additions & 22 deletions src/main/java/htsjdk/samtools/CRAMIterator.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,15 @@
import htsjdk.samtools.cram.structure.CramHeader;
import htsjdk.samtools.cram.structure.Slice;
import htsjdk.samtools.seekablestream.SeekableStream;
import htsjdk.samtools.util.Log;

import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.util.*;

import htsjdk.samtools.cram.CRAMException;
import htsjdk.samtools.util.RuntimeIOException;

public class CRAMIterator implements SAMRecordIterator {
private static final Log log = Log.getInstance(CRAMIterator.class);
private final CountingInputStream countingInputStream;
private final CramHeader cramHeader;
private final ArrayList<SAMRecord> records;
Expand All @@ -63,8 +61,7 @@ public ValidationStringency getValidationStringency() {
return validationStringency;
}

public void setValidationStringency(
final ValidationStringency validationStringency) {
public void setValidationStringency(final ValidationStringency validationStringency) {
this.validationStringency = validationStringency;
}

Expand All @@ -75,11 +72,13 @@ public void setValidationStringency(
private long samRecordIndex;
private ArrayList<CramCompressionRecord> cramRecords;

public CRAMIterator(final InputStream inputStream, final CRAMReferenceSource referenceSource, final ValidationStringency validationStringency)
throws IOException {
public CRAMIterator(final InputStream inputStream,
final CRAMReferenceSource referenceSource,
final ValidationStringency validationStringency) {
if (null == referenceSource) {
throw new CRAMException("A reference source is required for CRAM files");
}

this.countingInputStream = new CountingInputStream(inputStream);
this.referenceSource = referenceSource;
this.validationStringency = validationStringency;
Expand All @@ -88,17 +87,20 @@ public CRAMIterator(final InputStream inputStream, final CRAMReferenceSource ref
this.containerIterator = containerIterator;

firstContainerOffset = this.countingInputStream.getCount();
records = new ArrayList<SAMRecord>(CRAMContainerStreamWriter.DEFAULT_RECORDS_PER_SLICE);
records = new ArrayList<>(CRAMContainerStreamWriter.DEFAULT_RECORDS_PER_SLICE);
normalizer = new CramNormalizer(cramHeader.getSamFileHeader(),
referenceSource);
parser = new ContainerParser(cramHeader.getSamFileHeader());
}

public CRAMIterator(final SeekableStream seekableStream, final CRAMReferenceSource referenceSource, final long[] coordinates, final ValidationStringency validationStringency)
throws IOException {
public CRAMIterator(final SeekableStream seekableStream,
final CRAMReferenceSource referenceSource,
final long[] coordinates,
final ValidationStringency validationStringency) {
if (null == referenceSource) {
throw new CRAMException("A reference source is required for CRAM files");
}

this.countingInputStream = new CountingInputStream(seekableStream);
this.referenceSource = referenceSource;
this.validationStringency = validationStringency;
Expand All @@ -107,24 +109,24 @@ public CRAMIterator(final SeekableStream seekableStream, final CRAMReferenceSour
this.containerIterator = containerIterator;

firstContainerOffset = containerIterator.getFirstContainerOffset();
records = new ArrayList<SAMRecord>(CRAMContainerStreamWriter.DEFAULT_RECORDS_PER_SLICE);
records = new ArrayList<>(CRAMContainerStreamWriter.DEFAULT_RECORDS_PER_SLICE);
normalizer = new CramNormalizer(cramHeader.getSamFileHeader(),
referenceSource);
parser = new ContainerParser(cramHeader.getSamFileHeader());
}

@Deprecated
public CRAMIterator(final SeekableStream seekableStream, final CRAMReferenceSource referenceSource, final long[] coordinates)
throws IOException {
public CRAMIterator(final SeekableStream seekableStream,
final CRAMReferenceSource referenceSource,
final long[] coordinates) {
this(seekableStream, referenceSource, coordinates, ValidationStringency.DEFAULT_STRINGENCY);
}

public CramHeader getCramHeader() {
return cramHeader;
}

void nextContainer() throws IOException, IllegalArgumentException,
IllegalAccessException, CRAMException {
void nextContainer() throws IllegalArgumentException, CRAMException {

if (containerIterator != null) {
if (!containerIterator.hasNext()) {
Expand All @@ -149,7 +151,7 @@ void nextContainer() throws IOException, IllegalArgumentException,

records.clear();
if (cramRecords == null)
cramRecords = new ArrayList<CramCompressionRecord>(container.nofRecords);
cramRecords = new ArrayList<>(container.nofRecords);
else
cramRecords.clear();

Expand Down Expand Up @@ -250,11 +252,7 @@ public boolean advanceToAlignmentInContainer(final int refIndex, final int pos)
public boolean hasNext() {
if (container != null && container.isEOF()) return false;
if (!iterator.hasNext()) {
try {
nextContainer();
} catch (IOException | IllegalAccessException e) {
throw new SAMException(e);
}
}

return !records.isEmpty();
Expand Down Expand Up @@ -287,10 +285,10 @@ public void close() {
records.clear();
//noinspection EmptyCatchBlock
try {
if (countingInputStream != null)
if (countingInputStream != null) {
countingInputStream.close();
} catch (final IOException e) {
}
} catch (final RuntimeIOException e) { }
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/htsjdk/samtools/cram/CRAIEntry.java
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public CRAIEntry(final int sequenceId,
* @param line string formatted as a CRAI index entry
* @throws CRAIIndex.CRAIIndexException
*/
public CRAIEntry(final String line) throws CRAIIndex.CRAIIndexException {
public CRAIEntry(final String line) {
final String[] chunks = line.split("\t");
if (chunks.length != CRAI_INDEX_COLUMNS) {
throw new CRAIIndex.CRAIIndexException(
Expand Down
10 changes: 8 additions & 2 deletions src/main/java/htsjdk/samtools/cram/CRAIIndex.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import htsjdk.samtools.seekablestream.SeekableMemoryStream;
import htsjdk.samtools.seekablestream.SeekableStream;
import htsjdk.samtools.ValidationStringency;
import htsjdk.samtools.util.RuntimeIOException;

import java.io.*;
import java.util.*;
Expand Down Expand Up @@ -70,11 +71,16 @@ public void processContainer(final Container container) {
}
}

public static SeekableStream openCraiFileAsBaiStream(final File cramIndexFile, final SAMSequenceDictionary dictionary) throws IOException {
public static SeekableStream openCraiFileAsBaiStream(final File cramIndexFile, final SAMSequenceDictionary dictionary) {
try {
return openCraiFileAsBaiStream(new FileInputStream(cramIndexFile), dictionary);
}
catch (final FileNotFoundException e) {
throw new RuntimeIOException(e);
}
}

public static SeekableStream openCraiFileAsBaiStream(final InputStream indexStream, final SAMSequenceDictionary dictionary) throws IOException, CRAIIndexException {
public static SeekableStream openCraiFileAsBaiStream(final InputStream indexStream, final SAMSequenceDictionary dictionary) {
final List<CRAIEntry> full = CRAMCRAIIndexer.readIndex(indexStream).getCRAIEntries();
Collections.sort(full);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import htsjdk.samtools.cram.compression.ExternalCompressor;
import htsjdk.samtools.cram.encoding.writer.CramRecordWriter;
import htsjdk.samtools.cram.io.DefaultBitOutputStream;
import htsjdk.samtools.cram.io.ExposedByteArrayOutputStream;
import htsjdk.samtools.cram.structure.block.ExternalBlock;
import htsjdk.samtools.cram.structure.block.Block;
import htsjdk.samtools.cram.structure.CompressionHeader;
Expand Down
19 changes: 10 additions & 9 deletions src/main/java/htsjdk/samtools/cram/build/ContainerParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,11 @@
import htsjdk.samtools.ValidationStringency;
import htsjdk.samtools.cram.structure.AlignmentSpan;
import htsjdk.samtools.cram.encoding.reader.CramRecordReader;
import htsjdk.samtools.cram.encoding.reader.MultiRefSliceAlignmentSpanReader;
import htsjdk.samtools.cram.structure.CompressionHeader;
import htsjdk.samtools.cram.structure.Container;
import htsjdk.samtools.cram.structure.CramCompressionRecord;
import htsjdk.samtools.cram.structure.Slice;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -44,8 +42,8 @@ public ContainerParser(final SAMFileHeader samFileHeader) {
}

public List<CramCompressionRecord> getRecords(final Container container,
ArrayList<CramCompressionRecord> records, final ValidationStringency validationStringency) throws IllegalArgumentException,
IllegalAccessException {
ArrayList<CramCompressionRecord> records,
final ValidationStringency validationStringency) {
if (container.isEOF()) {
return Collections.emptyList();
}
Expand All @@ -61,7 +59,7 @@ public List<CramCompressionRecord> getRecords(final Container container,
return records;
}

public Map<Integer, AlignmentSpan> getReferences(final Container container, final ValidationStringency validationStringency) throws IOException {
public Map<Integer, AlignmentSpan> getReferences(final Container container, final ValidationStringency validationStringency) {
final Map<Integer, AlignmentSpan> containerSpanMap = new HashMap<>();
for (final Slice slice : container.slices) {
addAllSpans(containerSpanMap, getReferences(slice, container.header, validationStringency));
Expand All @@ -84,7 +82,7 @@ private static Map<Integer, AlignmentSpan> addAllSpans(final Map<Integer, Alignm
return spanMap;
}

Map<Integer, AlignmentSpan> getReferences(final Slice slice, final CompressionHeader header, final ValidationStringency validationStringency) throws IOException {
Map<Integer, AlignmentSpan> getReferences(final Slice slice, final CompressionHeader header, final ValidationStringency validationStringency) {
final Map<Integer, AlignmentSpan> spanMap = new HashMap<>();
switch (slice.sequenceId) {
case SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX:
Expand All @@ -102,7 +100,9 @@ Map<Integer, AlignmentSpan> getReferences(final Slice slice, final CompressionHe
}

ArrayList<CramCompressionRecord> getRecords(ArrayList<CramCompressionRecord> records,
final Slice slice, final CompressionHeader header, final ValidationStringency validationStringency) throws IllegalArgumentException {
final Slice slice,
final CompressionHeader header,
final ValidationStringency validationStringency) {
String seqName = SAMRecord.NO_ALIGNMENT_REFERENCE_NAME;
switch (slice.sequenceId) {
case SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX:
Expand Down Expand Up @@ -153,8 +153,9 @@ ArrayList<CramCompressionRecord> getRecords(ArrayList<CramCompressionRecord> rec
return records;
}

List<CramCompressionRecord> getRecords(final Slice slice, final CompressionHeader header, final ValidationStringency validationStringency)
throws IllegalArgumentException, IllegalAccessException {
List<CramCompressionRecord> getRecords(final Slice slice,
final CompressionHeader header,
final ValidationStringency validationStringency) {
return getRecords(null, slice, header, validationStringency);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import htsjdk.samtools.cram.structure.Container;
import htsjdk.samtools.cram.structure.ContainerIO;

import java.io.IOException;
import java.io.InputStream;

/**
Expand All @@ -19,11 +18,11 @@
*/
public class CramContainerHeaderIterator extends CramContainerIterator {

public CramContainerHeaderIterator(final InputStream inputStream) throws IOException {
public CramContainerHeaderIterator(final InputStream inputStream) {
super(inputStream);
}

protected Container containerFromStream(final Version cramVersion, final CountingInputStream countingStream) throws IOException {
protected Container containerFromStream(final Version cramVersion, final CountingInputStream countingStream) {
final Container container = ContainerIO.readContainerHeader(cramVersion.major, countingStream);
InputStreamUtils.skipFully(countingStream, container.containerByteSize);
return container;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import htsjdk.samtools.cram.structure.ContainerIO;
import htsjdk.samtools.cram.structure.CramHeader;

import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;

Expand All @@ -20,22 +19,18 @@ public class CramContainerIterator implements Iterator<Container> {
private boolean eof = false;
private long offset = 0;

public CramContainerIterator(final InputStream inputStream) throws IOException {
public CramContainerIterator(final InputStream inputStream) {
this.countingInputStream = new CountingInputStream(inputStream);
cramHeader = CramIO.readCramHeader(countingInputStream);
this.offset = countingInputStream.getCount();
}

void readNextContainer() {
try {
nextContainer = containerFromStream(cramHeader.getVersion(), countingInputStream);
final long containerSizeInBytes = countingInputStream.getCount() - offset;

nextContainer.offset = offset;
offset += containerSizeInBytes;
} catch (final IOException e) {
throw new RuntimeException(e);
}

if (nextContainer.isEOF()) {
eof = true;
Expand All @@ -48,9 +43,8 @@ void readNextContainer() {
* @param cramVersion
* @param countingStream
* @return The next Container from the stream.
* @throws IOException
*/
protected Container containerFromStream(final Version cramVersion, final CountingInputStream countingStream) throws IOException {
protected Container containerFromStream(final Version cramVersion, final CountingInputStream countingStream) {
return ContainerIO.readContainer(cramHeader.getVersion(), countingStream);
}

Expand Down
Loading