Skip to content

Commit

Permalink
Attempt to add dataset specification as defined in #61
Browse files Browse the repository at this point in the history
  • Loading branch information
@hanslovsky
hanslovsky committed Jun 14, 2018
1 parent d38fed4 commit 53719a8
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 52 deletions.
44 changes: 40 additions & 4 deletions src/main/java/org/janelia/saalfeldlab/paintera/N5Helpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,12 @@ public class N5Helpers

public static final String MAX_NUM_ENTRIES_KEY = "maxNumEntries";

public static final String PAINTERA_DATA_KEY = "painteraData";

public static final String PAINTERA_DATA_DATASET = "data";

public static final String PAINTERA_FRAGMENT_SEGMENT_ASSIGNMENT_DATASTE = "fragment-segment-assignment";

private static final Logger LOG = LoggerFactory.getLogger( MethodHandles.lookup().lookupClass() );

public static boolean isIntegerType( final DataType type )
Expand All @@ -133,6 +139,11 @@ public static boolean isIntegerType( final DataType type )
}
}

public static boolean isPainteraDataset( final N5Reader n5, final String group ) throws IOException
{
return n5.exists( group ) && n5.listAttributes( group ).containsKey( PAINTERA_DATA_KEY );
}

public static boolean isMultiScale( final N5Reader n5, final String dataset ) throws IOException
{
/* based on attribute */
Expand Down Expand Up @@ -245,6 +256,14 @@ public static String[] listAndSortScaleDatasets( final N5Reader n5, final String
return scaleDirs;
}

public static DataType getDataType( final N5Reader n5, final String group ) throws IOException
{
LOG.warn( "Getting data type for group/dataset {}", group );
if ( isPainteraDataset( n5, group ) ) { return getDataType( n5, group + "/" + PAINTERA_DATA_DATASET ); }
if ( isMultiScale( n5, group ) ) { return getDataType( n5, getFinestLevel( n5, group ) ); }
return n5.getDatasetAttributes( group ).getDataType();
}

public static void sortScaleDatasets( final String[] scaleDatasets )
{
Arrays.sort( scaleDatasets, ( f1, f2 ) -> {
Expand Down Expand Up @@ -319,7 +338,14 @@ public static void discoverSubdirectories(
{
try
{
if ( n5.datasetExists( pathName ) )
if ( isPainteraDataset( n5, pathName ) )
{
synchronized ( datasets )
{
datasets.add( pathName );
}
}
else if ( n5.datasetExists( pathName ) )
{
synchronized ( datasets )
{
Expand All @@ -328,8 +354,10 @@ public static void discoverSubdirectories(
}
else
{

String[] groups = null;
/* based on attribute */

boolean isMipmapGroup = Optional.ofNullable( n5.getAttribute( pathName, MULTI_SCALE_KEY, Boolean.class ) ).orElse( false );

/* based on groupd content (the old way) */
Expand Down Expand Up @@ -802,12 +830,19 @@ public static AffineTransform3D considerDownsampling(
return transform.concatenate( new Translation3D( shift ) );
}

public static FragmentSegmentAssignmentState assignments( final N5Writer writer, final String ds ) throws IOException
public static FragmentSegmentAssignmentState assignments( final N5Writer writer, final String group ) throws IOException
{
final String dataset = ds + ".fragment-segment-assignment";

if ( !isPainteraDataset( writer, group ) ) { return new FragmentSegmentAssignmentOnlyLocal(
TLongLongHashMap::new,
( ks, vs ) -> {
throw new UnableToPersist( "Persisting assignments not supported for non Paintera group/dataset " + group );
} ); }

final String dataset = group + "/" + PAINTERA_FRAGMENT_SEGMENT_ASSIGNMENT_DATASTE;

final Persister persister = ( keys, values ) -> {
// TODO handle zero length assignments?
// TODO how to handle zero length assignments?
if ( keys.length == 0 ) { throw new UnableToPersist( "Zero length data, will not persist fragment-segment-assignment." ); }
try
{
Expand Down Expand Up @@ -959,6 +994,7 @@ public static String getFinestLevel(
final N5Reader n5,
final String dataset ) throws IOException
{
LOG.warn( "Getting finest level for dataset {}", dataset );
final String[] scaleDirs = listAndSortScaleDatasets( n5, dataset );
return Paths.get( dataset, scaleDirs[ 0 ] ).toString();
}
Expand Down
7 changes: 3 additions & 4 deletions src/main/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,14 @@ public void accept( final CachedCellImg< UnsignedLongType, ? > canvas, final lon
{
try
{
final boolean isMultiscale = !n5.datasetExists( dataset );
final String dataset = N5Helpers.isPainteraDataset( n5, this.dataset ) ? this.dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : this.dataset;
final boolean isMultiscale = N5Helpers.isMultiScale( n5, dataset );

final CellGrid canvasGrid = canvas.getCellGrid();

final String highestResolutionDataset = isMultiscale ? Paths.get( dataset, N5Helpers.listAndSortScaleDatasets( n5, dataset )[ 0 ] ).toString() : dataset;

if ( !Optional.ofNullable( n5.getAttribute( highestResolutionDataset, N5Helpers.LABEL_MULTISETTYPE_KEY, Boolean.class ) ).orElse( false ) ) {
throw new RuntimeException( "Only label multiset type accepted currently!" );
}
if ( !Optional.ofNullable( n5.getAttribute( highestResolutionDataset, N5Helpers.LABEL_MULTISETTYPE_KEY, Boolean.class ) ).orElse( false ) ) { throw new RuntimeException( "Only label multiset type accepted currently!" ); }

final DatasetAttributes highestResolutionAttributes = n5.getDatasetAttributes( highestResolutionDataset );
final CellGrid highestResolutionGrid = new CellGrid( highestResolutionAttributes.getDimensions(), highestResolutionAttributes.getBlockSize() );
Expand Down
19 changes: 10 additions & 9 deletions src/main/java/org/janelia/saalfeldlab/paintera/data/n5/N5DataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ private static < T extends NativeType< T > > Function< Interpolation, Interpolat
{
return N5Helpers.isLabelMultisetType( n5, dataset )
? i -> new NearestNeighborInterpolatorFactory<>()
: ( Function ) realTypeInterpolation();
: ( Function ) realTypeInterpolation();
}

private static < T extends RealType< T > > Function< Interpolation, InterpolatorFactory< T, RandomAccessible< T > > > realTypeInterpolation()
Expand All @@ -91,27 +91,28 @@ private static < T extends RealType< T > > Function< Interpolation, Interpolator

@SuppressWarnings( { "unchecked", "rawtypes" } )
private static < D extends NativeType< D >, T extends Volatile< D > & NativeType< T > >
Triple< RandomAccessibleInterval< D >[], RandomAccessibleInterval< T >[], AffineTransform3D[] > getData(
final N5Reader reader,
final String dataset,
final AffineTransform3D transform,
final SharedQueue sharedQueue,
final int priority ) throws IOException
Triple< RandomAccessibleInterval< D >[], RandomAccessibleInterval< T >[], AffineTransform3D[] > getData(
final N5Reader reader,
final String dataset,
final AffineTransform3D transform,
final SharedQueue sharedQueue,
final int priority ) throws IOException
{
if ( N5Helpers.isPainteraDataset( reader, dataset ) ) { return getData( reader, dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET, transform, sharedQueue, priority ); }
final boolean isMultiscale = N5Helpers.isMultiScale( reader, dataset );
final boolean isLabelMultiset = N5Helpers.isLabelMultisetType( reader, dataset, isMultiscale );

if ( isLabelMultiset )
{
return isMultiscale
? ( Triple ) N5Helpers.openLabelMultisetMultiscale( reader, dataset, transform, sharedQueue, priority )
: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openLabelMutliset( reader, dataset, transform, sharedQueue, priority ) );
: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openLabelMutliset( reader, dataset, transform, sharedQueue, priority ) );
}
else
{
return isMultiscale
? ( Triple ) N5Helpers.openRawMultiscale( reader, dataset, transform, sharedQueue, priority )
: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openRaw( reader, dataset, transform, sharedQueue, priority ) );
: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openRaw( reader, dataset, transform, sharedQueue, priority ) );
}
}
}
55 changes: 20 additions & 35 deletions src/main/java/org/janelia/saalfeldlab/paintera/ui/opendialog/GenericBackendDialogN5.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,21 +219,21 @@ public GenericBackendDialogN5(
dataset.set( "" );
}

public void updateDatasetInfo( final String dataset, final DatasetInfo info )
public void updateDatasetInfo( final String group, final DatasetInfo info )
{

LOG.debug( "Updating dataset info for dataset {}", dataset );
LOG.debug( "Updating dataset info for dataset {}", group );
try
{
final N5Reader n5 = this.n5.get();
final String ds = N5Helpers.isMultiScale( n5, dataset ) ? N5Helpers.getFinestLevel( n5, dataset ) : dataset;
LOG.debug( "Got dataset={}, ds={}", dataset, ds );

setResolution( N5Helpers.getResolution( n5, dataset ) );
setOffset( N5Helpers.getOffset( n5, dataset ) );
final DatasetAttributes dsAttrs = n5.getDatasetAttributes( ds );
this.datasetInfo.minProperty().set( Optional.ofNullable( n5.getAttribute( dataset, MIN_KEY, Double.class ) ).orElse( N5Helpers.minForType( dsAttrs.getDataType() ) ) );
this.datasetInfo.maxProperty().set( Optional.ofNullable( n5.getAttribute( dataset, MAX_KEY, Double.class ) ).orElse( N5Helpers.maxForType( dsAttrs.getDataType() ) ) );

setResolution( N5Helpers.getResolution( n5, group ) );
setOffset( N5Helpers.getOffset( n5, group ) );

final DataType dataType = N5Helpers.getDataType( n5, group );

this.datasetInfo.minProperty().set( Optional.ofNullable( n5.getAttribute( group, MIN_KEY, Double.class ) ).orElse( N5Helpers.minForType( dataType ) ) );
this.datasetInfo.maxProperty().set( Optional.ofNullable( n5.getAttribute( group, MAX_KEY, Double.class ) ).orElse( N5Helpers.maxForType( dataType ) ) );
}
catch ( final IOException e )
{
Expand Down Expand Up @@ -290,17 +290,19 @@ public IdService idService()
final String dataset = this.dataset.get();

final Long maxId = n5.getAttribute( dataset, "maxId", Long.class );
final boolean isPainteraData = N5Helpers.isPainteraDataset( n5, dataset );
final long actualMaxId;
if ( maxId == null )
{
final String ds = isPainteraData ? dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : dataset;
if ( isLabelMultisetType() )
{
LOG.debug( "Getting id service for label multisets" );
actualMaxId = maxIdLabelMultiset( n5, dataset );
actualMaxId = maxIdLabelMultiset( n5, ds );
}
else if ( isIntegerType() )
{
actualMaxId = maxId( n5, dataset );
actualMaxId = maxId( n5, ds );
}
else
{
Expand Down Expand Up @@ -490,7 +492,12 @@ public boolean isLabelType() throws Exception

public boolean isLabelMultisetType() throws Exception
{
final Boolean attribute = getAttribute( LABEL_MULTISETTYPE_KEY, Boolean.class );
final N5Writer n5 = this.n5.get();
final String dataset = this.dataset.get();
final Boolean attribute = n5.getAttribute(
N5Helpers.isPainteraDataset( n5, dataset ) ? dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : dataset,
N5Helpers.LABEL_MULTISETTYPE_KEY,
Boolean.class );
LOG.debug( "Getting label multiset attribute: {}", attribute );
return Optional.ofNullable( attribute ).orElse( false );
}
Expand Down Expand Up @@ -535,28 +542,6 @@ public boolean isIntegerType() throws Exception
return new CommitCanvasN5( writer, dataset );
}

public < T > T getAttribute( final String key, final Class< T > clazz ) throws IOException
{
final N5Reader n5 = this.n5.get();
final String ds = this.dataset.get();

if ( n5.datasetExists( ds ) )
{
LOG.debug( "Getting attributes for {} and {}", n5, ds );
return n5.getAttribute( ds, key, clazz );
}

final String[] scaleDirs = N5Helpers.listAndSortScaleDatasets( n5, ds );

if ( scaleDirs.length > 0 )
{
LOG.warn( "Getting attributes for mipmap dataset: {} and {}", n5, scaleDirs[ 0 ] );
return n5.getAttribute( Paths.get( ds, scaleDirs[ 0 ] ).toString(), key, clazz );
}

throw new RuntimeException( String.format( "Cannot read dataset attributes for group %s and dataset %s.", n5, ds ) );
}

public ExecutorService propagationExecutor()
{
return this.propagationExecutor;
Expand Down

0 comments on commit 53719a8

Please sign in to comment.