add Begin End ranges (#585)

* add begin end ranges

* lazy math

* reordanise

* tests and fixes for lookups and dimensions

* beginend

* fix stack indexing stability

* test Begin ENd

* allow nightly failure

* continue-on-error

* bugfix

* some renaming

* add Begin End docs

* locus in docs

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.allow_failure }}
     strategy:
       fail-fast: true
       matrix:
@@ -23,10 +24,12 @@ jobs:
           - windows-latest
         arch:
           - x64
+        allow_failure: [false]
         include:
           - version: 'nightly'
             os: ubuntu-latest
             arch: x64
+            allow_failure: true
     steps:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@v1

docs/src/api/lookuparrays.md

@@ -19,7 +19,7 @@ Lookups.Transformed
 Dimensions.MergedLookup
 Lookups.NoLookup
 Lookups.AutoLookup
-Lookups.AutoIndex
+Lookups.AutoValues
 ```
 
 The generic value getter `val`
@@ -33,7 +33,6 @@ Lookup methods:
 ```@docs
 bounds
 hasselection
-Lookups.index
 Lookups.sampling
 Lookups.span
 Lookups.order
@@ -91,14 +90,15 @@ Lookups.Points
 Lookups.Intervals
 ```
 
-### Loci
+### Positions
 
 ```@docs
-Lookups.Locus
+Position
 Lookups.Center
 Lookups.Start
+Lookups.Begin
 Lookups.End
-Lookups.AutoLocus
+Lookups.AutoPosition
 ```
 
 ## Metadata

docs/src/dimarrays.md

@@ -116,6 +116,23 @@ Mixing them will throw an error:
 da1[X(3), 4]
 ```
 
+## Begin End indexing
+
+```@ansi dimarray
+da1[X=Begin+1, Y=End]
+```
+
+It also works in ranges, even with basic math:
+
+```@ansi dimarray
+da1[X=Begin:Begin+1, Y=Begin+1:End-1]
+```
+
+In base julia the keywords `begin` and `end` can be used to
+index the first or last element of an array. But this doesn't 
+work when named indexing is used. Instead you can use the types
+`Begin` and `End`.
+
 ::: info Indexing
 
 Indexing `AbstractDimArray`s works with `getindex`, `setindex!` and

src/DimensionalData.jl

@@ -38,7 +38,7 @@ using .Dimensions.Lookups
 using .Dimensions: StandardIndices, DimOrDimType, DimTuple, DimTupleOrEmpty, DimType, AllDims
 import .Lookups: metadata, set, _set, rebuild, basetypeof, 
     order, span, sampling, locus, val, index, bounds, intervalbounds,
-    hasselection, units, SelectorOrInterval
+    hasselection, units, SelectorOrInterval, Begin, End
 import .Dimensions: dims, refdims, name, lookup, kw2dims, hasdim, label, _astuple
 
 import DataAPI.groupby
@@ -56,6 +56,8 @@ export X, Y, Z, Ti, Dim, Coord
 # Selector
 export At, Between, Touches, Contains, Near, Where, All, .., Not, Bins, CyclicBins
 
+export Begin, End
+
 export AbstractDimArray, DimArray
 
 export AbstractDimVector, AbstractDimMatrix, AbstractDimVecOrMat, DimVector, DimMatrix, DimVecOrMat

src/Dimensions/Dimensions.jl

@@ -25,9 +25,10 @@ const LookupArrays = Lookups
 
 import .Lookups: rebuild, order, span, sampling, locus, val, index, set, _set,
     metadata, bounds, intervalbounds, units, basetypeof, unwrap, selectindices, hasselection,
-    shiftlocus, maybeshiftlocus, SelectorOrInterval, Interval
+    shiftlocus, maybeshiftlocus
 using .Lookups: StandardIndices, SelTuple, CategoricalEltypes,
-    LookupTrait, AllMetadata, LookupSetters
+    LookupTrait, AllMetadata, LookupSetters, AbstractBeginEndRange,
+    SelectorOrInterval, Interval
 
 using Base: tail, OneTo, @propagate_inbounds
 

src/Dimensions/dimension.jl

@@ -7,15 +7,14 @@ Example concrete implementations are [`X`](@ref), [`Y`](@ref), [`Z`](@ref),
 [`Ti`](@ref) (Time), and the custom [`Dim`]@ref) dimension.
 
 `Dimension`s label the axes of an `AbstractDimArray`,
-or other dimensional objects, and are used to index into the array.
+or other dimensional objects, and are used to index into an array.
 
-They may also provide an alternate index to lookup for each array axis.
-This may be any `AbstractVector` matching the array axis length, or a `Val`
-holding a tuple for compile-time index lookups.
+They may also wrap lookup values for each array axis.
+This may be any `AbstractVector` matching the array axis length,
+but will usually be converted to a `Lookup` when use in a constructed
+object.
 
-`Dimension`s also have `lookup` and `metadata` fields.
-
-`lookup` gives more details about the dimension, such as that it is
+A `Lookup` gives more details about the dimension, such as that it is
 [`Categorical`](@ref) or [`Sampled`](@ref) as [`Points`](@ref) or
 [`Intervals`](@ref) along some transect. DimensionalData will
 attempt to guess the lookup from the passed-in index value.
@@ -90,9 +89,6 @@ x = A[X(Between(3, 4)), Y(At('b'))]
  ↓ →   2021-01-01T00:00:00   2021-02-01T00:00:00  …   2021-12-01T00:00:00
  4    0.0                   0.0                      0.0
 ```
-
-`Dimension` objects may have [`lookup`](@ref) and [`metadata`](@ref) fields
-to track additional information about the data and the index, and their relationship.
 """
 abstract type Dimension{T} end
 
@@ -194,9 +190,6 @@ label(x) = string(string(name(x)), (units(x) === nothing ? "" : string(" ", unit
 # Lookups methods
 Lookups.metadata(dim::Dimension) = metadata(lookup(dim))
 
-Lookups.index(dim::Dimension{<:AbstractArray}) = index(val(dim))
-Lookups.index(dim::Dimension{<:Val}) = unwrap(index(val(dim)))
-
 Lookups.bounds(dim::Dimension) = bounds(val(dim))
 Lookups.intervalbounds(dim::Dimension, args...) = intervalbounds(val(dim), args...)
 for f in (:shiftlocus, :maybeshiftlocus)
@@ -255,6 +248,9 @@ end
 @inline selectindices(ds::DimTuple, sel::Tuple) = selectindices(val(ds), sel)
 @inline selectindices(dim::Dimension, sel) = selectindices(val(dim), sel)
 
+# Deprecated
+Lookups.index(dim::Dimension{<:AbstractArray}) = index(val(dim))
+Lookups.index(dim::Dimension{<:Val}) = unwrap(index(val(dim)))
 
 # Base methods
 const ArrayOrVal = Union{AbstractArray,Val}
@@ -309,7 +305,7 @@ _dim2boundsmatrix(::Locus, span::Explicit, lookup) = val(span)
 function _dim2boundsmatrix(::Locus, span::Regular, lookup)
     # Only offset starts and reuse them for ends, 
     # so floating point error is the same.
-    starts = Lookups._shiftindexlocus(Start(), lookup)
+    starts = Lookups._shiftlocus(Start(), lookup)
     dest = Array{eltype(starts),2}(undef, 2, length(starts))
     # Use `bounds` as the start/end values
     if order(lookup) isa ReverseOrdered

src/Dimensions/format.jl

@@ -11,7 +11,7 @@ Errors are thrown if dims don't match the array dims or size,
 and any fields holding `Auto-` objects are filled with guessed objects.
 
 If a [`Lookup`](@ref) hasn't been specified, a lookup is chosen
-based on the type and element type of the index.
+based on the type and element type of the values.
 """
 format(dims, A::AbstractArray) = format((dims,), A)
 function format(dims::NamedTuple, A::AbstractArray)
@@ -49,99 +49,99 @@ format(v, D::Type, axis::AbstractRange) = _valformaterror(v, D)
 
 # Format Lookups
 # No more identification required for NoLookup
-format(m::NoLookup, D::Type, index, axis::AbstractRange) = m
-format(m::NoLookup, D::Type, index::AutoIndex, axis::AbstractRange) = NoLookup(axis)
+format(m::NoLookup, D::Type, values, axis::AbstractRange) = m
+format(m::NoLookup, D::Type, values::AutoValues, axis::AbstractRange) = NoLookup(axis)
 # # AutoLookup
-function format(m::AutoLookup, D::Type, index::AbstractArray{T}, axis::AbstractRange) where T
-    # A mixed type index is Categorical
+function format(m::AutoLookup, D::Type, values::AbstractArray{T}, axis::AbstractRange) where T
+    # A mixed type lookup is Categorical
     m = if isconcretetype(T) 
         Sampled(; order=order(m), span=span(m), sampling=sampling(m), metadata=metadata(m))
     else
         o = order(m) isa AutoOrder ? Unordered() : order(m)
         Categorical(; order=o, metadata=metadata(m))
     end
-    format(m, D, index, axis)
+    format(m, D, values, axis)
 end
-function format(m::AutoLookup, D::Type, index::AbstractArray{<:CategoricalEltypes}, axis::AbstractRange)
-    o = _format(order(m), D, index)
-    return Categorical(index; order=o, metadata=metadata(m))
+function format(m::AutoLookup, D::Type, values::AbstractArray{<:CategoricalEltypes}, axis::AbstractRange)
+    o = _format(order(m), D, values)
+    return Categorical(values; order=o, metadata=metadata(m))
 end
-function format(m::Categorical, D::Type, index, axis::AbstractRange)
-    i = _format(index, axis)
-    o = _format(order(m), D, index)
+function format(m::Categorical, D::Type, values, axis::AbstractRange)
+    i = _format(values, axis)
+    o = _format(order(m), D, values)
     return rebuild(m; data=i, order=o)
 end
-# # Sampled
-function format(m::AbstractSampled, D::Type, index, axis::AbstractRange)
-    i = _format(index, axis)
-    o = _format(order(m), D, index)
-    sp = _format(span(m), D, index)
-    sa = _format(sampling(m), sp, D, index)
+# Sampled
+function format(m::AbstractSampled, D::Type, values, axis::AbstractRange)
+    i = _format(values, axis)
+    o = _format(order(m), D, values)
+    sp = _format(span(m), D, values)
+    sa = _format(sampling(m), sp, D, values)
     x = rebuild(m; data=i, order=o, span=sp, sampling=sa)
     return x
 end
-# # Transformed
-format(m::Transformed, D::Type, index::AutoIndex, axis::AbstractRange) =
+# Transformed
+format(m::Transformed, D::Type, values::AutoValues, axis::AbstractRange) =
     rebuild(m; dim=D(), data=axis)
-format(m::Transformed, D::Type, index, axis::AbstractRange) = rebuild(m; dim=D())
+format(m::Transformed, D::Type, values, axis::AbstractRange) = rebuild(m; dim=D())
 
-# Index
-_format(index::AbstractArray, axis::AbstractRange) = index
-_format(index::AutoLookup, axis::AbstractRange) = axis
+# Values
+_format(values::AbstractArray, axis::AbstractRange) = values
+_format(values::AutoLookup, axis::AbstractRange) = axis
 # Order
-_format(order::Order, D::Type, index) = order
-_format(order::AutoOrder, D::Type, index) = Lookups.orderof(index)
+_format(order::Order, D::Type, values) = order
+_format(order::AutoOrder, D::Type, values) = Lookups.orderof(values)
 # Span
-_format(span::AutoSpan, D::Type, index::Union{AbstractArray,Val}) =
-    _format(Irregular(), D, index)
-_format(span::AutoSpan, D::Type, index::AbstractRange) = Regular(step(index))
-_format(span::Regular{AutoStep}, D::Type, index::Union{AbstractArray,Val}) = _arraynosteperror()
-_format(span::Regular{AutoStep}, D::Type, index::LinRange) = Regular(step(index))
-_format(span::Regular{AutoStep}, D::Type, index::AbstractRange) = Regular(step(index))
-_format(span::Regular, D::Type, index::Union{AbstractArray,Val}) = span
-function _format(span::Regular, D::Type, index::AbstractRange)
-    step(span) isa Number && !(step(span) ≈ step(index)) && _steperror(index, span)
+_format(span::AutoSpan, D::Type, values::Union{AbstractArray,Val}) =
+    _format(Irregular(), D, values)
+_format(span::AutoSpan, D::Type, values::AbstractRange) = Regular(step(values))
+_format(span::Regular{AutoStep}, D::Type, values::Union{AbstractArray,Val}) = _arraynosteperror()
+_format(span::Regular{AutoStep}, D::Type, values::LinRange) = Regular(step(values))
+_format(span::Regular{AutoStep}, D::Type, values::AbstractRange) = Regular(step(values))
+_format(span::Regular, D::Type, values::Union{AbstractArray,Val}) = span
+function _format(span::Regular, D::Type, values::AbstractRange)
+    step(span) isa Number && !(step(span) ≈ step(values)) && _steperror(values, span)
     return span
 end
-function _format(span::Regular, D::Type, index::LinRange{T}) where T
-    step(span) isa Number && step(index) > zero(T) && !(step(span) ≈ step(index)) && _steperror(index, span)
+function _format(span::Regular, D::Type, values::LinRange{T}) where T
+    step(span) isa Number && step(values) > zero(T) && !(step(span) ≈ step(values)) && _steperror(values, span)
     return span
 end
-_format(span::Irregular{AutoBounds}, D, index) = Irregular(nothing, nothing)
-_format(span::Irregular{<:Tuple}, D, index) = span
-_format(span::Explicit, D, index) = span
+_format(span::Irregular{AutoBounds}, D, values) = Irregular(nothing, nothing)
+_format(span::Irregular{<:Tuple}, D, values) = span
+_format(span::Explicit, D, values) = span
 # Sampling
-_format(sampling::AutoSampling, span::Span, D::Type, index) = Points()
+_format(sampling::AutoSampling, span::Span, D::Type, values) = Points()
 _format(::AutoSampling, ::Span, D::Type, ::AbstractArray{<:IntervalSets.Interval}) =
     Intervals(Start())
-_format(sampling::AutoSampling, span::Explicit, D::Type, index) =
-    Intervals(_format(locus(sampling), D, index))
+_format(sampling::AutoSampling, span::Explicit, D::Type, values) =
+    Intervals(_format(locus(sampling), D, values))
 # For ambiguity, not likely to happen in practice
 _format(::AutoSampling, ::Explicit, D::Type, ::AbstractArray{<:IntervalSets.Interval}) =
-    Intervals(_format(locus(sampling), D, index))
-_format(sampling::Points, span::Span, D::Type, index) = sampling
-_format(sampling::Points, span::Explicit, D::Type, index) = _explicitpoints_error() 
-_format(sampling::Intervals, span::Span, D::Type, index) =
-    rebuild(sampling, _format(locus(sampling), D, index))
+    Intervals(_format(locus(sampling), D, values))
+_format(sampling::Points, span::Span, D::Type, values) = sampling
+_format(sampling::Points, span::Explicit, D::Type, values) = _explicitpoints_error() 
+_format(sampling::Intervals, span::Span, D::Type, values) =
+    rebuild(sampling, _format(locus(sampling), D, values))
 # Locus
-_format(locus::AutoLocus, D::Type, index) = Center()
+_format(locus::AutoLocus, D::Type, values) = Center()
 # Time dimensions need to default to the Start() locus, as that is
 # nearly always the _format and Center intervals are difficult to
 # calculate with DateTime step values.
-_format(locus::AutoLocus, D::Type{<:TimeDim}, index) = Start()
-_format(locus::Locus, D::Type, index) = locus
+_format(locus::AutoLocus, D::Type{<:TimeDim}, values) = Start()
+_format(locus::Locus, D::Type, values) = locus
 
-_order(index) = first(index) <= last(index) ? ForwardOrdered() : ReverseOrdered()
+_order(values) = first(values) <= last(values) ? ForwardOrdered() : ReverseOrdered()
 
 checkaxis(lookup::Transformed, axis) = nothing
 checkaxis(lookup, axis) = first(axes(lookup)) == axis || _checkaxiserror(lookup, axis)
 
 @noinline _explicitpoints_error() =
     throw(ArgumentError("Cannot use Explicit span with Points sampling"))
-@noinline _steperror(index, span) =
-    throw(ArgumentError("lookup step $(step(span)) does not match index step $(step(index))"))
+@noinline _steperror(values, span) =
+    throw(ArgumentError("lookup step $(step(span)) does not match lookup step $(step(values))"))
 @noinline _arraynosteperror() =
-    throw(ArgumentError("`Regular` must specify `step` size with an index other than `AbstractRange`"))
+    throw(ArgumentError("`Regular` must specify `step` size with values other than `AbstractRange`"))
 @noinline _checkaxiserror(lookup, axis) =
     throw(DimensionMismatch(
         "axes of $(basetypeof(lookup)) of $(first(axes(lookup))) do not match array axis of $axis"

src/Dimensions/indexing.jl

@@ -14,9 +14,9 @@ for f in (:getindex, :view, :dotview)
         @propagate_inbounds function Base.$f(d::Dimension{<:AbstractArray}, i::SelectorOrInterval)
             Base.$f(d, selectindices(val(d), i))
         end
-        # Everything else (like custom indexing from other packages) passes through to the parent
         @propagate_inbounds function Base.$f(d::Dimension{<:AbstractArray}, i)
-            Base.$f(parent(d), i)
+            x = Base.$f(parent(d), i)
+            x isa AbstractArray ? rebuild(d, x) : x
         end
     end
 end
@@ -28,9 +28,7 @@ end
 
 Convert a `Dimension` or `Selector` `I` to indices of `Int`, `AbstractArray` or `Colon`.
 """
-@inline dims2indices(dim::Dimension, I::StandardIndices) = I
 @inline dims2indices(dim::Dimension, I) = _dims2indices(dim, I)
-
 @inline dims2indices(x, I) = dims2indices(dims(x), I)
 @inline dims2indices(::Nothing, I) = _dimsnotdefinederror()
 @inline dims2indices(::Tuple{}, I) = ()
@@ -106,13 +104,16 @@ _unwrapdim(dim::Dimension) = val(dim)
 _unwrapdim(x) = x
 
 # Single dim methods
+# Simply unwrap dimensions
+@inline _dims2indices(dim::Dimension, seldim::Dimension) = _dims2indices(dim, val(seldim))
 # A Dimension type always means Colon(), as if it was constructed with the default value.
 @inline _dims2indices(dim::Dimension, ::Type{<:Dimension}) = Colon()
 # Nothing means nothing was passed for this dimension
+@inline _dims2indices(dim::Dimension, i::AbstractBeginEndRange) = i
+@inline _dims2indices(dim::Dimension, i::Union{LU.Begin,LU.End,Type{LU.Begin},Type{LU.End}}) = 
+    to_indices(parent(dim), (i,))[1]
 @inline _dims2indices(dim::Dimension, ::Nothing) = Colon()
-# Simply unwrap dimensions
-@inline _dims2indices(dim::Dimension, seldim::Dimension) = 
-    Lookups.selectindices(val(dim), val(seldim))
+@inline _dims2indices(dim::Dimension, x) = Lookups.selectindices(val(dim), x)
 
 function _extent_as_intervals(extent::Extents.Extent{Keys}) where Keys
     map(map(key2dim, Keys), values(extent)) do k, v