diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java index 155dbb8271a..55ea471c752 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/AbstractTableSubscription.java @@ -298,7 +298,7 @@ public Format getFormat(Column column) { } /** - * TableData type for both viewports and full table subscriptions. + * TableData type for full table subscriptions. */ @TsIgnore public static class SubscriptionEventData extends UpdateEventData implements ViewportData, SubscriptionTableData { @@ -328,6 +328,28 @@ public JsRangeSet getFullIndex() { } } + /** + * TableData type for viewport subscriptions. + */ + @TsIgnore + public static class ViewportEventData extends SubscriptionEventData { + public ViewportEventData(WebBarrageSubscription subscription, int rowStyleColumn, JsArray columns, + RangeSet added, RangeSet removed, RangeSet modified, ShiftedRange[] shifted) { + super(subscription, rowStyleColumn, columns, added, removed, modified, shifted); + } + + @Override + public Any getData(long key, Column column) { + return super.getData(fullRowSet.getRange().get(key), column); + } + + @Override + public Format getFormat(long index, Column column) { + return super.getFormat(fullRowSet.getRange().get(index), column); + } + } + + /** * Base type to allow trees to extend from here separately from tables. */ @@ -403,7 +425,7 @@ public Any getData(int index, Column column) { @Override public Any getData(long key, Column column) { - return subscription.getData(fullRowSet.getRange().get(key), column.getIndex()); + return subscription.getData(key, column.getIndex()); } @Override @@ -413,24 +435,23 @@ public Format getFormat(int index, Column column) { @Override public Format getFormat(long index, Column column) { - long key = fullRowSet.getRange().get(index); long cellColors = 0; long rowColors = 0; String numberFormat = null; String formatString = null; if (column.getStyleColumnIndex() != null) { - LongWrapper wrapper = subscription.getData(key, column.getStyleColumnIndex()).uncheckedCast(); + LongWrapper wrapper = subscription.getData(index, column.getStyleColumnIndex()).uncheckedCast(); cellColors = wrapper == null ? 0 : wrapper.getWrapped(); } if (rowStyleColumn != NO_ROW_FORMAT_COLUMN) { - LongWrapper wrapper = subscription.getData(key, column.getStyleColumnIndex()).uncheckedCast(); + LongWrapper wrapper = subscription.getData(index, column.getStyleColumnIndex()).uncheckedCast(); rowColors = wrapper == null ? 0 : wrapper.getWrapped(); } if (column.getFormatStringColumnIndex() != null) { - numberFormat = subscription.getData(key, column.getFormatStringColumnIndex()).uncheckedCast(); + numberFormat = subscription.getData(index, column.getFormatStringColumnIndex()).uncheckedCast(); } if (column.getFormatStringColumnIndex() != null) { - formatString = subscription.getData(key, column.getFormatStringColumnIndex()).uncheckedCast(); + formatString = subscription.getData(index, column.getFormatStringColumnIndex()).uncheckedCast(); } return new Format(cellColors, rowColors, numberFormat, formatString); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java index 203dc0c7890..113c3fd6c45 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/subscription/TableViewportSubscription.java @@ -139,7 +139,7 @@ protected void notifyUpdate(RangeSet rowsAdded, RangeSet rowsRemoved, RangeSet t if (rowsAdded.size() != rowsRemoved.size() && originalActive) { fireEventWithDetail(JsTable.EVENT_SIZECHANGED, size()); } - UpdateEventData detail = new SubscriptionEventData(barrageSubscription, rowStyleColumn, getColumns(), rowsAdded, + UpdateEventData detail = new ViewportEventData(barrageSubscription, rowStyleColumn, getColumns(), rowsAdded, rowsRemoved, totalMods, shifted); detail.setOffset(this.viewportRowSet.getFirstRow()); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java index ac70fd09128..bd08d045bf2 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/ChartData.java @@ -5,13 +5,12 @@ import elemental2.core.JsArray; import io.deephaven.web.client.api.Column; -import io.deephaven.web.client.api.JsRangeSet; import io.deephaven.web.client.api.JsTable; import io.deephaven.web.client.api.TableData; import io.deephaven.web.client.api.subscription.AbstractTableSubscription; import io.deephaven.web.client.api.subscription.SubscriptionTableData; -import io.deephaven.web.client.fu.JsSettings; import io.deephaven.web.shared.data.Range; +import io.deephaven.web.shared.data.RangeSet; import io.deephaven.web.shared.fu.JsFunction; import jsinterop.annotations.JsType; import jsinterop.base.Any; @@ -29,118 +28,51 @@ public class ChartData { private final JsTable table; - private final long[] indexes = new long[0];// in the browser, this array can grow + private RangeSet prevRanges; private final Map, JsArray>> cachedData = new HashMap<>(); public ChartData(JsTable table) { this.table = table; } - public void update(AbstractTableSubscription.UpdateEventData tableData) { - SubscriptionTableData data = (SubscriptionTableData) tableData; - Iterator addedIterator = data.getAdded().getRange().rangeIterator(); - Iterator removedIterator = data.getRemoved().getRange().rangeIterator(); - Iterator modifiedIterator = data.getModified().getRange().rangeIterator(); - - Range nextAdded = addedIterator.hasNext() ? addedIterator.next() : null; - Range nextRemoved = removedIterator.hasNext() ? removedIterator.next() : null; - Range nextModified = modifiedIterator.hasNext() ? modifiedIterator.next() : null; - int i = 0; - - // not useful for adding/modifying data, but fast and easy to use when removing rows - JsArray[] allColumns; - if (nextRemoved != null) { + public void update(AbstractTableSubscription.SubscriptionEventData tableData) { + RangeSet positionsForAddedKeys = tableData.getFullIndex().getRange().invert(tableData.getAdded().getRange()); + RangeSet positionsForRemovedKeys = prevRanges.invert(tableData.getRemoved().getRange()); + RangeSet positionsForModifiedKeys = + tableData.getFullIndex().getRange().invert(tableData.getModified().getRange()); + prevRanges = tableData.getFullIndex().getRange().copy(); + // removes first from the previous set of ranges + Iterator removedPositions = positionsForRemovedKeys.reverseRangeIterator(); + JsArray[] allColumns = null; + if (removedPositions.hasNext()) { // noinspection unchecked allColumns = cachedData.values().stream().flatMap(m -> m.values().stream()).toArray(JsArray[]::new); - } else { - allColumns = null; } - while (nextAdded != null || nextRemoved != null || nextModified != null) { - if (i >= indexes.length) { - // We're past the end, nothing possibly left to remove, just append all the new items - // Note that this is the first case we'll hit on initial snapshot - assert nextRemoved == null; - assert nextModified == null; - while (nextAdded != null) { - insertDataRange(tableData, nextAdded, i); - - // increment i past these new items so our offset is correct if there is a next block - i += nextAdded.size(); - - // not bothering with i or lastIndexSeen since we will break after this while loop - nextAdded = addedIterator.hasNext() ? addedIterator.next() : null; - } - break; - } - - long nextIndex = indexes[i]; - - // check for added items first, since we will insert items just before the current - // index, while the other two start at the current index - if (nextAdded != null && nextAdded.getFirst() < nextIndex) { - assert nextAdded.getLast() < nextIndex;// the whole range should be there if any is - - // update the index array and insert the actual data into our mapped columns - insertDataRange(tableData, nextAdded, i); - - // increment i past these new items, so that our "next" is actually next - i += nextAdded.size(); - - nextAdded = addedIterator.hasNext() ? addedIterator.next() : null; - } else if (nextModified != null && nextModified.getFirst() == nextIndex) { - assert nextModified.getLast() >= nextIndex; // somehow being asked to update an item which wasn't - // present - - // the updated block is contiguous, make sure there are at least that many items to tweak - assert indexes.length - i >= nextModified.size(); - - replaceDataRange(tableData, nextModified, i); - - // advance i past this section, since no other change can happen to these rows - i += nextModified.size(); - - nextModified = modifiedIterator.hasNext() ? modifiedIterator.next() : null; - } else if (nextRemoved != null && nextRemoved.getFirst() == nextIndex) { - assert nextRemoved.getLast() >= nextIndex; // somehow being asked to remove an item which wasn't present - - // the block being removed is contiguous, so make sure there are at least that many and splice them out - assert indexes.length - i >= nextRemoved.size(); - - // splice out the indexes - asArray(indexes).splice(i, (int) nextRemoved.size()); - - // splice out the data itself - assert allColumns != null; - for (JsArray column : allColumns) { - column.splice(i, (int) nextRemoved.size()); - } - - // don't in/decrement i, we'll just keep going - nextRemoved = removedIterator.hasNext() ? removedIterator.next() : null; - } else { - - // no match, keep reading - i++; + while (removedPositions.hasNext()) { + Range nextRemoved = removedPositions.next(); + for (JsArray column : allColumns) { + column.splice((int) nextRemoved.getFirst(), (int) nextRemoved.size()); } } - if (JsSettings.isDevMode()) { - assert tableData.getRows().length == indexes.length; - assert cachedData.values().stream().flatMap(m -> m.values().stream()) - .allMatch(arr -> arr.length == indexes.length); - assert cachedData.values().stream().flatMap(m -> m.values().stream()).allMatch(arr -> arr - .reduce((Object val, Any p1, int p2) -> ((Integer) val) + 1, 0) == indexes.length); + // then adds + Iterator addedPositions = positionsForAddedKeys.rangeIterator(); + while (addedPositions.hasNext()) { + Range nextAdded = addedPositions.next(); + insertDataRange(tableData, nextAdded); + } - JsRangeSet fullIndex = ((SubscriptionTableData) tableData).getFullIndex(); - PrimitiveIterator.OfLong iter = fullIndex.getRange().indexIterator(); - for (int j = 0; j < indexes.length; j++) { - assert indexes[j] == iter.nextLong(); - } + Iterator modifiedPositions = positionsForModifiedKeys.rangeIterator(); + while (modifiedPositions.hasNext()) { + Range nextModified = modifiedPositions.next(); + replaceDataRange(tableData, nextModified); } } - private void replaceDataRange(AbstractTableSubscription.UpdateEventData tableData, Range range, int offset) { + private void replaceDataRange(SubscriptionTableData tableData, Range positions) { + RangeSet keys = tableData.getFullIndex().getRange() + .subsetForPositions(RangeSet.ofRange(positions.getFirst(), positions.getLast()), true); // we don't touch the indexes at all, only need to walk each column and replace values in this range for (Entry, JsArray>> columnMap : cachedData.entrySet()) { Column col = table.findColumn(columnMap.getKey()); @@ -149,23 +81,27 @@ private void replaceDataRange(AbstractTableSubscription.UpdateEventData tableDat JsArray arr = mapFuncAndArray.getValue(); // rather than getting a slice and splicing it in, just update each value + PrimitiveIterator.OfLong iter = keys.indexIterator(); + int i = 0; if (func == null) { - for (int i = 0; i < range.size(); i++) { - arr.setAt(offset + i, tableData.getData(range.getFirst() + i, col)); + while (iter.hasNext()) { + arr.setAt(i++, tableData.getData(iter.next(), col)); } } else { - for (int i = 0; i < range.size(); i++) { - arr.setAt(offset + i, func.apply(tableData.getData(range.getFirst() + i, col))); + while (iter.hasNext()) { + arr.setAt(i++, tableData.getData(iter.next(), col)); } } } } } - private void insertDataRange(AbstractTableSubscription.UpdateEventData tableData, Range range, int offset) { - // splice in the new indexes - batchSplice(offset, asArray(indexes), longs(range)); - + /** + * From the event data, insert a contiguous range of rows to each column. + */ + private void insertDataRange(SubscriptionTableData tableData, Range positions) { + RangeSet keys = tableData.getFullIndex().getRange() + .subsetForPositions(RangeSet.ofRange(positions.getFirst(), positions.getLast()), false); // splice in data to each column for (Entry, JsArray>> columnMap : cachedData.entrySet()) { String columnName = columnMap.getKey(); @@ -176,13 +112,13 @@ private void insertDataRange(AbstractTableSubscription.UpdateEventData tableData // here we create a new array and splice it in, to avoid slowly growing the data array in the case // of many rows being added - Any[] values = values(tableData, func, col, range); - batchSplice(offset, arr, values); + Any[] values = values(tableData, func, col, keys); + batchSplice((int) positions.getFirst(), arr, values); } } } - private Any[] batchSplice(int offset, JsArray existingData, Any[] dataToInsert) { + private void batchSplice(int offset, JsArray existingData, Any[] dataToInsert) { int lengthToInsert = dataToInsert.length; JsArray jsArrayToInsert = Js.uncheckedCast(dataToInsert); @@ -193,40 +129,24 @@ private Any[] batchSplice(int offset, JsArray existingData, Any[] dataToIns existingData.splice(offset + i, 0, jsArrayToInsert.slice(i, Math.min(i + batchSize, lengthToInsert)).asArray(new Any[0])); } - - return Js.uncheckedCast(existingData); } - private Any[] values(AbstractTableSubscription.UpdateEventData tableData, JsFunction mapFunc, Column col, - Range insertedRange) { + private Any[] values(SubscriptionTableData tableData, JsFunction mapFunc, Column col, + RangeSet keys) { JsArray result = new JsArray<>(); + PrimitiveIterator.OfLong iter = keys.indexIterator(); if (mapFunc == null) { - for (int i = 0; i < insertedRange.size(); i++) { - result.push(tableData.getData(insertedRange.getFirst() + i, col)); + while (iter.hasNext()) { + result.push(tableData.getData(iter.next(), col)); } } else { - for (int i = 0; i < insertedRange.size(); i++) { - result.push(mapFunc.apply(tableData.getData(insertedRange.getFirst() + i, col))); + while (iter.hasNext()) { + result.push(mapFunc.apply(tableData.getData(iter.next(), col))); } } return Js.uncheckedCast(result); - - } - - private static JsArray asArray(Object obj) { - return Js.uncheckedCast(obj); - } - - private static Any[] longs(Range range) { - long[] longs = new long[(int) range.size()]; - - for (int i = 0; i < longs.length; i++) { - longs[i] = range.getFirst() + i; - } - - return Js.uncheckedCast(longs); } public JsArray getColumn(String columnName, JsFunction mappingFunc, TableData currentUpdate) { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java index bb3d6e0817d..cd156e01a98 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/plot/FigureSubscription.java @@ -258,11 +258,11 @@ private Promise subscribe(final Promise tablePromise this.currentData = new ChartData(table); sub.addEventListener(TableSubscription.EVENT_UPDATED, e -> { // refire with specifics for the columns that we're watching here, after updating data arrays - AbstractTableSubscription.UpdateEventData subscriptionUpdateData = - (AbstractTableSubscription.UpdateEventData) e.detail; + AbstractTableSubscription.SubscriptionEventData subscriptionUpdateData = + (AbstractTableSubscription.SubscriptionEventData) e.detail; currentData.update(subscriptionUpdateData); - CustomEventInit event = CustomEventInit.create(); + CustomEventInit event = CustomEventInit.create(); event.setDetail(new DataUpdateEvent(includedSeries.toArray(new JsSeries[0]), currentData, subscriptionUpdateData)); figure.fireEvent(JsFigure.EVENT_UPDATED, event); @@ -271,7 +271,7 @@ private Promise subscribe(final Promise tablePromise firstEventFired = true; if (downsampleAxisRange != null) { - CustomEventInit successInit = CustomEventInit.create(); + CustomEventInit successInit = CustomEventInit.create(); successInit.setDetail(includedSeries.toArray()); figure.fireEvent(JsFigure.EVENT_DOWNSAMPLEFINISHED, successInit); } diff --git a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java index 3e77c9ccec8..4260069c524 100644 --- a/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java +++ b/web/shared-beans/src/main/java/io/deephaven/web/shared/data/RangeSet.java @@ -400,6 +400,22 @@ public Iterator rangeIterator() { return sortedRanges.iterator(); } + public Iterator reverseRangeIterator() { + return new Iterator<>() { + int i = sortedRanges.size(); + + @Override + public boolean hasNext() { + return i > 0; + } + + @Override + public Range next() { + return sortedRanges.get(--i); + } + }; + } + public PrimitiveIterator.OfLong indexIterator() { if (isEmpty()) { return LongStream.empty().iterator(); @@ -664,13 +680,77 @@ public RangeSet subsetForPositions(RangeSet positions, boolean reversed) { return result; } + /** + * + * @param keys + * @return + */ + public RangeSet invert(RangeSet keys) { + if (keys.isEmpty()) { + return empty(); + } + if (isEmpty()) { + throw new IllegalArgumentException("Keys not found: " + keys); + } + ensureCardinalityCache(); + + List positions = new ArrayList<>(); + + int from = 0; + + Iterator keysIter = keys.rangeIterator(); + long startPos = -1; + long endPos = Long.MIN_VALUE; + while (keysIter.hasNext()) { + Range nextKeyRange = keysIter.next(); + + int index = Collections.binarySearch(sortedRanges.subList(from, sortedRanges.size()), nextKeyRange); + if (index < 0) { + index = -index - 2;// examine the previous element + } + index += from; + if (index < 0) { + throw new IllegalArgumentException("Key " + nextKeyRange.getFirst() + " not found"); + } + Range target = sortedRanges.get(index); + + long newStartPos = + (index == 0 ? 0 : this.cardinality[index - 1]) + (nextKeyRange.getFirst() - target.getFirst()); + if (newStartPos - 1 == endPos) { + // nothing to do, only grow the existing range + // endPos = newStartPos + (nextKeyRange.size() - 1); + } else { + // append the existing range and start a new one + if (endPos != Long.MIN_VALUE) { + positions.add(new Range(startPos, endPos)); + } + + startPos = newStartPos; + // endPos = startPos + (nextKeyRange.size() - 1); + } + endPos = newStartPos + (nextKeyRange.size() - 1); + + from = index; + } + assert endPos != Long.MIN_VALUE; + positions.add(new Range(startPos, endPos)); + + RangeSet result = RangeSet.fromSortedRanges(positions.toArray(new Range[0])); + assert result.size() == keys.size(); + return result; + } + + public long get(long key) { if (key == 0) { return getFirstRow(); } + if (key >= size()) { + return -1; + } ensureCardinalityCache(); - int pos = Arrays.binarySearch(cardinality, key); + int pos = Arrays.binarySearch(cardinality, 0, sortedRanges.size(), key); if (pos >= 0) { return sortedRanges.get(pos + 1).getFirst(); diff --git a/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java b/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java index a2c2522b2d9..1bf1f23bff8 100644 --- a/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java +++ b/web/shared-beans/src/test/java/io/deephaven/web/shared/data/RangeSetTest.java @@ -10,6 +10,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.PrimitiveIterator; import java.util.function.LongConsumer; import java.util.function.Supplier; @@ -565,6 +566,9 @@ public void testGet() { for (int i = 0; i < rows.length; i++) { assertEquals("i=" + i, rows[i], initialRange.get(i)); } + assertEquals(-1, initialRange.get(rows.length)); + assertEquals(-1, initialRange.get(rows.length + 1)); + assertEquals(-1, initialRange.get(rows.length + 100)); initialRange.removeRange(new Range(0, 1)); } @@ -632,4 +636,41 @@ public void testShift() { new ShiftedRange(new Range(29026, 29026), -2), }); } + + @Test + public void testInvert() { + RangeSet r = RangeSet.ofItems(4, 5, 7, 9, 10); + + assertEquals(RangeSet.ofRange(0, 4), r.invert(r)); + assertEquals(RangeSet.empty(), r.invert(RangeSet.empty())); + assertEquals(RangeSet.ofItems(0, 2, 4), r.invert(RangeSet.ofItems(4, 7, 10))); + assertEquals(RangeSet.ofItems(1, 2, 3), r.invert(RangeSet.ofItems(5, 7, 9))); + + RangeSet positions = RangeSet.ofItems(18, 37, 83, 88); + RangeSet keys = RangeSet.fromSortedRanges(new Range[] { + new Range(1073739467, 1073739511), new Range(1073739568, 1073739639), new Range(1073739700, 1073739767), + new Range(1073739828, 1073739895), new Range(1073739956, 1073740023), new Range(1073740083, 1073740151), + new Range(1073740214, 1073740279), new Range(1073740342, 1073740407), new Range(1073740464, 1073740535), + new Range(1073740597, 1073740663), new Range(1073740725, 1073740791), new Range(1073740851, 1073740924), + new Range(1073740937, 1073741052), new Range(1073741063, 1073741180), new Range(1073741189, 1073741308), + new Range(1073741310, 1073741436), new Range(1073741440, 1073741564), new Range(1073741569, 1073741692), + new Range(1073741696, 1073741948), new Range(1073741968, 1073742065), new Range(1073742096, 1073742184), + new Range(1073742224, 1073742325), new Range(1073742352, 1073742454), new Range(1073742480, 1073742579), + new Range(1073742608, 1073742702), new Range(1073742736, 1073742840), new Range(1073742864, 1073742967), + new Range(1073742992, 1073743084), new Range(1073743120, 1073743218), new Range(1073743248, 1073743348), + new Range(1073743376, 1073743465), new Range(1073743504, 1073743597), new Range(1073743632, 1073743741), + new Range(1073743760, 1073743869), new Range(1073743888, 1073743912) + }); + RangeSet selected = RangeSet.ofItems(1073739485, 1073739504, 1073739606, 1073739611); + PrimitiveIterator.OfLong selectedIter = selected.indexIterator(); + PrimitiveIterator.OfLong positionsIter = positions.indexIterator(); + while (selectedIter.hasNext()) { + assert positionsIter.hasNext(); + long s = selectedIter.nextLong(); + long p = positionsIter.nextLong(); + assertEquals(s, keys.get(p)); + } + assert !positionsIter.hasNext(); + assertEquals(positions, keys.invert(selected)); + } }