fix: Improve infinite scroll virtualization and item count handling in table widget

Author: jacquesikot

app/client/src/widgets/TableWidgetV2/component/TableBody/InifiniteScrollBody/index.tsx

@@ -15,7 +15,6 @@ interface InfiniteScrollBodyProps {
   innerElementType?: ReactElementType;
   isLoading: boolean;
   totalRecordsCount?: number;
-  itemCount: number;
   loadMoreFromEvaluations: () => void;
   pageSize: number;
 }
@@ -51,9 +50,9 @@ const InfiniteScrollBody = React.forwardRef(
               height={props.height}
               infiniteLoaderListRef={infiniteLoaderRef}
               innerElementType={props.innerElementType}
+              itemCount={itemCount}
               onItemsRendered={onItemsRendered}
               outerRef={ref}
-              pageSize={props.pageSize}
               rows={cachedRows}
               tableSizes={props.tableSizes}
             />

app/client/src/widgets/TableWidgetV2/component/TableBody/InifiniteScrollBody/useInfiniteVirtualization.tsx

@@ -27,42 +27,41 @@ export const useInfiniteVirtualization = ({
   rows,
   totalRecordsCount,
 }: UseInfiniteVirtualizationProps): UseInfiniteVirtualizationReturn => {
-  // Keep track of which pages are loaded
   const [loadedPages, setLoadedPages] = useState<LoadedRowsCache>({});
-
-  // Keep track of the last loaded page for appending new data
   const lastLoadedPageRef = useRef<number>(0);
+  const hasMoreDataRef = useRef<boolean>(true); // Track if more data is available
 
-  // Calculate max possible pages based on total records
   const maxPages = useMemo(() => {
     if (!totalRecordsCount) return Infinity;
 
     return Math.ceil(totalRecordsCount / pageSize);
   }, [totalRecordsCount, pageSize]);
 
-  // Effect to handle new data
   useEffect(() => {
     if (rows.length > 0) {
-      // Since we know we get pageSize number of rows each time (except possibly last page)
-      // we can calculate the current page index
       const currentPageIndex = lastLoadedPageRef.current;
 
-      // Store the current page of rows in cache
       setLoadedPages((prev) => ({
         ...prev,
         [currentPageIndex]: rows,
       }));
 
-      // Increment the last loaded page counter for next load
-      lastLoadedPageRef.current = currentPageIndex + 1;
+      // Only increment if we got a full page or some data
+      if (rows.length === pageSize) {
+        lastLoadedPageRef.current = currentPageIndex + 1;
+      } else if (rows.length < pageSize && rows.length > 0) {
+        // If we got less than a full page, assume this is the last page
+        hasMoreDataRef.current = false;
+      }
+    } else if (rows.length === 0 && lastLoadedPageRef.current > 0) {
+      // If no rows are returned and we've loaded at least one page, assume end of data
+      hasMoreDataRef.current = false;
     }
   }, [rows, pageSize]);
 
-  // Memoize the combined rows from cache
   const cachedRows = useMemo(() => {
     const allRows: ReactTableRowType<Record<string, unknown>>[] = [];
 
-    // Add all rows from cache in order
     Object.keys(loadedPages)
       .map(Number)
       .sort((a, b) => a - b)
@@ -77,25 +76,29 @@ export const useInfiniteVirtualization = ({
     (index: number) => {
       const pageIndex = Math.floor(index / pageSize);
 
-      // Consider items loaded if we've reached max pages or if the page is already loaded
-      return pageIndex >= maxPages || pageIndex < lastLoadedPageRef.current;
+      return (
+        pageIndex >= maxPages ||
+        pageIndex < lastLoadedPageRef.current ||
+        !hasMoreDataRef.current
+      );
     },
     [pageSize, maxPages],
   );
 
-  // const itemCount = useMemo(() => {
-  //   return totalRecordsCount || cachedRows.length;
-  // }, [totalRecordsCount, cachedRows.length]);
+  const itemCount = useMemo(() => {
+    // If we know there's no more data, cap itemCount at cachedRows.length
+    if (!hasMoreDataRef.current) {
+      return cachedRows.length;
+    }
+
+    return totalRecordsCount || cachedRows.length;
+  }, [totalRecordsCount, cachedRows.length]);
 
   const loadMoreItems = useCallback(
     async (startIndex: number, stopIndex: number) => {
-      if (!isLoading) {
+      if (!isLoading && hasMoreDataRef.current) {
         const targetPage = Math.floor(stopIndex / pageSize);
 
-        // Only load if:
-        // 1. We haven't loaded this page yet
-        // 2. We haven't reached the max number of pages
-        // 3. We're not currently loading
         if (targetPage >= lastLoadedPageRef.current && targetPage < maxPages) {
           loadMore();
         }
@@ -108,7 +111,7 @@ export const useInfiniteVirtualization = ({
 
   return {
     isItemLoaded,
-    itemCount: totalRecordsCount || Infinity,
+    itemCount,
     loadMoreItems,
     cachedRows,
   };

app/client/src/widgets/TableWidgetV2/component/TableBody/VirtualList.tsx

@@ -32,20 +32,20 @@ interface BaseVirtualListProps {
   height: number;
   tableSizes: TableSizes;
   rows: ReactTableRowType<Record<string, unknown>>[];
-  pageSize: number;
   innerElementType?: ReactElementType;
   outerRef?: React.Ref<SimpleBar>;
   onItemsRendered?: (props: ListOnItemsRenderedProps) => void;
   infiniteLoaderListRef?: React.Ref<FixedSizeList>;
+  itemCount: number;
 }
 
 const BaseVirtualList = React.memo(function BaseVirtualList({
   height,
   infiniteLoaderListRef,
   innerElementType,
+  itemCount,
   onItemsRendered,
   outerRef,
-  pageSize,
   rows,
   tableSizes,
 }: BaseVirtualListProps) {
@@ -58,7 +58,7 @@ const BaseVirtualList = React.memo(function BaseVirtualList({
         2 * tableSizes.VERTICAL_PADDING
       }
       innerElementType={innerElementType}
-      itemCount={Math.max(rows.length, pageSize)}
+      itemCount={itemCount}
       itemData={rows}
       itemSize={tableSizes.ROW_HEIGHT}
       onItemsRendered={onItemsRendered}

app/client/src/widgets/TableWidgetV2/widget/index.tsx

@@ -1960,7 +1960,7 @@ class TableWidgetV2 extends BaseWidget<TableWidgetProps, WidgetState> {
       originalIndex = rowIndex === 0 ? -1 : row[ORIGINAL_INDEX_KEY] ?? rowIndex;
     } else {
       row = filteredTableData[rowIndex];
-      originalIndex = row[ORIGINAL_INDEX_KEY] ?? rowIndex;
+      originalIndex = row ? row[ORIGINAL_INDEX_KEY] ?? rowIndex : rowIndex;
     }
 
     const isNewRow = this.props.isAddRowInProgress && rowIndex === 0;