split ComDOM streams into success, error, all. Patch several pages

src/app/moncomdom/page.tsx

@@ -68,7 +68,7 @@ export default function RSETable() {
     )
 
     const table = useReactTable<RSEOld>({
-        data: query.data || [],
+        data: query.data.all || [],
         columns: columns,
         getCoreRowModel: getCoreRowModel(),
         debugTable: true,

src/component-library/Pages/DID/PageDID.tsx

@@ -122,28 +122,28 @@ export const PageDID = (
                 />
                 <SubPage
                     show={showPageBools["subpage-rules"]()}
-                    run={() => { if (props.didRulesComDOM.query.data.length === 0) { props.didRulesComDOM.start() } }}
+                    run={() => { if (props.didRulesComDOM.query.data.all.length === 0) { props.didRulesComDOM.start() } }}
                     id="subpage-rules"
                 >
                     <PageDIDRules comdom={props.didRulesComDOM} />
                 </SubPage>
                 <SubPage
                     show={didtype === DIDType.FILE ? false : didtype === DIDType.DATASET ? subpageIndex === 1 : false}
-                    run={() => { if (props.didDatasetReplicasComDOM.query.data.length === 0) { props.didDatasetReplicasComDOM.start() } }}
+                    run={() => { if (props.didDatasetReplicasComDOM.query.data.all.length === 0) { props.didDatasetReplicasComDOM.start() } }}
                     id="subpage-dataset-replicas"
                 >
                     <PageDIDDatasetReplicas comdom={props.didDatasetReplicasComDOM} />
                 </SubPage>
                 <SubPage
                     show={showPageBools["subpage-file-replica-states"]()}
-                    run={() => { if (props.didFileReplicasComDOM.query.data.length === 0) { props.didFileReplicasComDOM.start() } }}
+                    run={() => { if (props.didFileReplicasComDOM.query.data.all.length === 0) { props.didFileReplicasComDOM.start() } }}
                     id="subpage-file-replica-states"
                 >
                     <PageDIDFilereplicas comdom={props.didFileReplicasComDOM} />
                 </SubPage>
                 <SubPage
                     show={showPageBools["subpage-file-replica-states-d"]()}
-                    run={() => { if (props.didContentsComDOM.query.data.length === 0) { props.didContentsComDOM.start() } }}
+                    run={() => { if (props.didContentsComDOM.query.data.all.length === 0) { props.didContentsComDOM.start() } }}
                     id="subpage-file-replica-states-d"
                 >
                     <PageDIDFilereplicasD
@@ -154,7 +154,7 @@ export const PageDID = (
                 </SubPage>
                 <SubPage
                     show={showPageBools["subpage-parent-dids"]()}
-                    run={() => { if (props.didParentsComDOM.query.data.length === 0) { props.didParentsComDOM.start() } }}
+                    run={() => { if (props.didParentsComDOM.query.data.all.length === 0) { props.didParentsComDOM.start() } }}
                     id="subpage-parent-dids"
                 >
                     <PageDIDByType comdom={props.didParentsComDOM} />
@@ -167,7 +167,7 @@ export const PageDID = (
                 </SubPage>
                 <SubPage
                     show={showPageBools["subpage-contents"]()}
-                    run={() => { if (props.didContentsComDOM.query.data.length === 0) { props.didContentsComDOM.start() } }}
+                    run={() => { if (props.didContentsComDOM.query.data.all.length === 0) { props.didContentsComDOM.start() } }}
                     id="subpage-contents"
                 >
                     <PageDIDByType showDIDType comdom={props.didContentsComDOM} />

src/component-library/Pages/Rule/CreateRule.tsx

@@ -219,11 +219,13 @@ export const CreateRule = (
         // build request for comdom
         const request: HTTPRequest = {
             url: new URL(`${process.env.NEXT_PUBLIC_WEBUI_HOST}/api/feature/list-account-rse-quotas`),
-            method: "GET",
+            method: "POST",
             headers: new Headers({
                 'Content-Type': 'application/json'
             } as HeadersInit),
-            body: null,
+            body: {
+                "requestedDIDs": Page0State.searchDIDSelection,
+            }
         }
         // run query
         await props.rseListComDOM.setRequest(request)

src/component-library/Pages/Rule/CreateRuleRSETable.tsx

@@ -33,7 +33,7 @@ export const CreateRuleRSETable = (
                             type="checkbox"
                             disabled={!info.row.getCanSelect()}
                             checked={info.row.getIsSelected()}
-                            onClick={e => {
+                            onChange={e => {
                                 info.row.toggleSelected()
                             }}
                         />

src/component-library/Pages/Rule/PageRule.tsx

@@ -42,7 +42,7 @@ export const PageRule = (
     const [subpageIndex, setSubpageIndex] = useState(0);
 
     useEffect(() => {
-        if (subpageIndex === 1 && props.ruleLocks.query.data.length === 0) {
+        if (subpageIndex === 1 && props.ruleLocks.query.data.all.length === 0) {
             // Opened locks tab, but no data yet => start load
             console.log(props.ruleLocks.query.data)
             props.ruleLocks.start()

src/component-library/StreamedTables/StreamedTable.tsx

@@ -6,11 +6,13 @@ import { TableHeader } from "./TableHeader";
 import { StreamedTableProps } from "./types";
 import { usePrepareTable } from "./helpers";
 
-export function StreamedTable<T extends BaseViewModel>(props: StreamedTableProps<T>) {
+export function StreamedTable<T>(props: StreamedTableProps<T>) {
     const { className, ...otherprops } = props
 
     const { table, rowSelection, setRowSelection, breakoutVisibility, setBreakoutVisibility } = usePrepareTable<T>({
-        tabledata: props.tablecomdom.query.data || [],
+        tabledata: props.tablecomdom.query.data.all || [],
+        successViewModels: props.tablecomdom.query.data.success,
+        errorViewModels: props.tablecomdom.query.data.error,
         tablecolumns: props.tablecolumns,
         tablestyling: props.tablestyling,
         tableselecting: props.tableselecting,

src/component-library/StreamedTables/TableBody.tsx

@@ -1,4 +1,4 @@
-import { Table, flexRender } from "@tanstack/react-table";
+import { Table, flexRender, TableState } from "@tanstack/react-table";
 import { twMerge } from "tailwind-merge";
 import { TableStyling } from "./types";
 

src/component-library/StreamedTables/TableErrorreader.tsx

@@ -22,7 +22,7 @@ const TableErrorelement: React.FC<JSX.IntrinsicElements["div"] & {message: strin
 }
 
 export const TableErrorreader: React.FC<JSX.IntrinsicElements["div"] & {
-    comdom: UseComDOM<BaseViewModel>,
+    comdom: UseComDOM<any>,
     showDetailedErrors: boolean,
     setShowDetailedErrors: (show: boolean) => void,
 }> = (
@@ -55,7 +55,7 @@ export const TableErrorreader: React.FC<JSX.IntrinsicElements["div"] & {
                     "h-48 overflow-y-scroll"
                 )}
             >
-                {comdom.query.data.filter(vm => vm.status === "error").map((vm, i) => {return (
+                {comdom.query.data.error.filter(vm => vm.status === "error").map((vm, i) => {return (
                     <TableErrorelement
                         key={i}
                         message={vm.message ?? "Unknown Error"}

src/component-library/StreamedTables/TableErrorstatus.tsx

@@ -16,7 +16,7 @@ export const TableErrorstatus: React.FC<JSX.IntrinsicElements["form"] & {
     const { className, ...otherprops } = props
     const [numBadRows, setNumBadRows] = useState<number>(0)
     useEffect(() => {
-        const data = comdom.query.data
+        const data = comdom.query.data.error
         if (data.length === 1 && data[0].status === "error") {
             // currently this takes precedence over the individual errors
             // expectation is that if only one element and it fails, it's a backend error

src/component-library/StreamedTables/TableFooter.tsx

@@ -6,7 +6,6 @@ import { TableFetchstatus } from "./TableFetchstatus";
 import { TableBreakout } from "./TableBreakout";
 import { TableErrorstatus } from "./TableErrorstatus";
 import { TableErrorreader } from "./TableErrorreader";
-import { BaseViewModel } from "@/lib/sdk/view-models";
 import { useState } from "react";
 
 /**
@@ -18,15 +17,15 @@ import { useState } from "react";
  */
 type TableFooterProps<T> = JSX.IntrinsicElements["tfoot"] & {
     table: Table<T>,
-    comdom: UseComDOM<T>
+    comdom: UseComDOM<any> // TODO: fix this any, use BaseViewModel
     breakout?: {
         breakoutVisibility: boolean,
         keys: Record<string, string>,
     }
     stacked?: boolean // to save horizontal space
 }
 
-export function TableFooter<T extends BaseViewModel>(props: TableFooterProps<T>) {
+export function TableFooter<T>(props: TableFooterProps<T>) {
     const { stacked, className, ...otherprops } = props
     const [showDetailedErrors, setShowDetailedErrors] = useState<boolean>(false)
 

src/component-library/StreamedTables/helpers.tsx

@@ -1,6 +1,6 @@
 import { DID } from "@/lib/core/entity/rucio";
 import { TableSelecting, TableStyling } from "./types";
-import { useReactTable, getCoreRowModel, getPaginationRowModel, getFilteredRowModel, getSortedRowModel, RowSelectionState } from "@tanstack/react-table";
+import { useReactTable, getCoreRowModel, getPaginationRowModel, getFilteredRowModel, getSortedRowModel, RowSelectionState, FilterFn } from "@tanstack/react-table";
 import { useEffect, useState } from "react";
 
 export function didToScopename(list: DID[]): string[] {
@@ -9,6 +9,8 @@ export function didToScopename(list: DID[]): string[] {
 
 export function usePrepareTable<T>(props: {
     tabledata: T[]
+    successViewModels?: T[]
+    errorViewModels?: T[]
     tablecolumns: any[]
     tablestyling?: TableStyling
     tableselecting?: TableSelecting<T>

src/component-library/StreamedTables/types.d.ts

@@ -1,3 +1,6 @@
+import { ViewModelTableData } from "@/lib/infrastructure/hooks/useComDOM"
+import { UseComDOM } from "@/lib/infrastructure/hooks/useComDOM"
+
 export type TableStyling = Partial<{
     visibility?: Record<string, boolean>
     tableHeadRowStyle?: string

src/lib/core/entity/rucio.ts

@@ -1,3 +1,5 @@
+import { BaseViewModel } from "@/lib/sdk/view-models";
+
 export type DateISO = string
 
 /**
@@ -289,7 +291,8 @@ export enum DIDType {
 }
 
 // replace this!
-export type RSEOld = {
+export interface RSEOld extends BaseViewModel {
+    status: 'success' | 'error'
     id: string;
     name: string;
     city: string,

src/lib/core/use-case/list-dids/pipeline-element-get-did.ts

@@ -12,8 +12,6 @@ export default class GetDIDsPipelineElement extends BaseStreamingPostProcessingP
     async makeGatewayRequest(requestModel: AuthenticatedRequestModel<ListDIDsRequest>, responseModel: ListDIDsResponse): Promise<DIDExtendedDTO> {
         try {
             const dto: DIDExtendedDTO = await this.didGateway.getDID(requestModel.rucioAuthToken, responseModel.scope, responseModel.name, DIDType.FILE);
-            const didMetaDTO: DIDMetaDTO = await this.didGateway.getDIDMeta(requestModel.rucioAuthToken, responseModel.scope, responseModel.name);
-            dto.did_type = didMetaDTO.did_type;
             return dto;
         } catch (error: any) {
             const errorDTO: DIDExtendedDTO = {

src/lib/infrastructure/hooks/useComDOM.ts

@@ -7,16 +7,28 @@ import ComDOMWrapper, {
 } from '@/lib/infrastructure/web-worker/comdom-wrapper'
 
 import { HTTPRequest } from '@/lib/sdk/http'
+import { BaseViewModel } from '@/lib/sdk/view-models'
 
 export type ComDOMError = {
     id: number
     message: string
     cause: string
 }
 
-export type UseComDOM<TData> = {
-    query: DefinedUseQueryResult<TData[], unknown>
+/**
+ * @description The data used by tables that render ViewModels.
+ */
+export type ViewModelTableData<TData extends BaseViewModel> = {
+    all: TData[]
+    success: TData[]
+    error: TData[]
+}
+
+export type UseComDOM<TData extends BaseViewModel> = {
+    query: DefinedUseQueryResult<ViewModelTableData<TData>, unknown>
     dataSink: React.MutableRefObject<TData[]>
+    successViewModels: React.MutableRefObject<TData[]>
+    errorViewModels: React.MutableRefObject<TData[]>
     status: UseComDOMStatus
     comDOMStatus: ComDOMStatus
     pollInterval: number
@@ -57,6 +69,8 @@ export enum UseComDOMStatus {
  * 
  * @returns query The query object from (tanstack/react-query) returned by the useQuery hook.
  * @returns dataSink A ref that contains the current data sink. This is same as query.data but is a ref instead of a state variable.
+ * @returns successViewModelsDataSink A ref that contains the current success view models. This is same as query.data but is a ref instead of a state variable and only contains view models with status 'success'.
+ * @returns errorViewModelsDataSink A ref that contains the current error view models. This is same as query.data but is a ref instead of a state variable and only contains view models with status 'error'.
  * @returns status {@link UseComDOMStatus}The status of the useComDOM hook, derived from the status of the query and the web worker.
  * @returns comDOMStatus {@link ComDOMStatus} The status of the ComDOM web worker.
  * @returns pollInterval The current poll interval of the query in milliseconds.
@@ -71,7 +85,7 @@ export enum UseComDOMStatus {
  * @returns setRequest A function that sets the current HTTP request object used by the ComDOM web worker.
  */
 
-export default function useComDOM<TData>(
+export default function useComDOM<TData extends BaseViewModel>(
     queryName: string,
     initialData: TData[] = [],
     proactiveRefetch: boolean = false,
@@ -80,6 +94,9 @@ export default function useComDOM<TData>(
     debug: boolean = false,
 ) {
     const dataSink = useRef<TData[]>(initialData)
+    const successViewModelsDataSink = useRef<TData[]>(initialData.filter(viewModel => viewModel.status === 'success') as TData[])
+    const errorViewModelsDataSink = useRef<TData[]>(initialData.filter(viewModel => viewModel.status === 'error') as TData[])
+
     const comDOMWrapper: IComDOMWrapper<TData> = useMemo(() => {
         return new ComDOMWrapper<TData>(debug)
     }, [debug])
@@ -163,8 +180,17 @@ export default function useComDOM<TData>(
                 setStatus(UseComDOMStatus.DONE)
                 return Promise.reject('ComDOM has finished fetching all data')
             }
-            dataSink.current.push(...batchResponse.data)
-            return dataSink.current
+            console.log('Batch Response', batchResponse)
+            const successViewModels = batchResponse.data.filter(viewModel => viewModel.status === 'success')
+            const errorViewModels = batchResponse.data.filter(viewModel => viewModel.status === 'error')
+            dataSink.current.push(...successViewModels)
+            successViewModelsDataSink.current.push(...successViewModels)
+            errorViewModelsDataSink.current.push(...errorViewModels)
+            return {
+                "all": dataSink.current,
+                "success": successViewModelsDataSink.current,
+                "error": errorViewModelsDataSink.current
+            }
         } catch (error: any) {
             reportError(error, 'Error fetching data from background thread')
             setStatus(UseComDOMStatus.ERROR)
@@ -175,7 +201,11 @@ export default function useComDOM<TData>(
     const query = useQuery({
         queryKey: queryKey,
         queryFn: queryFn,
-        initialData: initialData,
+        initialData: { 
+            "all": initialData,
+            "success": initialData.filter(viewModel => viewModel.status === 'success') as TData[],
+            "error": initialData.filter(viewModel => viewModel.status === 'error') as TData[],
+        },
         refetchInterval: pollInterval,
         refetchOnWindowFocus: proactiveRefetch,
         refetchOnMount: proactiveRefetch,
@@ -199,6 +229,8 @@ export default function useComDOM<TData>(
             }
             _log('Resetting data sink')
             dataSink.current = initialData
+            successViewModelsDataSink.current = []
+            errorViewModelsDataSink.current = []
             _log('Starting ComDOM with URL', req?.url.toString())
             if (req == null) {
                 throw new Error('HTTPRequest for streaming is null')
@@ -259,6 +291,8 @@ export default function useComDOM<TData>(
         try {
             queryClient.setQueriesData(queryKey, initialData)
             dataSink.current = initialData
+            successViewModelsDataSink.current = []
+            errorViewModelsDataSink.current = []
             _log('Data sink cleaned')
         } catch (error: any) {
             _log('Error cleaning ComDOM', error)
@@ -270,6 +304,8 @@ export default function useComDOM<TData>(
     return {
         query,
         dataSink,
+        successViewModels: successViewModelsDataSink,
+        errorViewModels: errorViewModelsDataSink,
         status,
         comDOMStatus,
         pollInterval,

src/lib/infrastructure/presenter/list-account-rse-quotas-presenter.ts

@@ -34,6 +34,10 @@ export default class ListAccountRSEQuotasPresenter extends BaseStreamingPresente
         const viewModel: RSEAccountUsageLimitViewModel = {
             ...responseModel,
         };
+        if (viewModel.has_quota && viewModel.bytes_limit === -1) {
+            viewModel.bytes_limit = -1;
+            viewModel.bytes_remaining = -1;
+        }
         return viewModel;
     }
 

src/lib/sdk/gateway-endpoints.ts

@@ -232,16 +232,20 @@ export abstract class BaseEndpoint<TDTO extends BaseDTO> {
         } else {
             try {
                 if(!this.parsedBodyAsText) {
-                    const data = await response.json();
-                    return this.createDTO(data);
+                    const data = await response.text();
+                    console.log("DEBUG: ", data)
+                    // TODO: Handle better: replace Infinity with -1
+                    const infinityPatch = data.replace(/Infinity/g, '-1');
+                    const json = JSON.parse(infinityPatch);
+                    return this.createDTO(json);
                 } else {
                     const text = await response.text();
                     return this.createDTO(text);
                 }
             } catch(error) {
                 return {
                     status: 'error',
-                    errorMessage: `An error occurred while fetching ${this.request.url}. Error: ${error}}`,
+                    errorMessage: `An error occurred while fetching and parsing response from ${this.request.url}. Error: ${error}}`,
                 } as TDTO;
             }
         }

test/api/did/list-dids-stream-error.test.ts

@@ -32,7 +32,7 @@ describe('DID API Tests', () => {
         }
 
         const dataset1StatusEndpoint: MockEndpoint = {
-            url: `${MockRucioServerFactory.RUCIO_HOST}/dids/test/dataset1/status`,
+            url: `${MockRucioServerFactory.RUCIO_HOST}/dids/test/dataset1/status?dynamic_depth=FILE`,
             method: 'GET',
             response: {
                 status: 200,
@@ -54,7 +54,7 @@ describe('DID API Tests', () => {
         }
 
         const dataset2StatusEndpoint: MockEndpoint = {
-            url: `${MockRucioServerFactory.RUCIO_HOST}/dids/test/dataset2/status`,
+            url: `${MockRucioServerFactory.RUCIO_HOST}/dids/test/dataset2/status?dynamic_depth=FILE`,
             method: 'GET',
             response: {
                 status: 200,
@@ -76,7 +76,7 @@ describe('DID API Tests', () => {
         }
 
         const dataset3StatusEndpoint: MockEndpoint = {
-            url: `${MockRucioServerFactory.RUCIO_HOST}/dids/test/dataset3/status`,
+            url: `${MockRucioServerFactory.RUCIO_HOST}/dids/test/dataset3/status?dynamic_depth=FILE`,
             method: 'GET',
             response: {
                 status: 200,