Client side pagination gap detection

__mocks__/react-native.ts

@@ -98,7 +98,10 @@ jest.doMock('react-native', () => {
             // so it seems easier to just run the callback immediately in tests.
             InteractionManager: {
                 ...ReactNative.InteractionManager,
-                runAfterInteractions: (callback: () => void) => callback(),
+                runAfterInteractions: (callback: () => void) => {
+                    callback();
+                    return {cancel: () => {}};
+                },
             },
         },
         ReactNative,

package.json

@@ -291,8 +291,8 @@
     "time-analytics-webpack-plugin": "^0.1.17",
     "ts-jest": "^29.1.2",
     "ts-node": "^10.9.2",
-    "type-fest": "^4.10.2",
-    "typescript": "^5.3.2",
+    "type-fest": "4.20.0",
+    "typescript": "^5.4.5",
     "wait-port": "^0.2.9",
     "webpack": "^5.76.0",
     "webpack-bundle-analyzer": "^4.5.0",

src/CONST.ts

@@ -4814,6 +4814,9 @@ const CONST = {
         DESC: 'desc',
     },
 
+    PAGINATION_START_ID: '-1',
+    PAGINATION_END_ID: '-2',
+
     SUBSCRIPTION_SIZE_LIMIT: 20000,
     SUBSCRIPTION_POSSIBLE_COST_SAVINGS: {
         COLLECT_PLAN: 10,

src/ONYXKEYS.ts

@@ -1,4 +1,4 @@
-import type {ValueOf} from 'type-fest';
+import type {ConditionalKeys, ValueOf} from 'type-fest';
 import type CONST from './CONST';
 import type * as FormTypes from './types/form';
 import type * as OnyxTypes from './types/onyx';
@@ -353,6 +353,7 @@ const ONYXKEYS = {
         REPORT_METADATA: 'reportMetadata_',
         REPORT_ACTIONS: 'reportActions_',
         REPORT_ACTIONS_DRAFTS: 'reportActionsDrafts_',
+        REPORT_ACTIONS_PAGES: 'reportActionsPages_',
         REPORT_ACTIONS_REACTIONS: 'reportActionsReactions_',
         REPORT_DRAFT_COMMENT: 'reportDraftComment_',
         REPORT_IS_COMPOSER_FULL_SIZE: 'reportIsComposerFullSize_',
@@ -569,6 +570,7 @@ type OnyxCollectionValuesMapping = {
     [ONYXKEYS.COLLECTION.REPORT_METADATA]: OnyxTypes.ReportMetadata;
     [ONYXKEYS.COLLECTION.REPORT_ACTIONS]: OnyxTypes.ReportActions;
     [ONYXKEYS.COLLECTION.REPORT_ACTIONS_DRAFTS]: OnyxTypes.ReportActionsDrafts;
+    [ONYXKEYS.COLLECTION.REPORT_ACTIONS_PAGES]: OnyxTypes.Pages;
     [ONYXKEYS.COLLECTION.REPORT_ACTIONS_REACTIONS]: OnyxTypes.ReportActionReactions;
     [ONYXKEYS.COLLECTION.REPORT_DRAFT_COMMENT]: string;
     [ONYXKEYS.COLLECTION.REPORT_IS_COMPOSER_FULL_SIZE]: boolean;
@@ -709,11 +711,12 @@ type OnyxFormDraftKey = keyof OnyxFormDraftValuesMapping;
 type OnyxValueKey = keyof OnyxValuesMapping;
 
 type OnyxKey = OnyxValueKey | OnyxCollectionKey | OnyxFormKey | OnyxFormDraftKey;
+type OnyxPagesKey = ConditionalKeys<OnyxValues, OnyxTypes.Pages>;
 
 type MissingOnyxKeysError = `Error: Types don't match, OnyxKey type is missing: ${Exclude<AllOnyxKeys, OnyxKey>}`;
 /** If this type errors, it means that the `OnyxKey` type is missing some keys. */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 type AssertOnyxKeys = AssertTypesEqual<AllOnyxKeys, OnyxKey, MissingOnyxKeysError>;
 
 export default ONYXKEYS;
-export type {OnyxCollectionKey, OnyxCollectionValuesMapping, OnyxFormDraftKey, OnyxFormKey, OnyxFormValuesMapping, OnyxKey, OnyxValueKey, OnyxValues};
+export type {OnyxCollectionKey, OnyxCollectionValuesMapping, OnyxFormDraftKey, OnyxFormKey, OnyxFormValuesMapping, OnyxKey, OnyxPagesKey, OnyxValueKey, OnyxValues};

src/libs/API/index.ts

@@ -1,5 +1,6 @@
 import type {OnyxUpdate} from 'react-native-onyx';
 import Onyx from 'react-native-onyx';
+import type {SetRequired} from 'type-fest';
 import Log from '@libs/Log';
 import * as Middleware from '@libs/Middleware';
 import * as SequentialQueue from '@libs/Network/SequentialQueue';
@@ -8,9 +9,9 @@ import * as Request from '@libs/Request';
 import * as PersistedRequests from '@userActions/PersistedRequests';
 import CONST from '@src/CONST';
 import type OnyxRequest from '@src/types/onyx/Request';
+import type {PaginatedRequest, PaginationConfig} from '@src/types/onyx/Request';
 import type Response from '@src/types/onyx/Response';
-import pkg from '../../../package.json';
-import type {ApiRequest, ApiRequestCommandParameters, ReadCommand, SideEffectRequestCommand, WriteCommand} from './types';
+import type {ApiCommand, ApiRequestCommandParameters, ApiRequestType, CommandOfType, ReadCommand, SideEffectRequestCommand, WriteCommand} from './types';
 
 // Setup API middlewares. Each request made will pass through a series of middleware functions that will get called in sequence (each one passing the result of the previous to the next).
 // Note: The ordering here is intentional as we want to Log, Recheck Connection, Reauthenticate, and Save the Response in Onyx. Errors thrown in one middleware will bubble to the next.
@@ -28,6 +29,8 @@ Request.use(Middleware.Reauthentication);
 // If an optimistic ID is not used by the server, this will update the remaining serialized requests using that optimistic ID to use the correct ID instead.
 Request.use(Middleware.HandleUnusedOptimisticID);
 
+Request.use(Middleware.Pagination);
+
 // SaveResponseInOnyx - Merges either the successData or failureData (or finallyData, if included in place of the former two values) into Onyx depending on if the call was successful or not. This needs to be the LAST middleware we use, don't add any
 // middlewares after this, because the SequentialQueue depends on the result of this middleware to pause the queue (if needed) to bring the app to an up-to-date state.
 Request.use(Middleware.SaveResponseInOnyx);
@@ -40,54 +43,83 @@ type OnyxData = {
 };
 
 /**
- * All calls to API.write() will be persisted to disk as JSON with the params, successData, and failureData (or finallyData, if included in place of the former two values).
- * This is so that if the network is unavailable or the app is closed, we can send the WRITE request later.
- *
- * @param command - Name of API command to call.
- * @param apiCommandParameters - Parameters to send to the API.
- * @param onyxData  - Object containing errors, loading states, and optimistic UI data that will be merged
- *                             into Onyx before and after a request is made. Each nested object will be formatted in
- *                             the same way as an API response.
- * @param [onyxData.optimisticData] - Onyx instructions that will be passed to Onyx.update() before the request is made.
- * @param [onyxData.successData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200.
- * @param [onyxData.failureData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode !== 200.
- * @param [onyxData.finallyData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200 or jsonCode !== 200.
+ * Prepare the request to be sent. Bind data together with request metadata and apply optimistic Onyx data.
  */
-function write<TCommand extends WriteCommand>(command: TCommand, apiCommandParameters: ApiRequestCommandParameters[TCommand], onyxData: OnyxData = {}) {
-    Log.info('Called API write', false, {command, ...apiCommandParameters});
-    const {optimisticData, ...onyxDataWithoutOptimisticData} = onyxData;
+function prepareRequest<TCommand extends ApiCommand>(command: TCommand, type: ApiRequestType, params: ApiRequestCommandParameters[TCommand], onyxData: OnyxData = {}): OnyxRequest {
+    Log.info('[API] Preparing request', false, {command, type});
 
-    // Optimistically update Onyx
+    const {optimisticData, ...onyxDataWithoutOptimisticData} = onyxData;
     if (optimisticData) {
+        Log.info('[API] Applying optimistic data', false, {command, type});
         Onyx.update(optimisticData);
     }
 
-    // Assemble the data we'll send to the API
+    const isWriteRequest = type === CONST.API_REQUEST_TYPE.WRITE;
+
+    // Prepare the data we'll send to the API
     const data = {
-        ...apiCommandParameters,
-        appversion: pkg.version,
-        apiRequestType: CONST.API_REQUEST_TYPE.WRITE,
+        ...params,
+        apiRequestType: type,
 
         // We send the pusherSocketID with all write requests so that the api can include it in push events to prevent Pusher from sending the events to the requesting client. The push event
         // is sent back to the requesting client in the response data instead, which prevents a replay effect in the UI. See https://github.com/Expensify/App/issues/12775.
-        pusherSocketID: Pusher.getPusherSocketID(),
+        pusherSocketID: isWriteRequest ? Pusher.getPusherSocketID() : undefined,
     };
 
-    // Assemble all the request data we'll be storing in the queue
-    const request: OnyxRequest = {
+    // Assemble all request metadata (used by middlewares, and for persisted requests stored in Onyx)
+    const request: SetRequired<OnyxRequest, 'data'> = {
         command,
-        data: {
-            ...data,
-
-            // This should be removed once we are no longer using deprecatedAPI https://github.com/Expensify/Expensify/issues/215650
-            shouldRetry: true,
-            canCancel: true,
-        },
+        data,
         ...onyxDataWithoutOptimisticData,
     };
 
+    // This should be removed once we are no longer using deprecatedAPI https://github.com/Expensify/Expensify/issues/215650
+    if (isWriteRequest) {
+        request.data.shouldRetry = true;
+        request.data.canCancel = true;
+    }
+
+    return request;
+}
+
+/**
+ * Process a prepared request according to its type.
+ */
+function processRequest(request: OnyxRequest, type: ApiRequestType): Promise<void | Response> {
     // Write commands can be saved and retried, so push it to the SequentialQueue
-    SequentialQueue.push(request);
+    if (type === CONST.API_REQUEST_TYPE.WRITE) {
+        SequentialQueue.push(request);
+        return Promise.resolve();
+    }
+
+    // Read requests are processed right away, but don't return the response to the caller
+    if (type === CONST.API_REQUEST_TYPE.READ) {
+        Request.processWithMiddleware(request);
+        return Promise.resolve();
+    }
+
+    // Requests with side effects process right away, and return the response to the caller
+    return Request.processWithMiddleware(request);
+}
+
+/**
+ * All calls to API.write() will be persisted to disk as JSON with the params, successData, and failureData (or finallyData, if included in place of the former two values).
+ * This is so that if the network is unavailable or the app is closed, we can send the WRITE request later.
+ *
+ * @param command - Name of API command to call.
+ * @param apiCommandParameters - Parameters to send to the API.
+ * @param onyxData  - Object containing errors, loading states, and optimistic UI data that will be merged
+ *                             into Onyx before and after a request is made. Each nested object will be formatted in
+ *                             the same way as an API response.
+ * @param [onyxData.optimisticData] - Onyx instructions that will be passed to Onyx.update() before the request is made.
+ * @param [onyxData.successData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200.
+ * @param [onyxData.failureData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode !== 200.
+ * @param [onyxData.finallyData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200 or jsonCode !== 200.
+ */
+function write<TCommand extends WriteCommand>(command: TCommand, apiCommandParameters: ApiRequestCommandParameters[TCommand], onyxData: OnyxData = {}): void {
+    Log.info('[API] Called API write', false, {command, ...apiCommandParameters});
+    const request = prepareRequest(command, CONST.API_REQUEST_TYPE.WRITE, apiCommandParameters, onyxData);
+    processRequest(request, CONST.API_REQUEST_TYPE.WRITE);
 }
 
 /**
@@ -111,36 +143,28 @@ function write<TCommand extends WriteCommand>(command: TCommand, apiCommandParam
  *                                    response back to the caller or to trigger reconnection callbacks when re-authentication is required.
  * @returns
  */
-function makeRequestWithSideEffects<TCommand extends SideEffectRequestCommand | WriteCommand | ReadCommand>(
+function makeRequestWithSideEffects<TCommand extends SideEffectRequestCommand>(
     command: TCommand,
     apiCommandParameters: ApiRequestCommandParameters[TCommand],
     onyxData: OnyxData = {},
-    apiRequestType: ApiRequest = CONST.API_REQUEST_TYPE.MAKE_REQUEST_WITH_SIDE_EFFECTS,
 ): Promise<void | Response> {
-    Log.info('Called API makeRequestWithSideEffects', false, {command, ...apiCommandParameters});
-    const {optimisticData, ...onyxDataWithoutOptimisticData} = onyxData;
-
-    // Optimistically update Onyx
-    if (optimisticData) {
-        Onyx.update(optimisticData);
-    }
-
-    // Assemble the data we'll send to the API
-    const data = {
-        ...apiCommandParameters,
-        appversion: pkg.version,
-        apiRequestType,
-    };
-
-    // Assemble all the request data we'll be storing
-    const request: OnyxRequest = {
-        command,
-        data,
-        ...onyxDataWithoutOptimisticData,
-    };
+    Log.info('[API] Called API makeRequestWithSideEffects', false, {command, ...apiCommandParameters});
+    const request = prepareRequest(command, CONST.API_REQUEST_TYPE.MAKE_REQUEST_WITH_SIDE_EFFECTS, apiCommandParameters, onyxData);
 
     // Return a promise containing the response from HTTPS
-    return Request.processWithMiddleware(request);
+    return processRequest(request, CONST.API_REQUEST_TYPE.WRITE);
+}
+
+/**
+ * Ensure all write requests on the sequential queue have finished responding before running read requests.
+ * Responses from read requests can overwrite the optimistic data inserted by
+ * write requests that use the same Onyx keys and haven't responded yet.
+ */
+function validateReadyToRead<TCommand extends ReadCommand>(command: TCommand) {
+    if (PersistedRequests.getLength() > 0) {
+        Log.info(`[API] '${command}' is waiting on ${PersistedRequests.getLength()} write commands`);
+    }
+    return SequentialQueue.waitForIdle();
 }
 
 /**
@@ -156,14 +180,48 @@ function makeRequestWithSideEffects<TCommand extends SideEffectRequestCommand |
  * @param [onyxData.failureData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode !== 200.
  * @param [onyxData.finallyData] - Onyx instructions that will be passed to Onyx.update() when the response has jsonCode === 200 or jsonCode !== 200.
  */
-function read<TCommand extends ReadCommand>(command: TCommand, apiCommandParameters: ApiRequestCommandParameters[TCommand], onyxData: OnyxData = {}) {
-    // Ensure all write requests on the sequential queue have finished responding before running read requests.
-    // Responses from read requests can overwrite the optimistic data inserted by
-    // write requests that use the same Onyx keys and haven't responded yet.
-    if (PersistedRequests.getLength() > 0) {
-        Log.info(`[API] '${command}' is waiting on ${PersistedRequests.getLength()} write commands`);
+function read<TCommand extends ReadCommand>(command: TCommand, apiCommandParameters: ApiRequestCommandParameters[TCommand], onyxData: OnyxData = {}): void {
+    Log.info('[API] Called API.read', false, {command, ...apiCommandParameters});
+
+    validateReadyToRead(command).then(() => {
+        const request = prepareRequest(command, CONST.API_REQUEST_TYPE.READ, apiCommandParameters, onyxData);
+        processRequest(request, CONST.API_REQUEST_TYPE.READ);
+    });
+}
+
+function paginate<TRequestType extends ApiRequestType, TCommand extends CommandOfType<TRequestType>>(
+    type: TRequestType,
+    command: TCommand,
+    apiCommandParameters: ApiRequestCommandParameters[TCommand],
+    onyxData: OnyxData,
+    config: PaginationConfig,
+): TRequestType extends typeof CONST.API_REQUEST_TYPE.MAKE_REQUEST_WITH_SIDE_EFFECTS ? Promise<Response | void> : void;
+function paginate<TRequestType extends ApiRequestType, TCommand extends CommandOfType<TRequestType>>(
+    type: TRequestType,
+    command: TCommand,
+    apiCommandParameters: ApiRequestCommandParameters[TCommand],
+    onyxData: OnyxData,
+    config: PaginationConfig,
+): Promise<Response | void> | void {
+    Log.info('[API] Called API.paginate', false, {command, ...apiCommandParameters});
+    const request: PaginatedRequest = {
+        ...prepareRequest(command, type, apiCommandParameters, onyxData),
+        ...config,
+        ...{
+            isPaginated: true,
+        },
+    };
+
+    if (type === CONST.API_REQUEST_TYPE.WRITE) {
+        processRequest(request, type);
+        return;
     }
-    SequentialQueue.waitForIdle().then(() => makeRequestWithSideEffects(command, apiCommandParameters, onyxData, CONST.API_REQUEST_TYPE.READ));
+
+    if (type === CONST.API_REQUEST_TYPE.MAKE_REQUEST_WITH_SIDE_EFFECTS) {
+        return processRequest(request, type);
+    }
+
+    validateReadyToRead(command as ReadCommand).then(() => processRequest(request, type));
 }
 
-export {write, makeRequestWithSideEffects, read};
+export {write, makeRequestWithSideEffects, read, paginate};

src/libs/API/types.ts

@@ -5,7 +5,7 @@ import type * as Parameters from './parameters';
 import type SignInUserParams from './parameters/SignInUserParams';
 import type UpdateBeneficialOwnersForBankAccountParams from './parameters/UpdateBeneficialOwnersForBankAccountParams';
 
-type ApiRequest = ValueOf<typeof CONST.API_REQUEST_TYPE>;
+type ApiRequestType = ValueOf<typeof CONST.API_REQUEST_TYPE>;
 
 const WRITE_COMMANDS = {
     SET_WORKSPACE_AUTO_REPORTING: 'SetWorkspaceAutoReporting',
@@ -566,4 +566,11 @@ type ApiRequestCommandParameters = WriteCommandParameters & ReadCommandParameter
 
 export {WRITE_COMMANDS, READ_COMMANDS, SIDE_EFFECT_REQUEST_COMMANDS};
 
-export type {ApiRequest, ApiRequestCommandParameters, WriteCommand, ReadCommand, SideEffectRequestCommand};
+type ApiCommand = WriteCommand | ReadCommand | SideEffectRequestCommand;
+type CommandOfType<TRequestType extends ApiRequestType> = TRequestType extends typeof CONST.API_REQUEST_TYPE.WRITE
+    ? WriteCommand
+    : TRequestType extends typeof CONST.API_REQUEST_TYPE.READ
+    ? ReadCommand
+    : SideEffectRequestCommand;
+
+export type {ApiCommand, ApiRequestType, ApiRequestCommandParameters, CommandOfType, WriteCommand, ReadCommand, SideEffectRequestCommand};