fix: wrong allCollections query pagination and pagetype timeout errors

packages/api/package.json

@@ -26,7 +26,8 @@
     "@sindresorhus/slugify": "^1.1.2",
     "dataloader": "^2.0.0",
     "fast-deep-equal": "^3.1.3",
-    "isomorphic-unfetch": "^3.1.0"
+    "isomorphic-unfetch": "^3.1.0",
+    "p-limit": "^4.0.0"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "2.2.0",

packages/api/src/platforms/vtex/clients/commerce/types/Portal.ts

@@ -1,14 +1,19 @@
-export interface PortalPagetype {
+export type PortalPagetype = ValidPortalPagetype | NotFoundPortalPagetype
+
+export interface ValidPortalPagetype {
   id: number
   name: string
   url: string
   title: string
   metaTagDescription: string
-  pageType:
-    | 'Brand'
-    | 'Category'
-    | 'Department'
-    | 'Subcategory'
-    | 'FullText'
-    | 'NotFound'
+  pageType: 'Brand' | 'Category' | 'Department' | 'Subcategory' | 'FullText'
+}
+
+export interface NotFoundPortalPagetype {
+  id: null
+  name: null
+  url: null
+  title: null
+  metaTagDescription: null
+  pageType: 'NotFound'
 }

packages/api/src/platforms/vtex/loaders/index.ts

@@ -1,15 +1,18 @@
 import { getSimulationLoader } from './simulation'
 import { getSkuLoader } from './sku'
+import { getPagetypeLoader } from './pagetype'
 import type { Context, Options } from '..'
 
 export type Loaders = ReturnType<typeof getLoaders>
 
 export const getLoaders = (options: Options, { clients }: Context) => {
   const skuLoader = getSkuLoader(options, clients)
   const simulationLoader = getSimulationLoader(options, clients)
+  const pagetypeLoader = getPagetypeLoader(options, clients)
 
   return {
     skuLoader,
     simulationLoader,
+    pagetypeLoader,
   }
 }

packages/api/src/platforms/vtex/loaders/pagetype.ts

@@ -0,0 +1,26 @@
+import DataLoader from 'dataloader'
+import pLimit from 'p-limit'
+
+import type { Options } from '..'
+import type { Clients } from '../clients'
+import type { PortalPagetype } from '../clients/commerce/types/Portal'
+
+// Limits concurrent requests to 20 so that they don't timeout
+const CONCURRENT_REQUESTS_MAX = 20
+
+export const getPagetypeLoader = (_: Options, clients: Clients) => {
+  const limit = pLimit(CONCURRENT_REQUESTS_MAX)
+
+  const loader = (slugs: readonly string[]) => {
+    return Promise.all(
+      slugs.map((s: string) =>
+        limit(() => clients.commerce.catalog.portal.pagetype(s))
+      )
+    )
+  }
+
+  return new DataLoader<string, PortalPagetype>(loader, {
+    // DataLoader is being used to cache requests, not to batch them
+    batch: false,
+  })
+}

packages/api/src/platforms/vtex/resolvers/collection.ts

@@ -2,21 +2,21 @@ import { slugify as baseSlugify } from '../utils/slugify'
 import type { Resolver } from '..'
 import type { Brand } from '../clients/commerce/types/Brand'
 import type { CategoryTree } from '../clients/commerce/types/CategoryTree'
-import type { PortalPagetype } from '../clients/commerce/types/Portal'
+import type { ValidPortalPagetype } from '../clients/commerce/types/Portal'
 
-type Root = Brand | (CategoryTree & { level: number }) | PortalPagetype
+type Root = Brand | (CategoryTree & { level: number }) | ValidPortalPagetype
 
 const isBrand = (x: any): x is Brand => x.type === 'brand'
 
-const isPortalPageType = (x: any): x is PortalPagetype =>
-  typeof x.pageType === 'string'
+const isValidPortalPageType = (x: any): x is ValidPortalPagetype =>
+  typeof x.pageType === 'string' && x.pageType !== 'NotFound'
 
 const slugify = (root: Root) => {
   if (isBrand(root)) {
     return baseSlugify(root.name)
   }
 
-  if (isPortalPageType(root)) {
+  if (isValidPortalPageType(root)) {
     return new URL(`https://${root.url}`).pathname.slice(1)
   }
 
@@ -27,7 +27,7 @@ export const StoreCollection: Record<string, Resolver<Root>> = {
   id: ({ id }) => id.toString(),
   slug: (root) => slugify(root),
   seo: (root) =>
-    isBrand(root) || isPortalPageType(root)
+    isBrand(root) || isValidPortalPageType(root)
       ? {
           title: root.title,
           description: root.metaTagDescription,
@@ -39,7 +39,7 @@ export const StoreCollection: Record<string, Resolver<Root>> = {
   type: (root) =>
     isBrand(root)
       ? 'Brand'
-      : isPortalPageType(root)
+      : isValidPortalPageType(root)
       ? root.pageType
       : root.level === 0
       ? 'Department'
@@ -51,7 +51,7 @@ export const StoreCollection: Record<string, Resolver<Root>> = {
         }
       : {
           selectedFacets: new URL(
-            isPortalPageType(root) ? `https://${root.url}` : root.url
+            isValidPortalPageType(root) ? `https://${root.url}` : root.url
           ).pathname
             .slice(1)
             .split('/')
@@ -62,7 +62,7 @@ export const StoreCollection: Record<string, Resolver<Root>> = {
         },
   breadcrumbList: async (root, _, ctx) => {
     const {
-      clients: { commerce },
+      loaders: { pagetypeLoader },
     } = ctx
 
     const slug = slugify(root)
@@ -78,17 +78,17 @@ export const StoreCollection: Record<string, Resolver<Root>> = {
       segments.slice(0, index + 1).join('/')
     )
 
-    const pageTypes = await Promise.all(
-      slugs.map((s) => commerce.catalog.portal.pagetype(s))
+    const validPageTypes = (await pagetypeLoader.loadMany(slugs)).filter(
+      isValidPortalPageType
     )
 
     return {
-      itemListElement: pageTypes.map((pageType, index) => ({
+      itemListElement: validPageTypes.map((pageType, index) => ({
         item: new URL(`https://${pageType.url}`).pathname,
         name: pageType.name,
         position: index + 1,
       })),
-      numberOfItems: pageTypes.length,
+      numberOfItems: validPageTypes.length,
     }
   },
 }

packages/api/src/platforms/vtex/resolvers/query.ts

@@ -12,6 +12,7 @@ import type {
 } from '../../../__generated__/schema'
 import type { CategoryTree } from '../clients/commerce/types/CategoryTree'
 import type { Context } from '../index'
+import type { ValidPortalPagetype } from '../clients/commerce/types/Portal'
 
 export const Query = {
   product: async (_: unknown, { locator }: QueryProductArgs, ctx: Context) => {
@@ -35,15 +36,15 @@ export const Query = {
     ctx: Context
   ) => {
     const {
-      clients: { commerce },
+      loaders: { pagetypeLoader },
     } = ctx
 
-    const result = await commerce.catalog.portal.pagetype(slug)
+    const result = await pagetypeLoader.load(slug)
 
     const whitelist = ['Brand', 'Category', 'Department', 'Subcategory']
 
     if (whitelist.includes(result.pageType)) {
-      return result
+      return result as ValidPortalPagetype
     }
 
     throw new NotFoundError(`Not Found: ${slug}`)
@@ -100,6 +101,7 @@ export const Query = {
         endCursor: products.total.toString(),
         totalCount: products.total,
       },
+      // after + index is bigger than after+first itself because of the array flat() above
       edges: skus.map((sku, index) => ({
         node: sku,
         cursor: (after + index).toString(),
@@ -140,20 +142,24 @@ export const Query = {
       ...categories,
     ]
 
+    const validCollections = collections
+      // Nullable slugs may cause one route to override the other
+      .filter((node) => Boolean(StoreCollection.slug(node, null, ctx, null)))
+
     return {
       pageInfo: {
-        hasNextPage: false,
-        hasPreviousPage: false,
+        hasNextPage: validCollections.length - after > first,
+        hasPreviousPage: after > 0,
         startCursor: '0',
-        endCursor: '0',
+        endCursor: (
+          Math.min(first, validCollections.length - after) - 1
+        ).toString(),
       },
-      edges: collections
-        // Nullable slugs may cause one route to override the other
-        .filter((node) => Boolean(StoreCollection.slug(node, null, ctx, null)))
-        .slice(after, first)
+      edges: validCollections
+        .slice(after, after + first)
         .map((node, index) => ({
           node,
-          cursor: index.toString(),
+          cursor: (after + index).toString(),
         })),
     }
   },

packages/gatsby-source-store/src/paginate.ts

@@ -22,11 +22,14 @@ export const RelayForward = (
     }
   },
   next(state, page) {
-    const tail = page.edges[page.edges.length - 1]
-    const first =
-      Number(state.variables.first) ?? Math.min(DEFAULT_PAGE_SIZE, maxItems)
+    const { first: rawFirst, after: maybeStateAfter } = state.variables
 
-    const after = tail?.cursor
+    const stateAfter = Number.isNaN(Number(maybeStateAfter))
+      ? 0
+      : Number(maybeStateAfter)
+
+    const first = Number(rawFirst) ?? DEFAULT_PAGE_SIZE
+    const after = (stateAfter + first).toString()
 
     return {
       variables: { first, after },

yarn.lock

@@ -12011,6 +12011,21 @@ chokidar@^3.5.2:
   optionalDependencies:
     fsevents "~2.3.2"
 
+chokidar@^3.5.3:
+  version "3.5.3"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.5.3.tgz#1cf37c8707b932bd1af1ae22c0432e2acd1903bd"
+  integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==
+  dependencies:
+    anymatch "~3.1.2"
+    braces "~3.0.2"
+    glob-parent "~5.1.2"
+    is-binary-path "~2.1.0"
+    is-glob "~4.0.1"
+    normalize-path "~3.0.0"
+    readdirp "~3.6.0"
+  optionalDependencies:
+    fsevents "~2.3.2"
+
 chownr@^1.1.1, chownr@^1.1.2, chownr@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
@@ -23166,6 +23181,13 @@ p-limit@^2.0.0, p-limit@^2.2.0:
   dependencies:
     p-try "^2.0.0"
 
+p-limit@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-4.0.0.tgz#914af6544ed32bfa54670b061cafcbd04984b644"
+  integrity sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==
+  dependencies:
+    yocto-queue "^1.0.0"
+
 p-locate@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43"
@@ -30191,10 +30213,8 @@ watchpack@^1.7.4:
   resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.7.5.tgz#1267e6c55e0b9b5be44c2023aed5437a2c26c453"
   integrity sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==
   dependencies:
-    chokidar "^3.4.1"
     graceful-fs "^4.1.2"
     neo-async "^2.5.0"
-    watchpack-chokidar2 "^2.0.1"
   optionalDependencies:
     chokidar "^3.4.1"
     watchpack-chokidar2 "^2.0.1"
@@ -30856,7 +30876,12 @@ ws@^7.3.0, ws@^7.3.1, ws@~7.4.2:
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.5.tgz#a484dd851e9beb6fdb420027e3885e8ce48986c1"
   integrity sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==
 
-ws@^8.1.0, ws@^8.2.3:
+ws@^8.1.0:
+  version "8.5.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.5.0.tgz#bfb4be96600757fe5382de12c670dab984a1ed4f"
+  integrity sha512-BWX0SWVgLPzYwF8lTzEy1egjhS4S4OEAHfsO8o65WOVsrnSRGaSiUaa9e0ggGlkMTtBlmOpEXiie9RUcBO86qg==
+
+ws@^8.2.3:
   version "8.3.0"
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.3.0.tgz#7185e252c8973a60d57170175ff55fdbd116070d"
   integrity sha512-Gs5EZtpqZzLvmIM59w4igITU57lrtYVFneaa434VROv4thzJyV6UjIL3D42lslWlI+D4KzLYnxSwtfuiO79sNw==
@@ -31069,6 +31094,11 @@ yocto-queue@^0.1.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
 
+yocto-queue@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
+  integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
+
 yoga-layout-prebuilt@^1.9.6:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/yoga-layout-prebuilt/-/yoga-layout-prebuilt-1.10.0.tgz#2936fbaf4b3628ee0b3e3b1df44936d6c146faa6"