diff --git a/src/app.tsx b/src/app.tsx index 670543a..9d7925a 100644 --- a/src/app.tsx +++ b/src/app.tsx @@ -4,6 +4,7 @@ import type { StacCollection, StacItem } from "stac-ts"; import Map from "./components/map"; import Overlay from "./components/overlay"; import { Toaster } from "./components/ui/toaster"; +import useStacChildren from "./hooks/stac-children"; import useStacValue from "./hooks/stac-value"; import type { BBox2D, Color } from "./types/map"; import type { DatetimeBounds, StacValue } from "./types/stac"; @@ -17,6 +18,7 @@ export default function App() { // State const [href, setHref] = useState(getInitialHref()); const fileUpload = useFileUpload({ maxFiles: 1 }); + const [userCollections, setCollections] = useState(); const [userItems, setItems] = useState(); const [picked, setPicked] = useState(); const [bbox, setBbox] = useState(); @@ -28,8 +30,6 @@ export default function App() { const { value, error, - collections, - catalogs, items: linkedItems, table, stacGeoparquetItem, @@ -39,6 +39,12 @@ export default function App() { datetimeBounds: filter ? datetimeBounds : undefined, stacGeoparquetItemId, }); + const collectionsLink = value?.links?.find((link) => link.rel === "data"); + const { catalogs, collections: linkedCollections } = useStacChildren({ + value, + enabled: !!value && !collectionsLink, + }); + const collections = collectionsLink ? userCollections : linkedCollections; const items = userItems || linkedItems; const filteredCollections = useMemo(() => { if (filter && collections) { @@ -161,6 +167,7 @@ export default function App() { value={value} error={error} catalogs={catalogs} + setCollections={setCollections} collections={collections} filteredCollections={filteredCollections} filter={filter} diff --git a/src/components/overlay.tsx b/src/components/overlay.tsx index 0d327c4..504ef6c 100644 --- a/src/components/overlay.tsx +++ b/src/components/overlay.tsx @@ -26,6 +26,7 @@ export default function Overlay({ value, error, catalogs, + setCollections, collections, filteredCollections, filter, @@ -43,6 +44,7 @@ export default function Overlay({ error: Error | undefined; value: StacValue | undefined; catalogs: StacCatalog[] | undefined; + setCollections: (collections: StacCollection[] | undefined) => void; collections: StacCollection[] | undefined; filteredCollections: StacCollection[] | undefined; fileUpload: UseFileUploadReturn; @@ -86,6 +88,7 @@ export default function Overlay({ value={picked || value} error={error} catalogs={catalogs} + setCollections={setCollections} collections={collections} filteredCollections={filteredCollections} fileUpload={fileUpload} diff --git a/src/components/panel.tsx b/src/components/panel.tsx index f0d203c..a7a4038 100644 --- a/src/components/panel.tsx +++ b/src/components/panel.tsx @@ -9,19 +9,27 @@ import { LuFolderSearch, LuLink, LuList, + LuPause, + LuPlay, LuSearch, + LuStepForward, } from "react-icons/lu"; import { Accordion, Alert, Box, + Button, + ButtonGroup, + Card, + Heading, HStack, Icon, SkeletonText, + Stack, type UseFileUploadReturn, } from "@chakra-ui/react"; -import { useQuery } from "@tanstack/react-query"; -import type { StacCatalog, StacCollection, StacItem } from "stac-ts"; +import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; +import type { StacCatalog, StacCollection, StacItem, StacLink } from "stac-ts"; import Assets from "./assets"; import Catalogs from "./catalogs"; import CollectionSearch from "./collection-search"; @@ -37,6 +45,7 @@ import type { BBox2D } from "../types/map"; import type { DatetimeBounds, StacAssets, + StacCollections, StacSearch, StacValue, } from "../types/stac"; @@ -46,6 +55,7 @@ export default function Panel({ value, error, catalogs, + setCollections, collections, filteredCollections, items, @@ -62,6 +72,7 @@ export default function Panel({ value: StacValue | undefined; error: Error | undefined; catalogs: StacCatalog[] | undefined; + setCollections: (collections: StacCollection[] | undefined) => void; collections: StacCollection[] | undefined; filteredCollections: StacCollection[] | undefined; items: StacItem[] | undefined; @@ -76,15 +87,8 @@ export default function Panel({ setDatetimeBounds: (bounds: DatetimeBounds | undefined) => void; }) { const [search, setSearch] = useState(); - const rootHref = value?.links?.find((link) => link.rel === "root")?.href; - const rootData = useQuery({ - queryKey: ["stac-value", rootHref], - enabled: !!rootHref, - queryFn: () => fetchStac(rootHref), - }); - const searchLinks = rootData.data?.links?.filter( - (link) => link.rel === "search" - ); + const [numberOfCollections, setNumberOfCollections] = useState(); + const [fetchAllCollections, setFetchAllCollections] = useState(false); const { links, assets, properties } = useMemo(() => { if (value) { if (value.type === "Feature") { @@ -101,21 +105,95 @@ export default function Panel({ return { links: undefined, assets: undefined, properties: undefined }; } }, [value]); + const { rootLink, collectionsLink, nextLink, prevLink, filteredLinks } = + useMemo(() => { + let rootLink: StacLink | undefined = undefined; + let collectionsLink: StacLink | undefined = undefined; + let nextLink: StacLink | undefined = undefined; + let prevLink: StacLink | undefined = undefined; + const filteredLinks = []; + if (links) { + for (const link of links) { + switch (link.rel) { + case "root": + rootLink = link; + break; + case "data": + collectionsLink = link; + break; + case "next": + nextLink = link; + break; + case "previous": + prevLink = link; + break; + } + // We already show children and items in their own pane + if (link.rel !== "child" && link.rel !== "item") + filteredLinks.push(link); + } + } + return { rootLink, collectionsLink, nextLink, prevLink, filteredLinks }; + }, [links]); + const rootData = useQuery({ + queryKey: ["stac-value", rootLink?.href], + enabled: !!rootLink, + queryFn: () => rootLink && fetchStac(rootLink.href), + }); + const searchLinks = useMemo(() => { + return rootData.data?.links?.filter((link) => link.rel === "search"); + }, [rootData.data]); + const collectionsResult = useInfiniteQuery({ + queryKey: ["stac-collections", collectionsLink?.href], + queryFn: async ({ pageParam }) => { + if (pageParam) { + return await fetch(pageParam).then((response) => { + if (response.ok) return response.json(); + else + throw new Error( + `Error while fetching collections from ${pageParam}` + ); + }); + } else { + return null; + } + }, + initialPageParam: collectionsLink?.href, + getNextPageParam: (lastPage: StacCollections | null) => + lastPage?.links?.find((link) => link.rel == "next")?.href, + enabled: !!collectionsLink, + }); + useEffect(() => { + setCollections( + collectionsResult.data?.pages.flatMap((page) => page?.collections || []) + ); + if (collectionsResult.data?.pages.at(0)?.numberMatched) + setNumberOfCollections(collectionsResult.data?.pages[0]?.numberMatched); + }, [collectionsResult.data, setCollections]); + useEffect(() => { + if ( + fetchAllCollections && + !collectionsResult.isFetching && + collectionsResult.hasNextPage + ) + collectionsResult.fetchNextPage(); + }, [fetchAllCollections, collectionsResult]); + useEffect(() => { + setFetchAllCollections(false); + setNumberOfCollections(undefined); + }, [value]); // Handled by the value if (properties?.description) delete properties["description"]; - const thumbnailAsset = - assets && - ((Object.keys(assets).includes("thumbnail") && assets["thumbnail"]) || - Object.values(assets).find((asset) => - asset.roles?.includes("thumbnail") - )); - const nextLink = links?.find((link) => link.rel === "next"); - const prevLink = links?.find((link) => link.rel === "previous"); - // We already provide linked children and items in their own pane. - const filteredLinks = links?.filter( - (link) => link.rel !== "child" && link.rel !== "item" - ); + const thumbnailAsset = useMemo(() => { + return ( + assets && + ((Object.keys(assets).includes("thumbnail") && assets["thumbnail"]) || + Object.values(assets).find((asset) => + asset.roles?.includes("thumbnail") + )) + ); + }, [assets]); useEffect(() => { setItems(undefined); @@ -123,28 +201,69 @@ export default function Panel({ return ( - {(href && value && ( - - )) || - (error && ( - - - - Error while fetching STAC value - {error.toString()} - - + + {(href && value && ( + )) || - (href && ) || ( - + (error && ( + + + + Error while fetching STAC value + {error.toString()} + + + )) || + (href && ) || ( + + )} + + {collectionsResult.hasNextPage && ( + + + Collection pagination + + + + + + + + )} + {value && ( @@ -164,7 +283,7 @@ export default function Panel({ <> Collections{" "} {(filteredCollections && - `(${filteredCollections?.length}/${collections.length})`) || + `(${filteredCollections?.length}/${numberOfCollections || collections.length})`) || `(${collections.length})`} } diff --git a/src/hooks/stac-children.ts b/src/hooks/stac-children.ts new file mode 100644 index 0000000..2ddc222 --- /dev/null +++ b/src/hooks/stac-children.ts @@ -0,0 +1,49 @@ +import { useMemo } from "react"; +import { useQueries } from "@tanstack/react-query"; +import type { StacValue } from "../types/stac"; +import { getStacJsonValue } from "../utils/stac"; + +export default function useStacChildren({ + value, + enabled, +}: { + value: StacValue | undefined; + enabled: boolean; +}) { + const results = useQueries({ + queries: + value?.links + ?.filter((link) => link.rel === "child") + .map((link) => { + return { + queryKey: ["stac-value", link.href], + queryFn: () => getStacJsonValue(link.href), + enabled: enabled, + }; + }) || [], + combine: (results) => { + return { + data: results.map((result) => result.data), + }; + }, + }); + + return useMemo(() => { + const collections = []; + const catalogs = []; + for (const value of results.data) { + switch (value?.type) { + case "Catalog": + catalogs.push(value); + break; + case "Collection": + collections.push(value); + break; + } + } + return { + collections: collections.length > 0 ? collections : undefined, + catalogs: catalogs.length > 0 ? catalogs : undefined, + }; + }, [results.data]); +} diff --git a/src/hooks/stac-value.ts b/src/hooks/stac-value.ts index 97d4e11..d26295e 100644 --- a/src/hooks/stac-value.ts +++ b/src/hooks/stac-value.ts @@ -1,9 +1,10 @@ -import { useEffect, useMemo, useState } from "react"; +import { useEffect, useState } from "react"; import type { UseFileUploadReturn } from "@chakra-ui/react"; import { AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; -import { useInfiniteQuery, useQueries, useQuery } from "@tanstack/react-query"; +import { useQueries, useQuery } from "@tanstack/react-query"; +import type { StacItem } from "stac-ts"; import { useDuckDb } from "duckdb-wasm-kit"; -import type { DatetimeBounds, StacCollections, StacValue } from "../types/stac"; +import type { DatetimeBounds, StacValue } from "../types/stac"; import { getStacJsonValue } from "../utils/stac"; import { getStacGeoparquet, @@ -24,6 +25,8 @@ export default function useStacValue({ }) { const { db } = useDuckDb(); const [connection, setConnection] = useState(); + const enableStacGeoparquet = + (connection && href && href.endsWith(".parquet")) || false; useEffect(() => { if (db && href?.endsWith(".parquet")) { @@ -41,9 +44,6 @@ export default function useStacValue({ } }, [db, href, fileUpload]); - const enableStacGeoparquet = - (connection && href && href.endsWith(".parquet")) || false; - const jsonResult = useQuery({ queryKey: ["stac-value", href], queryFn: () => getStacJsonValue(href || "", fileUpload), @@ -56,7 +56,7 @@ export default function useStacValue({ enabled: enableStacGeoparquet, }); const stacGeoparquetTableResult = useQuery({ - queryKey: ["stac-geoparquet", href, datetimeBounds], + queryKey: ["stac-geoparquet-table", href, datetimeBounds], queryFn: () => (href && connection && @@ -82,94 +82,29 @@ export default function useStacValue({ stacGeoparquetTableResult.error || undefined; - const collectionsLink = value?.links?.find((link) => link.rel == "data"); - const collectionsResult = useInfiniteQuery({ - queryKey: ["stac-collections", collectionsLink?.href], - queryFn: async ({ pageParam }) => { - if (pageParam) { - return await fetch(pageParam).then((response) => { - if (response.ok) return response.json(); - else - throw new Error( - `Error while fetching collections from ${pageParam}` - ); - }); - } else { - return null; - } - }, - initialPageParam: collectionsLink?.href, - getNextPageParam: (lastPage: StacCollections | null) => - lastPage?.links?.find((link) => link.rel == "next")?.href, - enabled: !!collectionsLink, - }); - // TODO add a ceiling on the number of collections to fetch - // https://github.com/developmentseed/stac-map/issues/101 - useEffect(() => { - if (!collectionsResult.isFetching && collectionsResult.hasNextPage) { - collectionsResult.fetchNextPage(); - } - }, [collectionsResult]); - - const linkResults = useQueries({ + const itemsResult = useQueries({ queries: value?.links - ?.filter((link) => link.rel === "child" || link.rel === "item") + ?.filter((link) => link.rel === "item") .map((link) => { return { queryKey: ["stac-value", link.href], - queryFn: () => getStacJsonValue(link.href), - enabled: !collectionsLink, + queryFn: () => getStacJsonValue(link.href) as Promise, + enabled: !!(href && value), }; }) || [], combine: (results) => { return { - data: results.map((result) => result.data), + data: results.map((result) => result.data).filter((value) => !!value), }; }, }); - const { collections, catalogs, items } = useMemo(() => { - if (collectionsLink) { - return { - collections: collectionsResult.data?.pages.flatMap( - (page) => page?.collections || [] - ), - catalogs: undefined, - items: undefined, - }; - } else { - const collections = []; - const catalogs = []; - const items = []; - for (const value of linkResults.data) { - switch (value?.type) { - case "Catalog": - catalogs.push(value); - break; - case "Collection": - collections.push(value); - break; - case "Feature": - items.push(value); - break; - } - } - return { - collections: collections.length > 0 ? collections : undefined, - catalogs: catalogs.length > 0 ? catalogs : undefined, - items: items.length > 0 ? items : undefined, - }; - } - }, [collectionsLink, collectionsResult.data, linkResults.data]); - return { value, error, - collections, - catalogs, - items, table, stacGeoparquetItem: stacGeoparquetItem.data, + items: itemsResult.data.length > 0 ? itemsResult.data : undefined, }; } diff --git a/src/types/stac.d.ts b/src/types/stac.d.ts index 5666784..9435a74 100644 --- a/src/types/stac.d.ts +++ b/src/types/stac.d.ts @@ -20,6 +20,7 @@ export type StacValue = export interface StacCollections { collections: StacCollection[]; links?: StacLink[]; + numberMatched?: number; } export interface NaturalLanguageCollectionSearchResult { diff --git a/tests/app.spec.tsx b/tests/app.spec.tsx index 70c1469..dfbb271 100644 --- a/tests/app.spec.tsx +++ b/tests/app.spec.tsx @@ -88,4 +88,16 @@ describe("app", () => { .element(app.getByRole("button", { name: "stac-geoparquet" })) .toBeVisible(); }); + + test("paginates collections", async () => { + window.history.pushState({}, "", "?href=https://stac.eoapi.dev"); + const app = renderApp(); + await expect + .element(app.getByRole("button", { name: "Fetch more collections" })) + .toBeVisible(); + await expect + .element(app.getByRole("button", { name: "Fetch all collections" })) + .toBeVisible(); + await app.getByRole("button", { name: "Fetch more collections" }).click(); + }); });