feat: staggered loading of results (#237)

Earlier, if the user query resulted in tens of thousands of results, the app would seem to hang since nothing besides the loading indicator was moving. This PR changes the result loading to be more responsive by updating the displayed count every time an API response is received

src/apis/fmiApi.ts

@@ -29,7 +29,7 @@ export async function GetRawXMLResponse(query: string) {
   return res
 }
 
-export default async function GetSimpleFmiResponse(
+export default async function* GetSimpleFmiResponse(
   query: string,
   params: CommonParameters,
   sites: Site[]
@@ -53,15 +53,14 @@ export default async function GetSimpleFmiResponse(
       try {
         const res = await getXmlResponse(QUERY_URL + query + fp)
         const elements = res.getElementsByTagName('BsWfs:BsWfsElement')
-        results.push(...parseSimpleResponse(Array.from(elements), site))
+        const values = parseSimpleResponse(Array.from(elements), site)
+        yield values.sort(sortByTimeAndParameters)
       } catch (e) {
         console.error(e)
         mainState.setError(true)
       }
     }
   }
-
-  return results.sort(sortByTimeAndParameters)
 }
 
 export function sortByTimeAndParameters(a: IFmiResult, b: IFmiResult) {

src/apis/sykeApi.ts

@@ -1,35 +1,40 @@
 import { useMainStateStore } from '@/stores/mainStateStore'
 
-export default async function getVeslaData(resource: string, query: string) {
+const apiBase = 'https://rajapinnat.ymparisto.fi/api/meritietoportaali/api/'
+
+/** provide a generator to loop through response OData pages */
+export default async function* getPagedODataResponse(
+  resource: string,
+  query: string
+): AsyncGenerator<IODataResponse> {
   const mainState = useMainStateStore()
   try {
-    let res = await getJsonResponse(resource, query)
-    const data = res.value
-
-    while (res.nextLink) {
+    let activeQuery = query
+    do {
+      const res = await postODataQuery(resource, activeQuery)
+      yield res
+      if (res.value.length == 0) {
+        return
+      }
+      if (!res.nextLink) {
+        return
+      }
       const params = new URLSearchParams(res.nextLink.split('?')[1])
       const skipValue = params.get('$skip')
-      if (!skipValue) {
-        break
+      if (skipValue) {
+        activeQuery = query + '&$skip=' + skipValue!
+      } else {
+        activeQuery = ''
       }
-      const newQuery = query + '&$skip=' + skipValue!
-      res = await getJsonResponse(resource, newQuery)
-      data.push(...res.value)
-    }
-    if (data.length && data[0] instanceof Object) {
-      data.forEach((obj) => {
-        obj.dataSource = 'SYKE'
-      })
-    }
-    return data
+    } while (activeQuery.length > 0)
   } catch (e) {
     console.error(e)
     mainState.setError(true)
     return null
   }
 }
 
-interface IODataResponse {
+export interface IODataResponse {
   nextLink: string
   value: any[]
 }
@@ -42,23 +47,22 @@ interface ErrorResponse {
   }
 }
 
-async function getJsonResponse(
+async function postODataQuery(
   resource: string,
   query: string
 ): Promise<IODataResponse> {
-  const response = await fetch(
-    'https://rajapinnat.ymparisto.fi/api/meritietoportaali/api/' +
-      resource +
-      '/$query?api-version=1.0',
-    {
-      method: 'post',
-      headers: {
-        Accept: 'application/json',
-        'Content-Type': 'text/plain',
-      },
-      body: query,
-    }
-  )
+  const response = await fetch(apiBase + resource + '/$query?api-version=1.0', {
+    method: 'post',
+    headers: {
+      Accept: 'application/json',
+      'Content-Type': 'text/plain',
+    },
+    body: query,
+  })
+  return await parseResponse(response)
+}
+
+async function parseResponse(response: Response) {
   if (!response.ok) {
     let errorObj: ErrorResponse | undefined
     try {
@@ -78,6 +82,11 @@ async function getJsonResponse(
   }
 
   const json = await response.json()
+  if (json.value.length && json.value[0] instanceof Object) {
+    json.value.forEach((obj: any) => {
+      obj.dataSource = 'SYKE'
+    })
+  }
   return {
     nextLink: json['@odata.nextLink'],
     value: json.value,

src/main.ts

@@ -20,9 +20,9 @@ library.add(faAngleDown)
 
 const pinia = createPinia()
 const app = createApp(App).component('font-awesome-icon', FontAwesomeIcon)
-app.use(pinia)
 app.use(i18n)
 app.use(OpenLayersMap)
+app.use(pinia)
 
 const mainState = useMainStateStore()
 app.config.errorHandler = (e) => {

src/queries/FMI/getMareographTemperatureQuery.ts

@@ -4,6 +4,10 @@ import { CommonParameters } from '@/queries/commonParameters'
 const query =
   '&request=getFeature&storedquery_id=fmi::observations::mareograph::simple&parameters=TW_PT1H_AVG'
 
-export async function getMareographTemperatures(params: CommonParameters) {
-  return await GetSimpleFmiResponse(query, params, params.mareographSites)
+//TODO: this could probably implemented inline in the store
+export async function* getMareographTemperatures(params: CommonParameters) {
+  const pages = GetSimpleFmiResponse(query, params, params.mareographSites)
+  for await (const page of pages) {
+    yield page
+  }
 }

src/queries/FMI/getWaterLevelQuery.ts

@@ -4,6 +4,10 @@ import { CommonParameters } from '@/queries/commonParameters'
 const query =
   '&request=getFeature&storedquery_id=fmi::observations::mareograph::simple&parameters=WATLEV'
 
-export async function getWaterLevels(params: CommonParameters) {
-  return await GetSimpleFmiResponse(query, params, params.mareographSites)
+//TODO: implement inline in store
+export async function* getWaterLevels(params: CommonParameters) {
+  const pages = GetSimpleFmiResponse(query, params, params.mareographSites)
+  for await (const page of pages) {
+    yield page
+  }
 }

src/queries/FMI/getWaveDataQuery.ts

@@ -9,12 +9,17 @@ export enum WaveQueryParameters {
   waterTemperature = 'TWATER',
 }
 
-export async function getWaveData(
+//TODO: implement inline in store
+export async function* getWaveData(
   params: CommonParameters,
   queryParams: WaveQueryParameters[]
 ) {
   const query =
     '&request=getFeature&storedquery_id=fmi::observations::wave::simple&parameters=' +
     queryParams.join(',')
-  return await GetSimpleFmiResponse(query, params, params.buoySites)
+  const pages = GetSimpleFmiResponse(query, params, params.buoySites)
+
+  for await (const page of pages) {
+    yield page
+  }
 }

src/queries/Vesla/getApiStatusQuery.ts

@@ -1,6 +1,8 @@
-import getVeslaData from '@/apis/sykeApi'
+import getPagedODataResponse from '@/apis/sykeApi'
 
 export async function sykeApiIsOnline() {
-  const result = await getVeslaData('', '')
-  return !!result
+  const pages = getPagedODataResponse('', '')
+  for await (const page of pages) {
+    return page.value.length > 0
+  }
 }

src/queries/Vesla/getObservationsQuery.ts

@@ -1,11 +1,12 @@
 import { CommonParameters } from '../commonParameters'
-import getVeslaData from '@/apis/sykeApi'
+import getPagedODataResponse from '@/apis/sykeApi'
 import {
   buildODataInFilterFromArray,
   cleanupTimePeriod,
   getTimeParametersForVeslaFilter,
   fromObservationToSykeFormat,
 } from '@/helpers'
+import { IResponseFormat } from '../IResponseFormat'
 
 const select = [
   'Time',
@@ -36,42 +37,43 @@ function getFilter(params: CommonParameters, obsCode: string) {
   return filter
 }
 
-export async function getObservations(
+export async function* getObservations(
   params: CommonParameters,
   obsCode: string
-) {
+): AsyncGenerator<IResponseFormat[]> {
   if (params.veslaSites.length === 0) {
     return []
   }
   const filter = getFilter(params, obsCode)
-  let results = await getVeslaData(
+  const pages = getPagedODataResponse(
     resource,
     query +
       '&$filter=' +
       filter +
       '&$expand=Site($select=Latitude,Longitude,Depth)'
   )
-  if (!results) {
-    return []
-  }
-  results = results.map((r) => fromObservationToSykeFormat(r))
-  if (params.datePeriodMonths?.start !== params.datePeriodMonths?.end) {
-    return cleanupTimePeriod(results, params)
+  for await (const page of pages) {
+    const res = page.value.map((r) => fromObservationToSykeFormat(r))
+    if (params.datePeriodMonths?.start !== params.datePeriodMonths?.end) {
+      yield cleanupTimePeriod(res, params)
+    } else {
+      yield res
+    }
   }
-  return results
 }
 
 export async function getObservationSiteIds(
   params: CommonParameters,
   obsCode: string
 ) {
   const filter = getFilter(params, obsCode)
-  let data = await getVeslaData(
+  const pages = getPagedODataResponse(
     resource,
     '$apply=filter(' + filter + ')/groupby((siteId))&$orderby=siteId'
   )
-  if (data) {
-    data = data.map((d) => d.siteId)
+  const data: number[] = []
+  for await (const page of pages) {
+    data.push(...page.value.map((d: any) => d.siteId))
   }
-  return data as number[]
+  return data
 }

src/queries/Vesla/getVeslaSitesQuery.ts

@@ -1,4 +1,4 @@
-import getVeslaData from '@/apis/sykeApi'
+import getPagedODataResponse from '@/apis/sykeApi'
 import { Site, SiteTypes } from '../site'
 import { chunkArray, buildODataInFilterFromArray } from '@/helpers'
 
@@ -13,25 +13,21 @@ export async function* getVeslaSites(ids: number[]) {
     if (chunk.find((i) => i > 0)) {
       const filter =
         '$filter=' + buildODataInFilterFromArray(chunk, 'SiteId', false)
-      const res = (await getVeslaData('Sites', query + filter)) as Array<{
-        siteId: number
-        name: string
-        latitude: number
-        longitude: number
-        depth: number | null
-      }>
 
-      yield res.map(
-        (r) =>
-          new Site(
-            r.siteId,
-            r.name,
-            r.latitude,
-            r.longitude,
-            r.depth,
-            SiteTypes.Vesla
-          )
-      )
+      const pages = getPagedODataResponse('Sites', query + filter)
+      for await (const page of pages) {
+        yield page.value.map(
+          (r) =>
+            new Site(
+              r.siteId,
+              r.name,
+              r.latitude,
+              r.longitude,
+              r.depth,
+              SiteTypes.Vesla
+            )
+        )
+      }
     }
   }
 }

src/queries/Vesla/getWaterQualityOptionsQuery.ts

@@ -1,4 +1,4 @@
-import getVeslaData from '@/apis/sykeApi'
+import getPagedODataResponse from '@/apis/sykeApi'
 
 const query =
   '$select=DeterminationCombinationId,NameFi,NameSv,NameEn&\
@@ -12,10 +12,10 @@ export interface IWaterQualityOption {
 }
 
 export async function getWaterQualityOptions() {
-  const res = await getVeslaData('DeterminationCombinations', query)
+  const pages = getPagedODataResponse('DeterminationCombinations', query)
   const options: IWaterQualityOption[] = []
-  if (res) {
-    res.forEach((value) => {
+  for await (const page of pages) {
+    page.value.forEach((value: any) => {
       options.push({
         id: value.determinationCombinationId,
         name_fi: value.nameFi,