From 17e8d55f72e39923fc936c50c0e5fee537d358c6 Mon Sep 17 00:00:00 2001 From: Michele Riva Date: Thu, 4 Jul 2024 14:31:52 +0200 Subject: [PATCH] feat(orama): updates Orama and simplifies Orama Cloud sync --- apps/site/package.json | 4 +- .../scripts/orama-search/sync-orama-cloud.mjs | 65 ++++--------------- package-lock.json | 10 +-- 3 files changed, 20 insertions(+), 59 deletions(-) diff --git a/apps/site/package.json b/apps/site/package.json index c40fe1fac17fb..c681a0cb509b3 100644 --- a/apps/site/package.json +++ b/apps/site/package.json @@ -39,8 +39,8 @@ "@heroicons/react": "~2.1.1", "@mdx-js/mdx": "^3.0.1", "@nodevu/core": "~0.1.0", - "@orama/highlight": "^0.1.5", - "@oramacloud/client": "^1.0.13", + "@orama/highlight": "^0.1.6", + "@oramacloud/client": "^1.3.2", "@radix-ui/react-accessible-icon": "^1.0.3", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-dialog": "^1.0.5", diff --git a/apps/site/scripts/orama-search/sync-orama-cloud.mjs b/apps/site/scripts/orama-search/sync-orama-cloud.mjs index ba80b38684253..276d19c4e45ad 100644 --- a/apps/site/scripts/orama-search/sync-orama-cloud.mjs +++ b/apps/site/scripts/orama-search/sync-orama-cloud.mjs @@ -1,16 +1,18 @@ +import { CloudManager } from '@oramacloud/client'; + import { siteContent } from './get-documents.mjs'; import { ORAMA_SYNC_BATCH_SIZE } from '../../next.constants.mjs'; -// The following follows the instructions at https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks +// The following follows the instructions at https://docs.orama.com/cloud/data-sources/custom-integrations/webhooks const INDEX_ID = process.env.ORAMA_INDEX_ID; const API_KEY = process.env.ORAMA_SECRET_KEY; -const ORAMA_API_BASE_URL = `https://api.oramasearch.com/api/v1/webhooks/${INDEX_ID}`; -const oramaHeaders = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${API_KEY}`, -}; +const oramaCloudManager = new CloudManager({ + api_key: API_KEY, +}); + +const oramaIndex = oramaCloudManager.index(INDEX_ID); console.log(`Syncing ${siteContent.length} documents to Orama Cloud index`); @@ -26,50 +28,9 @@ const runUpdate = async () => { console.log(`Sending ${batches.length} batches of ${batchSize} documents`); - await Promise.all(batches.map(insertBatch)); -}; - -// We call the "notify" API to upsert the documents in the index. -// Orama will keep a queue of all the documents we send, and will process them once we call the "deploy" API. -// Full docs on the "notify" API: https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks#updating-removing-inserting-elements-in-a-live-index -const insertBatch = async batch => { - const { ok, statusText } = await fetch(`${ORAMA_API_BASE_URL}/notify`, { - method: 'POST', - headers: oramaHeaders, - body: JSON.stringify({ upsert: batch }), - }); - - if (!ok) { - throw new Error(`Request to "/notify" failed with status: ${statusText}`); - } -}; - -// We call the "deploy" API to trigger a deployment of the index, which will process all the documents in the queue. -// Full docs on the "deploy" API: https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks#deploying-the-index -const triggerDeployment = async () => { - const { ok, statusText } = await fetch(`${ORAMA_API_BASE_URL}/deploy`, { - method: 'POST', - headers: oramaHeaders, - }); - - if (!ok) { - throw new Error(`Request to "/deploy" failed with status: ${statusText}`); - } -}; - -// We call the "snapshot" API to empty the index before inserting the new documents. -// The "snapshot" API is typically used to replace the entire index with a fresh set of documents, but we use it here to empty the index. -// This operation gets queued, so the live index will still be available until we call the "deploy" API and redeploy the index. -// Full docs on the "snapshot" API: https://docs.oramasearch.com/cloud/data-sources/custom-integrations/webhooks#inserting-a-snapshot -const emptyOramaIndex = async () => { - const { ok, statusText } = await fetch(`${ORAMA_API_BASE_URL}/snapshot`, { - method: 'POST', - headers: oramaHeaders, - body: JSON.stringify([]), - }); - - if (!ok) { - throw new Error(`Request to "/snapshot" failed with status: ${statusText}`); + for (const batch of batches) { + // In Orama, "update" is an upsert operation. + await oramaIndex.update(batch); } }; @@ -79,8 +40,8 @@ const emptyOramaIndex = async () => { // 3. Trigger a deployment // Once all these steps are done, the new documents will be available in the live index. // Allow Orama up to 1 minute to distribute the documents to all the 300+ nodes worldwide. -await emptyOramaIndex(); +await oramaIndex.empty(); await runUpdate(); -await triggerDeployment(); +await oramaIndex.deploy(); console.log('Orama Cloud sync completed successfully!'); diff --git a/package-lock.json b/package-lock.json index 862c33dd0c777..95e30082e36b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,8 +32,8 @@ "@heroicons/react": "~2.1.1", "@mdx-js/mdx": "^3.0.1", "@nodevu/core": "~0.1.0", - "@orama/highlight": "^0.1.5", - "@oramacloud/client": "^1.0.13", + "@orama/highlight": "^0.1.6", + "@oramacloud/client": "^1.3.2", "@radix-ui/react-accessible-icon": "^1.0.3", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-dialog": "^1.0.5", @@ -4360,9 +4360,9 @@ } }, "node_modules/@oramacloud/client": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@oramacloud/client/-/client-1.1.6.tgz", - "integrity": "sha512-TocXGSqoIA9MvKd094XcJWVtQcZu0/x0zXZfTjzlLJLhBQdr4R1xQkJ69GHlooD+r/6umlgpaWs+7J711XQZ2Q==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@oramacloud/client/-/client-1.3.2.tgz", + "integrity": "sha512-veF5z2T8YTOYvHe2a9Ls9NneyND8KHq67ij5RUsKH+tmbri2SIIRJwJ/38YmZpTur60QVhbsHxxQ5Hh76ZkqvA==", "dependencies": { "@orama/orama": "^2.0.16", "@paralleldrive/cuid2": "^2.2.1",