Integrate Orama for search (#6257)

* feat: adds basic orama structure

* feat: adds searchbox

* feat: integrates searchbox

* style: moves components to separate files

* feat: wip on searchbox

* feat: adds basic mobile styles

* tmp: work in progress

* work in progress

* feat: improves search page

* style: addresses feedbacks on code style

* style: addresses feedbacks on code style

* feat: adds texts management via i18n

* fix: encodes URL components

* style: addresses feedback

* style: addresses feedback

* docs: adds comments to Orama sync script

* style: addresses feedback

* style: addresses feedback

* style: addresses feedback

* refactor: moves components and hooks into the correct folder structure

* refactor: moves components and hooks into the correct folder structure

* refactor: moves components and hooks into the correct folder structure

* refactor: moves components and hooks into the correct folder structure

* refactor: moves components and hooks into the correct folder structure

* refactor: moves components and hooks into the correct folder structure

* style: addresses feedback

* style: addresses feedback

* style: addresses feedback

* style: addresses feedback

* style: addresses feedback

* ci: adds Orama sync script to gh workflows

* chore: removes useless log

* style: addresses feedback and adds tests

* feat: adds footer

* fix: fixes logo in light mode

* updates orama dependencies

* chore: updates orama dependencies to latest version

* chore: updates Orama client

* fix: fixes unexpected close of modal on click

* fix: fixes Orama logo

* chore: removes unused test attribute

* fix: code-reviews

* chore: minor copy changes

* fix: aggregate results and make them unique

---------

Signed-off-by: Michele Riva <ciao@micheleriva.it>
Co-authored-by: Claudio Wunder <cwunder@gnome.org>

.github/workflows/build.yml

@@ -123,3 +123,8 @@ jobs:
           # this should be a last resort in case by any chances the build memory gets too high
           # but in general this should never happen
           NODE_OPTIONS: '--max_old_space_size=4096'
+
+      - name: Sync Orama Cloud
+        if: github.ref == 'refs/heads/main'
+        run: |
+          npm run sync-orama

.gitignore

@@ -2,6 +2,7 @@
 node_modules
 npm-debug.log
 .npm
+.env.local
 
 # Next.js Build Output
 .next

app/[locale]/next-data/page-data/route.ts

@@ -15,30 +15,38 @@ export const GET = async () => {
     defaultLocale.code
   );
 
-  const availablePagesMetadata = allAvailbleRoutes.map(async pathname => {
-    const { source, filename } = await dynamicRouter.getMarkdownFile(
-      defaultLocale.code,
-      pathname
-    );
+  const availablePagesMetadata = allAvailbleRoutes
+    .filter(route => !route.startsWith('blog'))
+    .map(async pathname => {
+      const { source, filename } = await dynamicRouter.getMarkdownFile(
+        defaultLocale.code,
+        pathname
+      );
 
-    // Gets the title and the Description from the Page Metadata
-    const { title, description } = await dynamicRouter.getPageMetadata(
-      defaultLocale.code,
-      pathname
-    );
+      // Gets the title and the Description from the Page Metadata
+      const { title, description } = await dynamicRouter.getPageMetadata(
+        defaultLocale.code,
+        pathname
+      );
 
-    // Parser the Markdown source with `gray-matter` and then only
-    // grabs the markdown content and cleanses it by removing HTML/JSX tags
-    // removing empty/blank lines or lines just with spaces and trims each line
-    // from leading and trailing paddings/spaces
-    const cleanedContent = parseRichTextIntoPlainText(matter(source).content);
+      // Parser the Markdown source with `gray-matter` and then only
+      // grabs the markdown content and cleanses it by removing HTML/JSX tags
+      // removing empty/blank lines or lines just with spaces and trims each line
+      // from leading and trailing paddings/spaces
+      const cleanedContent = parseRichTextIntoPlainText(matter(source).content);
 
-    // Deflates a String into a base64 string-encoded (zlib compressed)
-    const deflatedSource = deflateSync(cleanedContent).toString('base64');
+      // Deflates a String into a base64 string-encoded (zlib compressed)
+      const deflatedSource = deflateSync(cleanedContent).toString('base64');
 
-    // Returns metadata of each page available on the Website
-    return { filename, pathname, title, description, content: deflatedSource };
-  });
+      // Returns metadata of each page available on the Website
+      return {
+        filename,
+        pathname,
+        title,
+        description,
+        content: deflatedSource,
+      };
+    });
 
   return Response.json(await Promise.all(availablePagesMetadata));
 };

components/Common/Search/States/WithAllResults.tsx

@@ -0,0 +1,40 @@
+import type { Results } from '@orama/orama';
+import NextLink from 'next/link';
+import { useParams } from 'next/navigation';
+import { useTranslations } from 'next-intl';
+import type { FC } from 'react';
+
+import type { SearchDoc } from '@/types';
+
+import styles from './index.module.css';
+
+type SearchResults = Results<SearchDoc>;
+
+type SeeAllProps = {
+  searchResults: SearchResults;
+  searchTerm: string;
+  selectedFacetName: string;
+  onSeeAllClick: () => void;
+};
+
+export const WithAllResults: FC<SeeAllProps> = props => {
+  const t = useTranslations();
+  const params = useParams();
+
+  const locale = params?.locale ?? 'en';
+  const resultsCount = props.searchResults?.count?.toLocaleString('en') ?? 0;
+  const searchParams = new URLSearchParams();
+
+  searchParams.set('q', props.searchTerm);
+  searchParams.set('section', props.selectedFacetName);
+
+  const allResultsURL = `/${locale}/search?${searchParams.toString()}`;
+
+  return (
+    <div className={styles.seeAllFulltextSearchResults}>
+      <NextLink href={allResultsURL} onClick={props.onSeeAllClick}>
+        {t('components.search.seeAll.text', { count: resultsCount })}
+      </NextLink>
+    </div>
+  );
+};

components/Common/Search/States/WithEmptyState.tsx

@@ -0,0 +1,14 @@
+import { useTranslations } from 'next-intl';
+import type { FC } from 'react';
+
+import styles from './index.module.css';
+
+export const WithEmptyState: FC = () => {
+  const t = useTranslations();
+
+  return (
+    <div className={styles.emptyStateContainer}>
+      {t('components.search.emptyState.text')}
+    </div>
+  );
+};

components/Common/Search/States/WithError.tsx

@@ -0,0 +1,14 @@
+import { useTranslations } from 'next-intl';
+import type { FC } from 'react';
+
+import styles from './index.module.css';
+
+export const WithError: FC = () => {
+  const t = useTranslations();
+
+  return (
+    <div className={styles.searchErrorContainer}>
+      {t('components.search.searchError.text')}
+    </div>
+  );
+};

components/Common/Search/States/WithNoResults.tsx

@@ -0,0 +1,16 @@
+import { useTranslations } from 'next-intl';
+import type { FC } from 'react';
+
+import styles from './index.module.css';
+
+type NoResultsProps = { searchTerm: string };
+
+export const WithNoResults: FC<NoResultsProps> = props => {
+  const t = useTranslations();
+
+  return (
+    <div className={styles.noResultsContainer}>
+      {t('components.search.noResults.text', { query: props.searchTerm })}
+    </div>
+  );
+};

components/Common/Search/States/WithPoweredBy.tsx

@@ -0,0 +1,41 @@
+'use client';
+
+import Image from 'next/image';
+import { useTranslations } from 'next-intl';
+import { useTheme } from 'next-themes';
+import { useEffect, useState } from 'react';
+
+import styles from './index.module.css';
+
+const getLogoURL = (theme: string = 'dark') =>
+  `https://website-assets.oramasearch.com/orama-when-${theme}.svg`;
+
+export const WithPoweredBy = () => {
+  const t = useTranslations();
+  const { resolvedTheme } = useTheme();
+  const [logoURL, setLogoURL] = useState<string>();
+
+  useEffect(() => setLogoURL(getLogoURL(resolvedTheme)), [resolvedTheme]);
+
+  return (
+    <div className={styles.poweredBy}>
+      {t('components.search.poweredBy.text')}
+
+      <a
+        href="https://oramasearch.com?utm_source=nodejs.org"
+        target="_blank"
+        rel="noreferer"
+      >
+        {logoURL && (
+          <Image
+            src={logoURL}
+            alt="Powered by OramaSearch"
+            className={styles.poweredByLogo}
+            width={80}
+            height={20}
+          />
+        )}
+      </a>
+    </div>
+  );
+};

components/Common/Search/States/WithSearchBox.tsx

@@ -0,0 +1,193 @@
+'use client';
+
+import {
+  MagnifyingGlassIcon,
+  ChevronLeftIcon,
+} from '@heroicons/react/24/outline';
+import type { Results, Nullable } from '@orama/orama';
+import classNames from 'classnames';
+import { useState, useRef, useEffect } from 'react';
+import type { FC } from 'react';
+
+import styles from '@/components/Common/Search/States/index.module.css';
+import { WithAllResults } from '@/components/Common/Search/States/WithAllResults';
+import { WithEmptyState } from '@/components/Common/Search/States/WithEmptyState';
+import { WithError } from '@/components/Common/Search/States/WithError';
+import { WithNoResults } from '@/components/Common/Search/States/WithNoResults';
+import { WithPoweredBy } from '@/components/Common/Search/States/WithPoweredBy';
+import { WithSearchResult } from '@/components/Common/Search/States/WithSearchResult';
+import { useClickOutside } from '@/hooks/react-client';
+import { useRouter } from '@/navigation.mjs';
+import { DEFAULT_ORAMA_QUERY_PARAMS } from '@/next.constants.mjs';
+import { search as oramaSearch, getInitialFacets } from '@/next.orama.mjs';
+import type { SearchDoc } from '@/types';
+import { debounce } from '@/util/debounce';
+
+type Facets = { [key: string]: number };
+
+type SearchResults = Nullable<Results<SearchDoc>>;
+
+type SearchBoxProps = { onClose: () => void };
+
+export const WithSearchBox: FC<SearchBoxProps> = ({ onClose }) => {
+  const [searchTerm, setSearchTerm] = useState('');
+  const [searchResults, setSearchResults] = useState<SearchResults>(null);
+  const [selectedFacet, setSelectedFacet] = useState<number>(0);
+  const [searchError, setSearchError] = useState<Nullable<Error>>(null);
+
+  const router = useRouter();
+  const searchInputRef = useRef<HTMLInputElement>(null);
+  const searchBoxRef = useRef<HTMLDivElement>(null);
+
+  const search = (term: string) => {
+    oramaSearch({
+      term,
+      ...DEFAULT_ORAMA_QUERY_PARAMS,
+      mode: 'fulltext',
+      returning: [
+        'path',
+        'pageSectionTitle',
+        'pageTitle',
+        'path',
+        'siteSection',
+      ],
+      ...filterBySection(),
+    })
+      .then(setSearchResults)
+      .catch(setSearchError);
+  };
+
+  useClickOutside(searchBoxRef, () => {
+    reset();
+    onClose();
+  });
+
+  useEffect(() => {
+    searchInputRef.current?.focus();
+
+    getInitialFacets().then(setSearchResults).catch(setSearchError);
+
+    return reset;
+  }, []);
+
+  useEffect(
+    () => debounce(() => search(searchTerm), 1000),
+    // we don't need to care about memoization of search function
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [searchTerm, selectedFacet]
+  );
+
+  const reset = () => {
+    setSearchTerm('');
+    setSearchResults(null);
+    setSelectedFacet(0);
+  };
+
+  const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
+    e.preventDefault();
+    router.push(`/search?q=${searchTerm}&section=${selectedFacetName}`);
+    onClose();
+  };
+
+  const changeFacet = (idx: number) => setSelectedFacet(idx);
+
+  const filterBySection = () => {
+    if (selectedFacet === 0) {
+      return {};
+    }
+
+    return { where: { siteSection: { eq: selectedFacetName } } };
+  };
+
+  const facets: Facets = {
+    all: searchResults?.count ?? 0,
+    ...(searchResults?.facets?.siteSection?.values ?? {}),
+  };
+
+  const selectedFacetName = Object.keys(facets)[selectedFacet];
+
+  return (
+    <div className={styles.searchBoxModalContainer}>
+      <div className={styles.searchBoxModalPanel} ref={searchBoxRef}>
+        <div className={styles.searchBoxInnerPanel}>
+          <div className={styles.searchBoxInputContainer}>
+            <button
+              onClick={onClose}
+              className={styles.searchBoxBackIconContainer}
+            >
+              <ChevronLeftIcon className={styles.searchBoxBackIcon} />
+            </button>
+
+            <MagnifyingGlassIcon
+              className={styles.searchBoxMagnifyingGlassIcon}
+            />
+
+            <form onSubmit={onSubmit}>
+              <input
+                ref={searchInputRef}
+                type="search"
+                className={styles.searchBoxInput}
+                onChange={event => setSearchTerm(event.target.value)}
+                value={searchTerm}
+              />
+            </form>
+          </div>
+
+          <div className={styles.fulltextSearchSections}>
+            {Object.keys(facets).map((facetName, idx) => (
+              <button
+                key={facetName}
+                className={classNames(styles.fulltextSearchSection, {
+                  [styles.fulltextSearchSectionSelected]: selectedFacet === idx,
+                })}
+                onClick={() => changeFacet(idx)}
+              >
+                {facetName}
+                <span className={styles.fulltextSearchSectionCount}>
+                  ({facets[facetName].toLocaleString('en')})
+                </span>
+              </button>
+            ))}
+          </div>
+
+          <div className={styles.fulltextResultsContainer}>
+            {searchError && <WithError />}
+
+            {!searchError && !searchTerm && <WithEmptyState />}
+
+            {!searchError && searchTerm && (
+              <>
+                {searchResults &&
+                  searchResults.count > 0 &&
+                  searchResults.hits.map(hit => (
+                    <WithSearchResult
+                      key={hit.id}
+                      hit={hit}
+                      searchTerm={searchTerm}
+                    />
+                  ))}
+
+                {searchResults && searchResults.count === 0 && (
+                  <WithNoResults searchTerm={searchTerm} />
+                )}
+
+                {searchResults && searchResults.count > 8 && (
+                  <WithAllResults
+                    searchResults={searchResults}
+                    searchTerm={searchTerm}
+                    selectedFacetName={selectedFacetName}
+                    onSeeAllClick={onClose}
+                  />
+                )}
+              </>
+            )}
+          </div>
+
+          <div className={styles.fulltextSearchFooter}>
+            <WithPoweredBy />
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};