main_test.go

package meilisearch

import (
	"bufio"
	"context"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/stretchr/testify/require"
	"io"
	"net/http"
	"os"
	"strings"
	"testing"
)

var (
	masterKey           = "masterKey"
	defaultRankingRules = []string{
		"words", "typo", "proximity", "attribute", "sort", "exactness",
	}
	defaultTypoTolerance = TypoTolerance{
		Enabled: true,
		MinWordSizeForTypos: MinWordSizeForTypos{
			OneTypo:  5,
			TwoTypos: 9,
		},
		DisableOnWords:      []string{},
		DisableOnAttributes: []string{},
	}
	defaultPagination = Pagination{
		MaxTotalHits: 1000,
	}
	defaultFaceting = Faceting{
		MaxValuesPerFacet: 100,
		SortFacetValuesBy: map[string]SortFacetType{
			"*": SortFacetTypeAlpha,
		},
	}
)

var testNdjsonDocuments = []byte(`{"id": 1, "name": "Alice In Wonderland"}
{"id": 2, "name": "Pride and Prejudice"}
{"id": 3, "name": "Le Petit Prince"}
{"id": 4, "name": "The Great Gatsby"}
{"id": 5, "name": "Don Quixote"}
`)

var testCsvDocuments = []byte(`id,name
1,Alice In Wonderland
2,Pride and Prejudice
3,Le Petit Prince
4,The Great Gatsby
5,Don Quixote
`)

type docTest struct {
	ID   string `json:"id"`
	Name string `json:"name"`
}

type docTestBooks struct {
	BookID int    `json:"book_id"`
	Title  string `json:"title"`
	Tag    string `json:"tag"`
	Year   int    `json:"year"`
}

func setup(t *testing.T, host string, options ...Option) ServiceManager {
	t.Helper()

	opts := make([]Option, 0)
	opts = append(opts, WithAPIKey(masterKey))
	opts = append(opts, options...)

	if host == "" {
		host = getenv("MEILISEARCH_URL", "http://localhost:7700")
	}

	sv := New(host, opts...)
	return sv
}

func cleanup(services ...ServiceManager) func() {
	return func() {
		for _, s := range services {
			_, _ = deleteAllIndexes(s)
			_, _ = deleteAllKeys(s)
		}
	}
}

func getPrivateKey(sv ServiceManager) (key string) {
	list, err := sv.GetKeys(nil)
	if err != nil {
		return ""
	}
	for _, key := range list.Results {
		if strings.Contains(key.Name, "Default Admin API Key") || (key.Description == "") {
			return key.Key
		}
	}
	return ""
}

func getPrivateUIDKey(sv ServiceManager) (key string) {
	list, err := sv.GetKeys(nil)
	if err != nil {
		return ""
	}
	for _, key := range list.Results {
		if strings.Contains(key.Name, "Default Admin API Key") || (key.Description == "") {
			return key.UID
		}
	}
	return ""
}

func deleteAllIndexes(sv ServiceManager) (ok bool, err error) {
	list, err := sv.ListIndexes(nil)
	if err != nil {
		return false, err
	}

	for _, index := range list.Results {
		task, _ := sv.DeleteIndex(index.UID)
		_, err := sv.WaitForTask(task.TaskUID, 0)
		if err != nil {
			return false, err
		}
	}

	return true, nil
}

func deleteAllKeys(sv ServiceManager) (ok bool, err error) {
	list, err := sv.GetKeys(nil)
	if err != nil {
		return false, err
	}

	for _, key := range list.Results {
		if strings.Contains(key.Description, "Test") || (key.Description == "") {
			_, err = sv.DeleteKey(key.Key)
			if err != nil {
				return false, err
			}
		}
	}

	return true, nil
}

func getenv(key, fallback string) string {
	value := os.Getenv(key)
	if len(value) == 0 {
		return fallback
	}
	return value
}

func testWaitForTask(t *testing.T, i IndexManager, u *TaskInfo) {
	t.Helper()
	r, err := i.WaitForTask(u.TaskUID, 0)
	require.NoError(t, err)
	require.Equal(t, TaskStatusSucceeded, r.Status, fmt.Sprintf("Task failed: %#+v", r))
}

func testWaitForBatchTask(t *testing.T, i IndexManager, u []TaskInfo) {
	for _, id := range u {
		_, err := i.WaitForTask(id.TaskUID, 0)
		require.NoError(t, err)
	}
}

func setUpEmptyIndex(sv ServiceManager, index *IndexConfig) (resp *IndexResult, err error) {
	task, err := sv.CreateIndex(index)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}
	finalTask, _ := sv.WaitForTask(task.TaskUID, 0)
	if finalTask.Status != "succeeded" {
		cleanup(sv)
		return setUpEmptyIndex(sv, index)
	}
	return sv.GetIndex(index.Uid)
}

func setUpBasicIndex(sv ServiceManager, indexUID string) {
	index := sv.Index(indexUID)

	documents := []map[string]interface{}{
		{"book_id": 123, "title": "Pride and Prejudice"},
		{"book_id": 456, "title": "Le Petit Prince"},
		{"book_id": 1, "title": "Alice In Wonderland"},
		{"book_id": 1344, "title": "The Hobbit"},
		{"book_id": 4, "title": "Harry Potter and the Half-Blood Prince"},
		{"book_id": 42, "title": "The Hitchhiker's Guide to the Galaxy"},
	}

	task, err := index.AddDocuments(documents)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	finalTask, _ := index.WaitForTask(task.TaskUID, 0)
	if finalTask.Status != "succeeded" {
		os.Exit(1)
	}
}

func setupMovieIndex(t *testing.T, client ServiceManager) IndexManager {
	t.Helper()

	idx := client.Index("indexUID")

	testdata, err := os.Open("./testdata/movies.json")
	require.NoError(t, err)
	defer testdata.Close()

	tests := make([]map[string]interface{}, 0)

	require.NoError(t, json.NewDecoder(testdata).Decode(&tests))

	task, err := idx.AddDocuments(tests)
	require.NoError(t, err)
	testWaitForTask(t, idx, task)

	task, err = idx.UpdateFilterableAttributes(&[]string{"id"})
	require.NoError(t, err)
	testWaitForTask(t, idx, task)

	return idx
}

func setUpIndexForFaceting(client ServiceManager) {
	idx := client.Index("indexUID")

	booksTest := []docTestBooks{
		{BookID: 123, Title: "Pride and Prejudice", Tag: "Romance", Year: 1813},
		{BookID: 456, Title: "Le Petit Prince", Tag: "Tale", Year: 1943},
		{BookID: 1, Title: "Alice In Wonderland", Tag: "Tale", Year: 1865},
		{BookID: 1344, Title: "The Hobbit", Tag: "Epic fantasy", Year: 1937},
		{BookID: 4, Title: "Harry Potter and the Half-Blood Prince", Tag: "Epic fantasy", Year: 2005},
		{BookID: 42, Title: "The Hitchhiker's Guide to the Galaxy", Tag: "Epic fantasy", Year: 1978},
		{BookID: 742, Title: "The Great Gatsby", Tag: "Tragedy", Year: 1925},
		{BookID: 834, Title: "One Hundred Years of Solitude", Tag: "Tragedy", Year: 1967},
		{BookID: 17, Title: "In Search of Lost Time", Tag: "Modernist literature", Year: 1913},
		{BookID: 204, Title: "Ulysses", Tag: "Novel", Year: 1922},
		{BookID: 7, Title: "Don Quixote", Tag: "Satiric", Year: 1605},
		{BookID: 10, Title: "Moby Dick", Tag: "Novel", Year: 1851},
		{BookID: 730, Title: "War and Peace", Tag: "Historical fiction", Year: 1865},
		{BookID: 69, Title: "Hamlet", Tag: "Tragedy", Year: 1598},
		{BookID: 32, Title: "The Odyssey", Tag: "Epic", Year: 1571},
		{BookID: 71, Title: "Madame Bovary", Tag: "Novel", Year: 1857},
		{BookID: 56, Title: "The Divine Comedy", Tag: "Epic", Year: 1303},
		{BookID: 254, Title: "Lolita", Tag: "Novel", Year: 1955},
		{BookID: 921, Title: "The Brothers Karamazov", Tag: "Novel", Year: 1879},
		{BookID: 1032, Title: "Crime and Punishment", Tag: "Crime fiction", Year: 1866},
		{BookID: 1039, Title: "The Girl in the white shirt", Tag: "white shirt", Year: 1999},
		{BookID: 1050, Title: "星の王子さま", Tag: "物語", Year: 1943},
	}
	task, err := idx.AddDocuments(booksTest)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	finalTask, _ := idx.WaitForTask(task.TaskUID, 0)
	if finalTask.Status != "succeeded" {
		os.Exit(1)
	}
}

func setUpIndexWithNestedFields(client ServiceManager, indexUID string) {
	index := client.Index(indexUID)

	documents := []map[string]interface{}{
		{"id": 1, "title": "Pride and Prejudice", "info": map[string]interface{}{"comment": "A great book", "reviewNb": 50}},
		{"id": 2, "title": "Le Petit Prince", "info": map[string]interface{}{"comment": "A french book", "reviewNb": 600}},
		{"id": 3, "title": "Le Rouge et le Noir", "info": map[string]interface{}{"comment": "Another french book", "reviewNb": 700}},
		{"id": 4, "title": "Alice In Wonderland", "comment": "A weird book", "info": map[string]interface{}{"comment": "A weird book", "reviewNb": 800}},
		{"id": 5, "title": "The Hobbit", "info": map[string]interface{}{"comment": "An awesome book", "reviewNb": 900}},
		{"id": 6, "title": "Harry Potter and the Half-Blood Prince", "info": map[string]interface{}{"comment": "The best book", "reviewNb": 1000}},
		{"id": 7, "title": "The Hitchhiker's Guide to the Galaxy"},
	}
	task, err := index.AddDocuments(documents)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	finalTask, _ := index.WaitForTask(task.TaskUID, 0)
	if finalTask.Status != "succeeded" {
		os.Exit(1)
	}
}

func setUpIndexWithVector(client *meilisearch, indexUID string) (resp *IndexResult, err error) {
	req := &internalRequest{
		endpoint:    "/experimental-features",
		method:      http.MethodPatch,
		contentType: "application/json",
		withRequest: map[string]interface{}{
			"vectorStore": true,
		},
	}

	if err := client.client.executeRequest(context.Background(), req); err != nil {
		return nil, err
	}

	idx := client.Index(indexUID)
	taskInfo, err := idx.UpdateSettings(&Settings{
		Embedders: map[string]Embedder{
			"default": {
				Source:     "userProvided",
				Dimensions: 3,
			},
		},
	})
	if err != nil {
		return nil, err
	}
	settingsTask, err := idx.WaitForTask(taskInfo.TaskUID, 0)
	if err != nil {
		return nil, err
	}
	if settingsTask.Status != TaskStatusSucceeded {
		return nil, fmt.Errorf("Update settings task failed: %#+v", settingsTask)
	}

	documents := []map[string]interface{}{
		{"book_id": 123, "title": "Pride and Prejudice", "_vectors": map[string]interface{}{"default": []float64{0.1, 0.2, 0.3}}},
		{"book_id": 456, "title": "Le Petit Prince", "_vectors": map[string]interface{}{"default": []float64{2.4, 8.5, 1.6}}},
	}

	taskInfo, err = idx.AddDocuments(documents)
	if err != nil {
		return nil, err
	}

	finalTask, _ := idx.WaitForTask(taskInfo.TaskUID, 0)
	if finalTask.Status != TaskStatusSucceeded {
		return nil, fmt.Errorf("Add documents task failed: %#+v", finalTask)
	}

	return client.GetIndex(indexUID)
}

func setUpDistinctIndex(client ServiceManager, indexUID string) {
	idx := client.Index(indexUID)

	atters := []string{"product_id", "title", "sku", "url"}
	task, err := idx.UpdateFilterableAttributes(&atters)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	finalTask, _ := idx.WaitForTask(task.TaskUID, 0)
	if finalTask.Status != "succeeded" {
		os.Exit(1)
	}

	documents := []map[string]interface{}{
		{"product_id": 123, "title": "white shirt", "sku": "sku1234", "url": "https://example.com/products/p123"},
		{"product_id": 456, "title": "red shirt", "sku": "sku213", "url": "https://example.com/products/p456"},
		{"product_id": 1, "title": "green shirt", "sku": "sku876", "url": "https://example.com/products/p1"},
		{"product_id": 1344, "title": "blue shirt", "sku": "sku963", "url": "https://example.com/products/p1344"},
		{"product_id": 4, "title": "yellow shirt", "sku": "sku9064", "url": "https://example.com/products/p4"},
		{"product_id": 42, "title": "gray shirt", "sku": "sku964", "url": "https://example.com/products/p42"},
	}
	task, err = idx.AddDocuments(documents)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	finalTask, _ = idx.WaitForTask(task.TaskUID, 0)
	if finalTask.Status != "succeeded" {
		os.Exit(1)
	}
}

func testParseCsvDocuments(t *testing.T, documents io.Reader) []map[string]interface{} {
	var (
		docs   []map[string]interface{}
		header []string
	)
	r := csv.NewReader(documents)
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		}
		require.NoError(t, err)
		if header == nil {
			header = record
			continue
		}
		doc := make(map[string]interface{})
		for i, key := range header {
			doc[key] = record[i]
		}
		docs = append(docs, doc)
	}
	return docs
}

func testParseNdjsonDocuments(t *testing.T, documents io.Reader) []map[string]interface{} {
	var docs []map[string]interface{}
	scanner := bufio.NewScanner(documents)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" {
			continue
		}
		doc := make(map[string]interface{})
		err := json.Unmarshal([]byte(line), &doc)
		require.NoError(t, err)
		docs = append(docs, doc)
	}
	require.NoError(t, scanner.Err())
	return docs
}