Support for custom csv delimiter - Fixes #292

meilisearch · Apr 27, 2023 · 04dcd2e · 04dcd2e
1 parent e878c6d
commit 04dcd2e
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 4 deletions.
diff --git a/lib/src/index.dart b/lib/src/index.dart
@@ -88,6 +88,7 @@ abstract class MeiliSearchIndex {
   Future<Task> addDocumentsCsv(
     String documents, {
     String? primaryKey,
+    String? csvDelimiter,
   });
 
   /// {@macro meili.add_docs}
@@ -120,6 +121,7 @@ abstract class MeiliSearchIndex {
   Future<List<Task>> addDocumentsCsvInBatches(
     String documents, {
     String? primaryKey,
+    String? csvDelimiter,
     int batchSize = 1000,
   });
 
@@ -173,6 +175,7 @@ abstract class MeiliSearchIndex {
   Future<Task> updateDocumentsCsv(
     String documents, {
     String? primaryKey,
+    String? csvDelimiter,
   });
 
   /// {@template meili.update_docs_batches}
@@ -195,6 +198,7 @@ abstract class MeiliSearchIndex {
     String documents, {
     String? primaryKey,
     int batchSize = 1000,
+    String? csvDelimiter,
   });
 
   /// {@macro meili.update_docs_batches}

diff --git a/lib/src/index_impl.dart b/lib/src/index_impl.dart
@@ -198,12 +198,14 @@ class MeiliSearchIndexImpl implements MeiliSearchIndex {
   Future<Task> addDocumentsCsv(
     String documents, {
     String? primaryKey,
+    String? csvDelimiter,
   }) {
     return _getTask(http.postMethod(
       '/indexes/$uid/documents',
       data: documents,
       queryParameters: {
         if (primaryKey != null) 'primaryKey': primaryKey,
+        if (csvDelimiter != null) 'csvDelimiter': csvDelimiter,
       },
       contentType: _csvContentType,
     ));
@@ -238,6 +240,7 @@ class MeiliSearchIndexImpl implements MeiliSearchIndex {
     String documents, {
     String? primaryKey,
     int batchSize = 1000,
+    String? csvDelimiter,
   }) {
     final ls = LineSplitter();
     final split = ls.convert(documents);
@@ -248,6 +251,7 @@ class MeiliSearchIndexImpl implements MeiliSearchIndex {
             (slice) => addDocumentsCsv(
               [header, ...slice].join('\n'),
               primaryKey: primaryKey,
+              csvDelimiter: csvDelimiter,
             ),
           ),
     );
@@ -305,12 +309,17 @@ class MeiliSearchIndexImpl implements MeiliSearchIndex {
   }
 
   @override
-  Future<Task> updateDocumentsCsv(String documents, {String? primaryKey}) {
+  Future<Task> updateDocumentsCsv(
+    String documents, {
+    String? primaryKey,
+    String? csvDelimiter,
+  }) {
     return _getTask(http.putMethod(
       '/indexes/$uid/documents',
       data: documents,
       queryParameters: {
         if (primaryKey != null) 'primaryKey': primaryKey,
+        if (csvDelimiter != null) 'csvDelimiter': csvDelimiter,
       },
       contentType: _csvContentType,
     ));
@@ -345,6 +354,7 @@ class MeiliSearchIndexImpl implements MeiliSearchIndex {
     String documents, {
     String? primaryKey,
     int batchSize = 1000,
+    String? csvDelimiter,
   }) {
     final ls = LineSplitter();
     final split = ls.convert(documents);
@@ -356,6 +366,7 @@ class MeiliSearchIndexImpl implements MeiliSearchIndex {
             (slice) => updateDocumentsCsv(
               [header, ...slice].join('\n'),
               primaryKey: primaryKey,
+              csvDelimiter: csvDelimiter,
             ),
           ),
     );

diff --git a/test/documents_test.dart b/test/documents_test.dart
@@ -8,6 +8,7 @@ import 'utils/wait_for.dart';
 import 'utils/client.dart';
 import 'utils/books.dart';
 
+const _customCSVDelimiter = ';';
 void main() {
   group('Documents', () {
     setUpClient();
@@ -104,6 +105,17 @@ void main() {
 
           await testAddedData();
         });
+        test('CSV with custom delimiter', () async {
+          await index
+              .addDocumentsCsv(
+                dataAsCSV(data, delimiter: _customCSVDelimiter),
+                primaryKey: kbookId,
+                csvDelimiter: _customCSVDelimiter,
+              )
+              .waitFor(client: client);
+
+          await testAddedData();
+        });
 
         test('NDJson', () async {
           await index
@@ -193,6 +205,17 @@ void main() {
 
           await testUpdatedData();
         });
+        test('CSV With custom delimiter', () async {
+          await index
+              .updateDocumentsCsv(
+                dataAsCSV(updateData, delimiter: _customCSVDelimiter),
+                primaryKey: kbookId,
+                csvDelimiter: _customCSVDelimiter,
+              )
+              .waitFor(client: client);
+
+          await testUpdatedData();
+        });
 
         test('NDJson', () async {
           await index
@@ -313,6 +336,16 @@ void main() {
 
           await testAddedBatches(tasks);
         });
+        test('CSV with custom delimiter', () async {
+          final tasks = await index.addDocumentsCsvInBatches(
+            dataAsCSV(data, delimiter: _customCSVDelimiter),
+            batchSize: batchSize,
+            primaryKey: kbookId,
+            csvDelimiter: _customCSVDelimiter,
+          );
+
+          await testAddedBatches(tasks);
+        });
 
         test('NDJSON', () async {
           final tasks = await index.addDocumentsNdjsonInBatches(
@@ -397,6 +430,16 @@ void main() {
 
           await testUpdatedBatches(tasks);
         });
+        test('CSV with custom delimiter', () async {
+          final tasks = await index.updateDocumentsCsvInBatches(
+            dataAsCSV(updateData, delimiter: _customCSVDelimiter),
+            batchSize: batchSize,
+            primaryKey: kbookId,
+            csvDelimiter: _customCSVDelimiter,
+          );
+
+          await testUpdatedBatches(tasks);
+        });
 
         test('NDJSON', () async {
           final tasks = await index.updateDocumentsNdjsonInBatches(

diff --git a/test/utils/books_data.dart b/test/utils/books_data.dart
@@ -65,7 +65,7 @@ enum CSVHeaderTypes {
   unkown,
 }
 
-String dataAsCSV(List<Map<String, Object?>> data) {
+String dataAsCSV(List<Map<String, Object?>> data, {String delimiter = ','}) {
   final csvHeaders = <String, CSVHeaderTypes?>{};
   final csvDataBuffer = StringBuffer();
   for (final element in data) {
@@ -91,7 +91,7 @@ String dataAsCSV(List<Map<String, Object?>> data) {
         (obj) => csvHeaderEntries
             .map((e) => e.key)
             .map((headerKey) => json.encode(obj[headerKey] ?? ""))
-            .join(','),
+            .join(delimiter),
       )
       .forEach(csvDataBuffer.writeln);
 
@@ -103,7 +103,7 @@ String dataAsCSV(List<Map<String, Object?>> data) {
             ? ':boolean'
             : null;
     return jsonEncode('${header.key}${typeStr ?? ""}');
-  }).join(",");
+  }).join(delimiter);
 
   return '$headerStr\n${csvDataBuffer.toString()}';
 }