feat!: return data as NDJSON instead of JSON

BREAKING CHANGE: return data as NDJSON instead of JSON Refs: #126
GenSpectrum · Apr 17, 2024 · c9feee1 · c9feee1
1 parent 685db9f
commit c9feee1
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 18 deletions.
diff --git a/endToEndTests/test/query.test.js b/endToEndTests/test/query.test.js
@@ -18,7 +18,7 @@ function readFilesRecursively(directoryPath) {
 
     if (fs.statSync(filePath).isDirectory()) {
       fileList = fileList.concat(readFilesRecursively(filePath));
-    } else {
+    } else if (filePath.endsWith('.json')) {
       fileList.push(filePath);
     }
   });
@@ -44,22 +44,16 @@ describe('The /query endpoint', () => {
         .post('/query')
         .send(testCase.query)
         .expect(200)
-        .expect('Content-Type', 'application/json')
+        .expect('Content-Type', 'application/x-ndjson')
         .expect(headerToHaveDataVersion);
-      try {
-        expect(response.body).to.deep.equal({ queryResult: testCase.expectedQueryResult });
-      } catch (e) {
-        const data = JSON.stringify(response.body, null, 2); // The `null` and `2` arguments format the JSON for readability
-        const errorFileName = 'actual_result_' + testCase.fileName;
-        fs.writeFile(errorFileName, data, err => {
-          if (err) {
-            console.error('Failed to write file', err);
-          } else {
-            console.log('Actual response saved to ' + errorFileName);
-          }
-        });
-        throw e;
-      }
+
+      let responseLines = response.text
+        .split(/\n/)
+        .filter(it => it !== '')
+        .map(it => JSON.parse(it));
+
+      const errorMessage = 'Actual result was: ' + response.text + '\n';
+      expect(responseLines, errorMessage).to.deep.equal(testCase.expectedQueryResult);
     })
   );
 

diff --git a/src/silo_api/query_handler.cpp b/src/silo_api/query_handler.cpp
@@ -31,17 +31,20 @@ void QueryHandler::post(
 
    SPDLOG_INFO("Request Id [{}] - received query: {}", request_id, query);
 
-   response.setContentType("application/json");
    try {
       const auto fixed_database = database_mutex.getDatabase();
 
       const auto query_result = fixed_database.database.executeQuery(query);
 
       response.set("data-version", fixed_database.database.getDataVersion().toString());
 
+      response.setContentType("application/x-ndjson");
       std::ostream& out_stream = response.send();
-      out_stream << nlohmann::json(query_result);
+      for (const auto& entry : query_result.query_result) {
+         out_stream << nlohmann::json(entry) << std::endl;
+      }
    } catch (const silo::QueryParseException& ex) {
+      response.setContentType("application/json");
       SPDLOG_INFO("Query is invalid: " + query + " - exception: " + ex.what());
       response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_BAD_REQUEST);
       std::ostream& out_stream = response.send();