perf: enable compression for insertion (#494)

hyperdxio · Sep 8, 2024 · c5efb84 · c5efb84
1 parent f7ae1a4
commit c5efb84
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 6 deletions.
diff --git a/.changeset/four-cobras-do.md b/.changeset/four-cobras-do.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': patch
+---
+
+perf: enable compression for insertion
diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts
@@ -1308,7 +1308,7 @@ Array [
 
   it('clientInsertWithRetries (success)', async () => {
     jest
-      .spyOn(clickhouse.client, 'insert')
+      .spyOn(clickhouse.insertCHClient, 'insert')
       .mockRejectedValueOnce(new Error('first error'))
       .mockRejectedValueOnce(new Error('second error'))
       .mockResolvedValueOnce(null as any);
@@ -1320,12 +1320,12 @@ Array [
       timeout: 100,
     });
 
-    expect(clickhouse.client.insert).toHaveBeenCalledTimes(3);
+    expect(clickhouse.insertCHClient.insert).toHaveBeenCalledTimes(3);
   });
 
   it('clientInsertWithRetries (fail)', async () => {
     jest
-      .spyOn(clickhouse.client, 'insert')
+      .spyOn(clickhouse.insertCHClient, 'insert')
       .mockRejectedValueOnce(new Error('first error'))
       .mockRejectedValueOnce(new Error('second error'));
 
@@ -1340,7 +1340,7 @@ Array [
       expect(error.message).toBe('second error');
     }
 
-    expect(clickhouse.client.insert).toHaveBeenCalledTimes(2);
+    expect(clickhouse.insertCHClient.insert).toHaveBeenCalledTimes(2);
     expect.assertions(2);
   });
 

diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts
@@ -1,4 +1,5 @@
 import {
+  ClickHouseClientConfigOptions,
   createClient,
   ErrorLogParams as _CHErrorLogParams,
   Logger as _CHLogger,
@@ -156,7 +157,7 @@ export class CHLogger implements _CHLogger {
 }
 
 // TODO: move this to somewhere else
-export const client = createClient({
+const CH_CLIENT_CONFIG_OPTIONS: ClickHouseClientConfigOptions = {
   url: config.CLICKHOUSE_HOST,
   username: config.CLICKHOUSE_USER,
   password: config.CLICKHOUSE_PASSWORD,
@@ -183,6 +184,15 @@ export const client = createClient({
   log: {
     LoggerClass: CHLogger,
   },
+};
+
+export const client = createClient(CH_CLIENT_CONFIG_OPTIONS);
+export const insertCHClient = createClient({
+  ...CH_CLIENT_CONFIG_OPTIONS,
+  compression: {
+    request: true,
+    response: true,
+  },
 });
 
 export const getLogStreamTableName = (
@@ -380,7 +390,7 @@ export const clientInsertWithRetries = async <T>({
   const ts = Date.now();
   while (maxRetries > 0) {
     try {
-      await client.insert({
+      await insertCHClient.insert({
         table,
         values,
         format: 'JSONEachRow',