From 50f34f2b38df2beb60b15ee66f44960d59dbbe6e Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Jan 2025 16:29:31 +0300 Subject: [PATCH 1/8] questdb reduce auto_flush_rows to 5k --- services/libs/questdb/src/ilp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/questdb/src/ilp.ts b/services/libs/questdb/src/ilp.ts index 40f4a7bb3..527bc5a2d 100644 --- a/services/libs/questdb/src/ilp.ts +++ b/services/libs/questdb/src/ilp.ts @@ -15,7 +15,7 @@ export const getClientILP = (): Sender => { const host = process.env['CROWD_QUESTDB_ILP_HOST'] const port = process.env['CROWD_QUESTDB_ILP_PORT'] const token = process.env['CROWD_QUESTDB_ILP_TOKEN'] - const autoflush = 'auto_flush=on;auto_flush_rows=10000;auto_flush_interval=30000;' + const autoflush = 'auto_flush=on;auto_flush_rows=5000;auto_flush_interval=30000;' log.trace({ host, port }, 'Creating QuestDB client (ILP) instance!') From 0bfb29016f8c3d537ff9fe67e6eafa4b132cc9e7 Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Jan 2025 18:16:27 +0300 Subject: [PATCH 2/8] increase max_buf_size to 200MB --- services/libs/questdb/src/ilp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/questdb/src/ilp.ts b/services/libs/questdb/src/ilp.ts index 527bc5a2d..8c636a207 100644 --- a/services/libs/questdb/src/ilp.ts +++ b/services/libs/questdb/src/ilp.ts @@ -21,7 +21,7 @@ export const getClientILP = (): Sender => { let conn = `http::addr=${host}:${port};${autoflush}` if (getEnv() === 'prod') { - conn = `https::addr=${host}:${port};token=${token};${autoflush}` + conn = `https::addr=${host}:${port};token=${token};${autoflush};max_buf_size=209715200` } client = Sender.fromConfig(conn) From 78ecf89a0aae92af1746f0427852213b09e1ae9b Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Jan 2025 18:22:17 +0300 Subject: [PATCH 3/8] fix option config --- services/libs/questdb/src/ilp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/questdb/src/ilp.ts b/services/libs/questdb/src/ilp.ts index 8c636a207..e402beed9 100644 --- a/services/libs/questdb/src/ilp.ts +++ b/services/libs/questdb/src/ilp.ts @@ -21,7 +21,7 @@ export const getClientILP = (): Sender => { let conn = `http::addr=${host}:${port};${autoflush}` if (getEnv() === 'prod') { - conn = `https::addr=${host}:${port};token=${token};${autoflush};max_buf_size=209715200` + conn = `https::addr=${host}:${port};token=${token};max_buf_size=209715200;${autoflush}` } client = Sender.fromConfig(conn) From 34ce88edc268b58d6a68e0d8d6a064757e98c00f Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Jan 2025 18:31:57 +0300 Subject: [PATCH 4/8] do more dramatic changes to see if it helps with max buf size errors --- services/libs/questdb/src/ilp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/questdb/src/ilp.ts b/services/libs/questdb/src/ilp.ts index e402beed9..43006033c 100644 --- a/services/libs/questdb/src/ilp.ts +++ b/services/libs/questdb/src/ilp.ts @@ -15,7 +15,7 @@ export const getClientILP = (): Sender => { const host = process.env['CROWD_QUESTDB_ILP_HOST'] const port = process.env['CROWD_QUESTDB_ILP_PORT'] const token = process.env['CROWD_QUESTDB_ILP_TOKEN'] - const autoflush = 'auto_flush=on;auto_flush_rows=5000;auto_flush_interval=30000;' + const autoflush = 'auto_flush=on;auto_flush_rows=1000;auto_flush_interval=15000;' log.trace({ host, port }, 'Creating QuestDB client (ILP) instance!') From 6d4e35a3972ef34452f46a76ed683e52c01e24f5 Mon Sep 17 00:00:00 2001 From: anil Date: Thu, 2 Jan 2025 18:44:54 +0300 Subject: [PATCH 5/8] reduce batches even more --- services/libs/questdb/src/ilp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/questdb/src/ilp.ts b/services/libs/questdb/src/ilp.ts index 43006033c..bf9137200 100644 --- a/services/libs/questdb/src/ilp.ts +++ b/services/libs/questdb/src/ilp.ts @@ -15,7 +15,7 @@ export const getClientILP = (): Sender => { const host = process.env['CROWD_QUESTDB_ILP_HOST'] const port = process.env['CROWD_QUESTDB_ILP_PORT'] const token = process.env['CROWD_QUESTDB_ILP_TOKEN'] - const autoflush = 'auto_flush=on;auto_flush_rows=1000;auto_flush_interval=15000;' + const autoflush = 'auto_flush=on;auto_flush_rows=500;auto_flush_interval=10000;' log.trace({ host, port }, 'Creating QuestDB client (ILP) instance!') From 0e22522dab6572695a308aac3c2f0dd3d50bb6c9 Mon Sep 17 00:00:00 2001 From: anil Date: Fri, 3 Jan 2025 11:56:01 +0300 Subject: [PATCH 6/8] add size check to attributes when adding activities to prevent malformed attributes to be added causing too large errors --- services/libs/data-access-layer/src/activities/ilp.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/libs/data-access-layer/src/activities/ilp.ts b/services/libs/data-access-layer/src/activities/ilp.ts index 02e0c140a..bf8554bb7 100644 --- a/services/libs/data-access-layer/src/activities/ilp.ts +++ b/services/libs/data-access-layer/src/activities/ilp.ts @@ -213,6 +213,13 @@ function objectToBytes(input: object): string { input = {} } + const stringified = JSON.stringify(input) + + // check size + if (new TextEncoder().encode(stringified).length > 512) { + return JSON.stringify({}) + } + return JSON.stringify(input) } From 52176de64019cc316f3cf70ec14bb59622735dad Mon Sep 17 00:00:00 2001 From: anil Date: Fri, 3 Jan 2025 11:58:01 +0300 Subject: [PATCH 7/8] revert auto flush settings --- services/libs/questdb/src/ilp.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/libs/questdb/src/ilp.ts b/services/libs/questdb/src/ilp.ts index bf9137200..40f4a7bb3 100644 --- a/services/libs/questdb/src/ilp.ts +++ b/services/libs/questdb/src/ilp.ts @@ -15,13 +15,13 @@ export const getClientILP = (): Sender => { const host = process.env['CROWD_QUESTDB_ILP_HOST'] const port = process.env['CROWD_QUESTDB_ILP_PORT'] const token = process.env['CROWD_QUESTDB_ILP_TOKEN'] - const autoflush = 'auto_flush=on;auto_flush_rows=500;auto_flush_interval=10000;' + const autoflush = 'auto_flush=on;auto_flush_rows=10000;auto_flush_interval=30000;' log.trace({ host, port }, 'Creating QuestDB client (ILP) instance!') let conn = `http::addr=${host}:${port};${autoflush}` if (getEnv() === 'prod') { - conn = `https::addr=${host}:${port};token=${token};max_buf_size=209715200;${autoflush}` + conn = `https::addr=${host}:${port};token=${token};${autoflush}` } client = Sender.fromConfig(conn) From f079e769f9a96b4d5c78d28ae404368a8ce70023 Mon Sep 17 00:00:00 2001 From: anil Date: Fri, 3 Jan 2025 12:08:51 +0300 Subject: [PATCH 8/8] increase limit a bit --- services/libs/data-access-layer/src/activities/ilp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/libs/data-access-layer/src/activities/ilp.ts b/services/libs/data-access-layer/src/activities/ilp.ts index bf8554bb7..d4980a546 100644 --- a/services/libs/data-access-layer/src/activities/ilp.ts +++ b/services/libs/data-access-layer/src/activities/ilp.ts @@ -216,7 +216,7 @@ function objectToBytes(input: object): string { const stringified = JSON.stringify(input) // check size - if (new TextEncoder().encode(stringified).length > 512) { + if (new TextEncoder().encode(stringified).length > 2000) { return JSON.stringify({}) }