Skip to content

Commit

Permalink
[BEAM-14429] Force java load test on dataflow runner v2 forceNumIniti…
Browse files Browse the repository at this point in the history 
…alBundles to 1 (apache#17576)
  • Loading branch information
@y1chi
y1chi committed May 9, 2022
1 parent 1af17bd commit 178441d
Show file tree
Hide file tree
Showing 6 changed files with 68 additions and 18 deletions.
10 changes: 10 additions & 0 deletions .test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

def loadTestConfigurations = { mode, isStreaming ->
[
[
Expand All @@ -47,6 +49,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -55,6 +58,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down Expand Up @@ -86,6 +90,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 5
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -94,6 +99,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down Expand Up @@ -126,6 +132,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 200000
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -134,6 +141,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down Expand Up @@ -166,6 +174,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -174,6 +183,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down
10 changes: 10 additions & 0 deletions .test-infra/jenkins/job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

def loadTestConfigurations = { mode, isStreaming ->
[
[
Expand All @@ -47,6 +49,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -55,6 +58,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down Expand Up @@ -86,6 +90,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 5
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -94,6 +99,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down Expand Up @@ -126,6 +132,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 200000
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -134,6 +141,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down Expand Up @@ -166,6 +174,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand All @@ -174,6 +183,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 2000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 1000
}
""".trim().replaceAll("\\s", ""),
Expand Down
19 changes: 14 additions & 5 deletions .test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java11.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

def loadTestConfigurations = { mode, isStreaming ->
[
[
Expand All @@ -46,7 +48,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 200000000,
"keySizeBytes": 1,
"valueSizeBytes": 9
"valueSizeBytes": 9,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
Expand Down Expand Up @@ -77,7 +80,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
Expand Down Expand Up @@ -109,7 +113,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000,
"keySizeBytes": 10000,
"valueSizeBytes": 90000
"valueSizeBytes": 90000,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
Expand Down Expand Up @@ -141,7 +146,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 5000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 4,
Expand Down Expand Up @@ -172,7 +178,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 2500000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 8,
Expand Down Expand Up @@ -204,6 +211,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 200,
"hotKeyFraction": 1
}
Expand Down Expand Up @@ -237,6 +245,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 10,
"hotKeyFraction": 1
}
Expand Down
19 changes: 14 additions & 5 deletions .test-infra/jenkins/job_LoadTests_GBK_Dataflow_V2_Java17.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

def loadTestConfigurations = { mode, isStreaming ->
[
[
Expand All @@ -46,7 +48,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 200000000,
"keySizeBytes": 1,
"valueSizeBytes": 9
"valueSizeBytes": 9,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
Expand Down Expand Up @@ -77,7 +80,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
Expand Down Expand Up @@ -109,7 +113,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 20000,
"keySizeBytes": 10000,
"valueSizeBytes": 90000
"valueSizeBytes": 90000,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 1,
Expand Down Expand Up @@ -141,7 +146,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 5000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 4,
Expand Down Expand Up @@ -172,7 +178,8 @@ def loadTestConfigurations = { mode, isStreaming ->
{
"numRecords": 2500000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
fanout : 8,
Expand Down Expand Up @@ -204,6 +211,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 200,
"hotKeyFraction": 1
}
Expand Down Expand Up @@ -237,6 +245,7 @@ def loadTestConfigurations = { mode, isStreaming ->
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90,
"forceNumInitialBundles": 1,
"numHotKeys": 10,
"hotKeyFraction": 1
}
Expand Down
14 changes: 10 additions & 4 deletions .test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java11.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

def commonLoadTestConfig = { jobType, isStreaming ->
[
[
Expand All @@ -46,7 +48,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 10,
Expand Down Expand Up @@ -78,7 +81,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 200,
Expand Down Expand Up @@ -111,7 +115,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
Expand Down Expand Up @@ -144,7 +149,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
Expand Down
14 changes: 10 additions & 4 deletions .test-infra/jenkins/job_LoadTests_ParDo_Dataflow_V2_Java17.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import PhraseTriggeringPostCommitBuilder
import CronJobBuilder
import InfluxDBCredentialsHelper

// TODO(BEAM-14229): remove forceNumInitialBundles once source issue resolved.

def commonLoadTestConfig = { jobType, isStreaming ->
[
[
Expand All @@ -46,7 +48,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 10,
Expand Down Expand Up @@ -78,7 +81,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 200,
Expand Down Expand Up @@ -111,7 +115,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
Expand Down Expand Up @@ -144,7 +149,8 @@ def commonLoadTestConfig = { jobType, isStreaming ->
{
"numRecords": 20000000,
"keySizeBytes": 10,
"valueSizeBytes": 90
"valueSizeBytes": 90,
"forceNumInitialBundles": 1
}
""".trim().replaceAll("\\s", ""),
iterations : 1,
Expand Down

0 comments on commit 178441d

Please sign in to comment.