Skip to content

Commit 35cd5f6

Browse files
martin-matmartin-mat
committed
Workload tests adapted to centralized result logging
Signed-off-by: Martin Matyas <martin.matyas@gmail.com>
1 parent a1797d3 commit 35cd5f6

File tree

10 files changed

+481
-865
lines changed

10 files changed

+481
-865
lines changed

‎embedded_files/points.yml

+63-6
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,34 @@
88
neutral: 0
99

1010
- name: reasonable_image_size
11+
emoji: "⚖👀"
1112
tags: [microservice, dynamic, workload, cert, normal]
1213
- name: specialized_init_system
14+
emoji: "🚀"
1315
tags: [microservice, dynamic, workload]
1416
- name: reasonable_startup_time
1517
tags: [microservice, dynamic, workload, cert, normal]
16-
- name: single_process_type
18+
- name: single_process_type
19+
emoji: "⚖👀"
1720
tags: [microservice, dynamic, workload, essential, cert]
1821
pass: 100
22+
- name: zombie_handled
23+
emoji: "⚖👀"
24+
tags: [microservice, dynamic, workload, normal]
1925
- name: service_discovery
26+
emoji: "⚖👀"
2027
tags: [microservice, dynamic, workload, cert, bonus]
2128
pass: 1
2229
fail: 0
2330
- name: shared_database
31+
emoji: "💾"
2432
tags: [microservice, dynamic, workload, cert, normal]
25-
- name: sig_term_handled
33+
- name: sig_term_handled
34+
emoji: "⚖👀"
2635
tags: [microservice, dynamic, workload, normal]
2736

2837
- name: cni_compatible
38+
emoji: "🔓🔑"
2939
tags: [compatibility, dynamic, workload, cert, normal]
3040
# - name: cni_spec
3141
# tags: compatibility, dynamic
@@ -41,23 +51,29 @@
4151
#- name: check_reaped
4252
# tags: state, dynamic, configuration
4353

44-
- name: privileged
54+
- name: privileged
55+
emoji: "🔓🔑"
4556
tags: [security, dynamic, workload]
4657
# required: true
47-
- name: privilege_escalation
58+
- name: privilege_escalation
59+
emoji: "🔓🔑"
4860
tags: [security, dynamic, workload, cert, normal]
4961
- name: symlink_file_system
62+
emoji: "🔓🔑"
5063
tags: [security, dynamic, workload, cert, normal]
5164
- name: application_credentials
65+
emoji: "🔓🔑"
5266
tags: [security, dynamic, workload, cert, normal]
5367
- name: host_network
68+
emoji: "🔓🔑"
5469
tags: [security, dynamic, workload, cert, normal]
5570
#- name: shells
5671
# tags: security, dynamic
5772
#- name: protected_access
5873
# tags: security, dynamic
5974

60-
- name: increase_decrease_capacity
75+
- name: increase_decrease_capacity
76+
emoji: "📦📈📉"
6177
tags: [compatibility, dynamic, workload, essential, cert]
6278
pass: 100
6379
#- name: small_autoscaling
@@ -67,28 +83,36 @@
6783
# - name: network_chaos
6884
# tags: resilience, dynamic, workload
6985
- name: pod_network_latency
86+
emoji: "🗡️💀♻"
7087
tags: [resilience, dynamic, workload, cert, bonus]
7188
pass: 1
7289
fail: 0
7390
- name: pod_network_corruption
91+
emoji: "🗡️💀♻"
7492
tags: [resilience, dynamic, workload, cert, bonus]
7593
pass: 1
7694
fail: 0
7795
- name: pod_network_duplication
96+
emoji: "🗡️💀♻"
7897
tags: [resilience, dynamic, workload, cert, bonus]
7998
pass: 1
8099
fail: 0
81100
- name: pod_delete
101+
emoji: "🗡️💀♻"
82102
tags: [resilience, dynamic, workload, cert, normal]
83103
- name: pod_io_stress
104+
emoji: "🗡️💀♻"
84105
tags: [resilience, dynamic, workload, cert, bonus]
85106
pass: 1
86107
fail: 0
87108
- name: pod_memory_hog
109+
emoji: "🗡️💀♻"
88110
tags: [resilience, dynamic, workload, cert, normal]
89111
- name: disk_fill
112+
emoji: "🗡️💀♻"
90113
tags: [resilience, dynamic, workload, cert, normal]
91114
- name: pod_dns_error
115+
emoji: "🗡️💀♻"
92116
tags: [resilience, dynamic, workload, cert, bonus]
93117
pass: 1
94118
fail: 0
@@ -98,17 +122,22 @@
98122
#- name: versioned_helm_chart
99123
# tags: configuration, dynamic, workload
100124
- name: versioned_tag
125+
emoji: "🏷️"
101126
tags: [configuration, dynamic, workload]
102127
- name: ip_addresses
128+
emoji: "📶🏃⏲️"
103129
pass: 0
104130
fail: -1
105131
tags: [configuration, static, workload]
106132
- name: operator_installed
133+
emoji: "⚖️👀"
107134
tags: [configuration, dynamic, workload, cert, bonus]
108135
- name: liveness
136+
emoji: "⎈🧫"
109137
tags: [resilience, dynamic, workload, essential, cert]
110138
pass: 100
111139
- name: readiness
140+
emoji: "⎈🧫"
112141
tags: [resilience, dynamic, workload, essential, cert]
113142
pass: 100
114143
#- name: no_volume_with_configuration
@@ -130,10 +159,12 @@
130159
tags: [configuration, dynamic, workload, essential, cert]
131160
pass: 100
132161
- name: secrets_used
162+
emoji: "🧫"
133163
tags: [configuration, dynamic, workload, cert, bonus]
134164
pass: 1
135165
fail: 0
136166
- name: immutable_configmap
167+
emoji: "⚖️"
137168
tags: [configuration, dynamic, workload, cert, bonus]
138169
pass: 1
139170
fail: 0
@@ -150,10 +181,13 @@
150181
# tags: observability, dynamic, workload
151182

152183
- name: helm_deploy
184+
emoji: "⚙🛠️⬆☁"
153185
tags: [compatibility, dynamic, workload, cert, normal]
154186
- name: helm_chart_valid
187+
emoji: "⎈📝☑"
155188
tags: [compatibility, dynamic, workload, cert, normal]
156189
- name: helm_chart_published
190+
emoji: "⎈📦🌐"
157191
tags: [compatibility, dynamic, workload, cert, normal]
158192

159193
# - name: chaos_network_loss
@@ -164,21 +198,26 @@
164198
# tags: resilience, dynamic, workload
165199

166200
- name: volume_hostpath_not_found
201+
emoji: "💾"
167202
tags: [state, dynamic, workload]
168203
- name: no_local_volume_configuration
204+
emoji: "💾"
169205
tags: [state, dynamic, workload, cert, bonus]
170206
pass: 1
171207
fail: 0
172-
- name: elastic_volumes
208+
- name: elastic_volumes
209+
emoji: "🧫"
173210
tags: [state, dynamic, workload, cert, bonus]
174211
pass: 1
175212
fail: 0
176213
- name: database_persistence
214+
emoji: "🧫"
177215
tags: [state, dynamic, workload]
178216
pass5: 5
179217
pass3: 3
180218
fail: -1
181219
- name: node_drain
220+
emoji: "🗡️💀♻"
182221
tags: [state, dynamic, workload, essential, cert]
183222
pass: 100
184223

@@ -207,46 +246,57 @@
207246
tags: ["platform", "platform:security", "dynamic"]
208247

209248
- name: service_account_mapping
249+
emoji: "🔓🔑"
210250
tags: [security, dynamic, workload, cert, normal]
211251

212252
- name: privileged_containers
253+
emoji: "🔓🔑"
213254
tags: [security, dynamic, workload, essential, cert]
214255
pass: 100
215256

216257
- name: non_root_containers
258+
emoji: "🔓🔑"
217259
tags: [security, dynamic, workload, essential, cert]
218260
pass: 100
219261

220262
- name: host_pid_ipc_privileges
263+
emoji: "🔓🔑"
221264
tags: [security, dynamic, workload, cert, normal]
222265

223266
- name: linux_hardening
267+
emoji: "🔓🔑"
224268
tags: [security, dynamic, workload, cert, bonus]
225269
pass: 1
226270
fail: 0
227271

228272
- name: resource_policies
273+
emoji: "🔓🔑"
229274
tags: [security, dynamic, workload, cert, essential]
230275
pass: 100
231276

232277
- name: immutable_file_systems
278+
emoji: "🔓🔑"
233279
tags: [security, dynamic, workload, cert, bonus]
234280
pass: 1
235281
fail: 0
236282

237283
- name: hostpath_mounts
284+
emoji: "🔓🔑"
238285
tags: [security, dynamic, workload, essential, cert]
239286
pass: 100
240287

241288
- name: ingress_egress_blocked
289+
emoji: "🔓🔑"
242290
tags: [security, dynamic, workload, cert, bonus]
243291
pass: 1
244292
fail: 0
245293

246294
- name: insecure_capabilities
295+
emoji: "🔓🔑"
247296
tags: [security, dynamic, workload, cert, normal]
248297

249298
- name: sysctls
299+
emoji: "🔓🔑"
250300
tags: [security, dynamic, workload, cert, normal]
251301

252302
- name: log_output
@@ -274,29 +324,36 @@
274324
pass: 1
275325
fail: 0
276326
- name: alpha_k8s_apis
327+
emoji: "⭕🔍"
277328
tags: [configuration, dynamic, workload]
278329

279330
- name: container_sock_mounts
331+
emoji: "🔓🔑"
280332
tags: [security, dynamic, workload, essential, cert]
281333
pass: 100
282334

283335
- name: require_labels
336+
emoji: "🏷️"
284337
tags: [configuration, dynamic, workload, cert, normal]
285338

286339
- name: helm_tiller
287340
tags: ["platform", "platform:security", "dynamic"]
288341

289342
- name: external_ips
343+
emoji: "🔓🔑"
290344
tags: [security, dynamic, workload, cert, normal]
291345

292346
- name: selinux_options
347+
emoji: "🔓🔑"
293348
tags: [security, dynamic, workload, essential, cert]
294349
pass: 100
295350

296351
- name: default_namespace
352+
emoji: "🏷️"
297353
tags: [configuration, dynamic, workload, cert, normal]
298354

299355
- name: latest_tag
356+
emoji: "🏷️"
300357
tags: [configuration, dynamic, workload, essential, cert]
301358
pass: 100
302359

‎src/tasks/utils/points.cr

+1-1
Original file line numberDiff line numberDiff line change
@@ -556,7 +556,7 @@ module CNFManager
556556
Log.debug { "task #{task} emoji: #{md["emoji"]?}" }
557557
resp = md["emoji"]
558558
else
559-
resp = [] of String
559+
resp = ""
560560
end
561561
end
562562

‎src/tasks/workload/5g_validator.cr

+11-25
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,7 @@ end
1717
desc "Test if a 5G core is valid"
1818
task "smf_upf_core_validator" do |t, args|
1919
#todo change to 5g_core_validator
20-
CNFManager::Task.task_runner(args) do |args, config|
21-
task_start_time = Time.utc
22-
testsuite_task = "smf_upf_core_validator"
23-
Log.for(testsuite_task).info { "Starting test" }
20+
CNFManager::Task.task_runner(args, task: t) do |args, config|
2421

2522
# todo add other resilience and compatiblity tests
2623

@@ -33,19 +30,15 @@ end
3330

3431
desc "Test if a 5G core has SMF/UPF heartbeat"
3532
task "smf_upf_heartbeat" do |t, args|
36-
CNFManager::Task.task_runner(args) do |args, config|
37-
task_start_time = Time.utc
38-
testsuite_task = "smf_upf_heartbeat"
39-
Log.for(testsuite_task).info { "Starting test" }
40-
Log.for(testsuite_task).info { "named args: #{args.named}" }
33+
CNFManager::Task.task_runner(args, task: t) do |args, config|
34+
Log.for(t.name).info { "named args: #{args.named}" }
4135
baseline_count : Int32 | Float64 | String | Nil
4236
if args.named["baseline_count"]?
4337
baseline_count = args.named["baseline_count"].to_i
4438
else
4539
baseline_count = nil
4640
end
4741

48-
Log.debug { "cnf_config: #{config}" }
4942
suci_found : Bool | Nil
5043
smf = config.cnf_config[:smf_label]?
5144
upf = config.cnf_config[:upf_label]?
@@ -133,24 +126,18 @@ task "smf_upf_heartbeat" do |t, args|
133126
end
134127

135128
#todo move this to validator code code
136-
if heartbeat_found
137-
resp = upsert_passed_task(testsuite_task,"✔️ PASSED: Chaos service degradation is less than 50%.", task_start_time)
129+
if heartbeat_found
130+
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "Chaos service degradation is less than 50%")
138131
else
139-
resp = upsert_failed_task(testsuite_task, "✖️ FAILED: Chaos service degradation is more than 50%.", task_start_time)
132+
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Chaos service degradation is more than 50%")
140133
end
141-
resp
142134
end
143135
end
144136

145137
#todo move to 5g test files
146138
desc "Test if a 5G core supports SUCI Concealment"
147-
task "suci_enabled" do |_, args|
148-
CNFManager::Task.task_runner(args) do |args, config|
149-
task_start_time = Time.utc
150-
testsuite_task = "suci_enabled"
151-
Log.for(testsuite_task).info { "Starting test" }
152-
153-
Log.debug { "cnf_config: #{config}" }
139+
task "suci_enabled" do |t, args|
140+
CNFManager::Task.task_runner(args, task: t) do |args, config|
154141
suci_found : Bool | Nil
155142
core = config.cnf_config[:amf_label]?
156143
Log.info { "core: #{core}" }
@@ -192,12 +179,11 @@ task "suci_enabled" do |_, args|
192179
end
193180

194181

195-
if suci_found
196-
resp = upsert_passed_task(testsuite_task,"✔️ PASSED: Core uses SUCI 5g authentication", task_start_time)
182+
if suci_found
183+
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "Core uses SUCI 5g authentication")
197184
else
198-
resp = upsert_failed_task(testsuite_task, "✖️ FAILED: Core does not use SUCI 5g authentication", task_start_time)
185+
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Core does not use SUCI 5g authentication")
199186
end
200-
resp
201187
ensure
202188
Helm.delete("ueransim")
203189
ClusterTools.uninstall

0 commit comments

Comments
 (0)