Skip to content
This repository was archived by the owner on Sep 3, 2023. It is now read-only.

Commit 6eb4906

Browse files
isaac-baekisaac-baek
authored
[AWIBOF-7506] Fix queue type logging (#8)
- Remove node sampling option Signed-off-by: isaac.baek <isaac.baek@samsung.com>
1 parent 1916ca4 commit 6eb4906

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+152
-951
lines changed

‎CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set(CMAKE_C_COMPILER "gcc")
44

55
project(
66
AIR
7-
VERSION 0.8.5
7+
VERSION 0.8.7
88
DESCRIPTION "Analytics In Real-time"
99
LANGUAGES CXX C
1010
)

‎config/system_test.cfg

+1-12
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,6 @@ User could change or add/remove sentences.
4646
If a sentence has not this option in GROUP and NODE paragraph, that follows DEFAULT paragraph's rule.
4747
User can control this option by air_cli, air_tui at run-time.
4848
49-
* NodeSamplingRatio {Mandatory, Valid value: 1~99999 (probability = 1/N)}
50-
: Queue type of NODE sentences may effect this sampling rule. For example, when this value is set to 1000,
51-
airlog(...) API actually collect raw data once a thousand.
52-
User can control this option by air_cli, air_tui at run-time.
53-
5449
* NodeIndexSize {Mandatory, Valid value: 1~999}
5550
: NodeIndex could be used to profiling numerical separation for the same NODE sentence.
5651
For example, when NodeIndexSize value is set to 10, user can distinguish 10 maximum different(hashed)
@@ -60,7 +55,7 @@ User could change or add/remove sentences.
6055
6156
[DEFAULT]
6257
"AirBuild: True, StreamingInterval: 3, FileWrite: On, RemainingFileCount: 2,
63-
NodeBuild: True, NodeRun: On, NodeSamplingRatio: 1000, NodeIndexSize: 10"
58+
NodeBuild: True, NodeRun: On, NodeIndexSize: 10"
6459
[/DEFAULT]
6560
6661
@@ -81,9 +76,6 @@ User could change or add/remove sentences.
8176
* NodeRun {Optional, Valid value: On/Off}
8277
: Same as default rule
8378
84-
* NodeSamplingRatio {Optional, Valid value: 1~99999 (probability = 1/N)}
85-
: Same as default rule
86-
8779
* NodeIndexSize {Optional, Valid value: 1~999}
8880
: Same as default rule
8981
@@ -184,9 +176,6 @@ User could change or add/remove sentences.
184176
* Run {Optional, Valid value: On/Off}
185177
: Same as DEFAULT rule
186178
187-
* SamplingRatio {Optional, Valid value: 1~99999 (probability = 1/N)}
188-
: Same as DEFAULT rule
189-
190179
* IndexSize {Optional, Valid value: 1~999}
191180
: Same as DEFAULT rule
192181

‎config/unit_test.cfg

+5-6
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,15 @@ This configuration is used for unit test
44
55
[DEFAULT]
66
"StreamingInterval:1, FileWrite: On, AirBuild:True,
7-
NodeBuild:False, NodeRun:Off, NodeSamplingRatio: 1000,
8-
NodeIndexSize:32, RemainingFileCount: 3"
7+
NodeBuild:False, NodeRun:Off, NodeIndexSize:32, RemainingFileCount: 3"
98
[/DEFAULT]
109
1110
[GROUP]
1211
"Group: POS "
1312
"Group: POS_JOURNAL ,
1413
NodeBuild: True, NodeIndexSize: 100"
1514
"Group: POS_IO , NodeBuild: True, NodeRun: Off"
16-
"Group: POS_META, NodeBuild: False, NodeRun: On, NodeSamplingRatio: 100"
15+
"Group: POS_META, NodeBuild: False, NodeRun: On"
1716
"Group: POS_RSC, NodeBuild: False, NodeRun: On"
1817
[/GROUP]
1918
@@ -48,11 +47,11 @@ NodeBuild: True, NodeIndexSize: 100"
4847
4948
"Node: Q_SUBMIT, Type: QUEUE, Group: POS, Filter: Basic"
5049
"Node: Q_REACTOR, Type: QUEUE, Build: True, Group: POS, Filter: Basic"
51-
"Node: Q_EVENT, Type: QUEUE, Build : True, Run: Off, SamplingRatio: 1000, Group: POS, Filter: Basic"
50+
"Node: Q_EVENT, Type: QUEUE, Build : True, Run: Off, Group: POS, Filter: Basic"
5251
"Node: Q_SCHEDULING, Filter: Basic
53-
, Type: QUEUE, Build : True, Run : Off, SamplingRatio: 1000,
52+
, Type: QUEUE, Build : True, Run : Off,
5453
Group: POS"
55-
"Node: Q_IOWORER, Type: QUEUE, Filter: Basic, Build: True, Run: Off, SamplingRatio: 10,
54+
"Node: Q_IOWORER, Type: QUEUE, Filter: Basic, Build: True, Run: Off,
5655
Group: POS"
5756
5857
"Node: UTIL_PSD, Filter: Basic, Type: Utilization, Group: POS_RSC"

‎src/collection/CollectionManager.cpp

-71
Original file line numberDiff line numberDiff line change
@@ -165,13 +165,6 @@ collection::CollectionManager::UpdateCollection(
165165
result = _UpdateInit(type1, type2, value1, value2);
166166
break;
167167

168-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE_WITH_RANGE)):
169-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE_ALL)):
170-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE)):
171-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE_WITH_GROUP)):
172-
result = _UpdateSamplingRate(type1, type2, value1, value2);
173-
break;
174-
175168
default:
176169
result = -1;
177170
break;
@@ -350,67 +343,3 @@ collection::CollectionManager::_UpdateInit(
350343

351344
return 0;
352345
}
353-
354-
int
355-
collection::CollectionManager::_UpdateNodeSamplingRate(
356-
uint32_t node_index, uint32_t ratio)
357-
{
358-
if (collector[node_index] != nullptr)
359-
{
360-
return collector[node_index]->SetSamplingRate(ratio);
361-
}
362-
return 0;
363-
}
364-
365-
int
366-
collection::CollectionManager::_UpdateSamplingRate(
367-
uint32_t type1, uint32_t type2, uint32_t value1, uint32_t value2)
368-
{
369-
// value1: rate, value2: node info(node id, range, group id)
370-
uint32_t lower_bit = value2 & 0x0000FFFF;
371-
uint32_t upper_bit = (value2 >> 16) & 0x0000FFFF;
372-
int result {0};
373-
374-
switch (type2)
375-
{
376-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE)):
377-
result = _UpdateNodeSamplingRate(value2, value1);
378-
break;
379-
380-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE_WITH_RANGE)):
381-
for (uint32_t i = upper_bit; i <= lower_bit; i++)
382-
{
383-
result = _UpdateNodeSamplingRate(i, value1);
384-
if (result != 0)
385-
{
386-
break;
387-
}
388-
}
389-
break;
390-
391-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE_WITH_GROUP)):
392-
for (uint32_t i = 0;
393-
i < cfg::GetSentenceCount(config::ParagraphType::NODE); i++)
394-
{
395-
result = _UpdateNodeSamplingRate(i, value1);
396-
if (result != 0)
397-
{
398-
break;
399-
}
400-
}
401-
break;
402-
403-
case (to_dtype(pi::Type2::SET_SAMPLING_RATE_ALL)):
404-
for (uint32_t i = 0; i < MAX_NID_SIZE; i++)
405-
{
406-
result = _UpdateNodeSamplingRate(i, value1);
407-
if (result != 0)
408-
{
409-
break;
410-
}
411-
}
412-
break;
413-
}
414-
415-
return result;
416-
}

‎src/collection/CollectionManager.h

-3
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,6 @@ class CollectionManager
117117
void _InitNode(uint32_t node_index);
118118
int _UpdateInit(
119119
uint32_t type1, uint32_t type2, uint32_t value1, uint32_t value2);
120-
int _UpdateNodeSamplingRate(uint32_t node_index, uint32_t ratio);
121-
int _UpdateSamplingRate(
122-
uint32_t type1, uint32_t type2, uint32_t value1, uint32_t value2);
123120

124121
Collector* collector[cfg::GetSentenceCount(config::ParagraphType::NODE)] {
125122
nullptr,

‎src/collection/Collector.h

-5
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,6 @@ class Collector
3636
{
3737
}
3838
virtual ~Collector(void);
39-
int
40-
SetSamplingRate(uint32_t rate)
41-
{
42-
return writer->SetSamplingRate(rate);
43-
}
4439
inline void
4540
LogData(lib::Data* data, uint64_t value)
4641
{

‎src/collection/writer/CountWriter.h

-5
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,6 @@ class CountWriter : public Writer
6060
count_data->period_num_req_positive += 1;
6161
}
6262
}
63-
int
64-
SetSamplingRate(uint32_t rate) override
65-
{
66-
return 0;
67-
}
6863

6964
private:
7065
const uint64_t MAX_INT64_VALUE {9223372036854775807};

‎src/collection/writer/HistogramWriter.h

-5
Original file line numberDiff line numberDiff line change
@@ -111,11 +111,6 @@ class HistogramWriter : public Writer
111111

112112
hist_data->access = true;
113113
}
114-
int
115-
SetSamplingRate(uint32_t rate) override
116-
{
117-
return 0;
118-
}
119114
};
120115

121116
} // namespace collection

‎src/collection/writer/LatencyWriter.h

-6
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,6 @@ class LatencyWriter : public Writer
104104
return true;
105105
}
106106

107-
int
108-
SetSamplingRate(uint32_t rate) override
109-
{
110-
return 0;
111-
}
112-
113107
private:
114108
};
115109

‎src/collection/writer/PerformanceWriter.h

-5
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,6 @@ class PerformanceWriter : public Writer
6262
}
6363
}
6464
}
65-
int
66-
SetSamplingRate(uint32_t rate) override
67-
{
68-
return 0;
69-
}
7065
};
7166

7267
} // namespace collection

‎src/collection/writer/QueueWriter.cpp

-6
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,3 @@
2323
*/
2424

2525
#include "src/collection/writer/QueueWriter.h"
26-
27-
void
28-
collection::QueueWriter::_UpdateRand(void)
29-
{
30-
mersenne.seed(std::rand());
31-
}

‎src/collection/writer/QueueWriter.h

-74
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,6 @@
2525
#ifndef AIR_COLLECTION_QUEUE_WRITER_H
2626
#define AIR_COLLECTION_QUEUE_WRITER_H
2727

28-
#include <cmath>
29-
#include <random>
30-
3128
#include "src/collection/writer/Writer.h"
3229
#include "src/lib/Data.h"
3330

@@ -38,7 +35,6 @@ class QueueWriter : public Writer
3835
public:
3936
QueueWriter(void)
4037
{
41-
_UpdateRand();
4238
}
4339
virtual ~QueueWriter(void)
4440
{
@@ -51,10 +47,6 @@ class QueueWriter : public Writer
5147
return;
5248
}
5349
lib::QueueData* queue_data = static_cast<lib::QueueData*>(data);
54-
if (false == IsSampling(queue_data))
55-
{
56-
return;
57-
}
5850
queue_data->access = true;
5951
queue_data->period_qd_sum += q_depth;
6052
queue_data->period_num_req++;
@@ -63,74 +55,8 @@ class QueueWriter : public Writer
6355
queue_data->period_qd_max = q_depth;
6456
}
6557
}
66-
int
67-
SetSamplingRate(uint32_t rate) override
68-
{
69-
if (rate >= MIN_RATIO && rate <= MAX_RATIO)
70-
{
71-
sampling_rate = rate;
72-
_UpdateRand();
73-
74-
return 0;
75-
}
76-
else
77-
{
78-
return -2; // boundray error
79-
}
80-
}
81-
inline bool
82-
IsSampling(lib::QueueData* queue_data)
83-
{
84-
bool result {false};
85-
if (sampling_rate != queue_data->sampling_rate)
86-
{
87-
queue_data->sampling_rate = sampling_rate;
88-
queue_data->logging_point = 0;
89-
queue_data->num_called = 0;
90-
queue_data->mersenne = mersenne;
91-
}
92-
if (queue_data->logging_point == queue_data->num_called)
93-
{
94-
result = true;
95-
}
96-
else
97-
{
98-
result = false;
99-
}
100-
if (queue_data->num_called % sampling_rate == 0)
101-
{
102-
queue_data->num_called = 0;
103-
if (sampling_rate < 10)
104-
{
105-
queue_data->logging_point = sampling_rate;
106-
}
107-
else
108-
{
109-
uint32_t rand_uint = std::uniform_int_distribution<uint32_t> {
110-
0, sampling_rate}(queue_data->mersenne);
111-
float rand_float = rand_uint * 0.8;
112-
queue_data->logging_point = rand_float + (sampling_rate * 0.1);
113-
}
114-
}
115-
queue_data->num_called++;
116-
return result;
117-
}
118-
uint32_t
119-
GetLoggingPoint(lib::QueueData* queue_data)
120-
{
121-
return queue_data->logging_point;
122-
}
12358

12459
private:
125-
void _UpdateRand(void);
126-
static const uint32_t PADDING {10};
127-
static const uint32_t DEFAULT_RATIO {1000}; // tmp. 1000
128-
static const uint32_t MIN_RATIO {1};
129-
static const uint32_t MAX_RATIO {10000};
130-
131-
uint32_t sampling_rate {DEFAULT_RATIO};
132-
std::random_device rand;
133-
std::mt19937 mersenne;
13460
};
13561

13662
} // namespace collection

‎src/collection/writer/UtilizationWriter.h

-5
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,6 @@ class UtilizationWriter : public Writer
5151

5252
util_data->period_usage += usage;
5353
}
54-
int
55-
SetSamplingRate(uint32_t rate) override
56-
{
57-
return 0;
58-
}
5954
};
6055

6156
} // namespace collection

‎src/collection/writer/Writer.h

-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ class Writer
3737
}
3838
virtual void LogData(lib::Data* data, uint64_t value) = 0;
3939
virtual void InformInit(lib::AccData* data);
40-
virtual int SetSamplingRate(uint32_t rate) = 0;
4140
};
4241

4342
} // namespace collection

‎src/config/ConfigChecker.h

+6-6
Original file line numberDiff line numberDiff line change
@@ -182,18 +182,18 @@ class ConfigChecker
182182

183183
private:
184184
static constexpr air::string_view default_keys[NUM_DEFAULT_KEY] {"AirBuild",
185-
"NodeBuild", "NodeRun", "NodeSamplingRatio", "NodeIndexSize",
186-
"StreamingInterval", "FileWrite", "RemainingFileCount"};
185+
"NodeBuild", "NodeRun", "NodeIndexSize", "StreamingInterval", "FileWrite",
186+
"RemainingFileCount"};
187187
static constexpr air::string_view group_keys[NUM_GROUP_KEY] {
188-
"Group", "NodeBuild", "NodeRun", "NodeSamplingRatio", "NodeIndexSize"};
188+
"Group", "NodeBuild", "NodeRun", "NodeIndexSize"};
189189
static constexpr air::string_view filter_keys[NUM_FILTER_KEY] {
190190
"Filter", "Item"};
191191
static constexpr air::string_view bucket_keys[NUM_BUCKET_KEY] {
192192
"Bucket", "Bounds", "Scale"};
193-
static constexpr air::string_view node_keys[NUM_NODE_KEY] {"Node", "Type",
194-
"Group", "Filter", "Build", "Run", "SamplingRatio", "IndexSize", "Bucket"};
193+
static constexpr air::string_view node_keys[NUM_NODE_KEY] {
194+
"Node", "Type", "Group", "Filter", "Build", "Run", "IndexSize", "Bucket"};
195195

196-
static constexpr uint32_t num_mandatory_list[5] {8, 1, 2, 3, 4};
196+
static constexpr uint32_t num_mandatory_list[5] {7, 1, 2, 3, 4};
197197
};
198198

199199
} // namespace config

0 commit comments

Comments
 (0)