##// END OF EJS Templates
GIT_REPOSITORIES » LPP » Users » mperrinel » SciQLop-fork
RSS

Fork of HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop

Modif test fuzzing to enable custom test on specific operation
perrinel -
r1396:115d3bfd69ec
parent child
Show More
Side by Side Unified
Context file:
r1396:115d3bfd69ec Loading diff...:
Show file before | Show file after | Hide comments
@@ -13,6 +13,7 const QString PROVIDER_PROPERTY = QStringLiteral("provider");
13 const QString OPERATION_DELAY_BOUNDS_PROPERTY = QStringLiteral("operationDelays");
13 const QString OPERATION_DELAY_BOUNDS_PROPERTY = QStringLiteral("operationDelays");
14 const QString VALIDATORS_PROPERTY = QStringLiteral("validators");
14 const QString VALIDATORS_PROPERTY = QStringLiteral("validators");
15 const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY = QStringLiteral("validationFrequencyBounds");
15 const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY = QStringLiteral("validationFrequencyBounds");
16 const QString CUSTOM_OPERATIONS_PROPERTY = QStringLiteral("customOperations");
16
17
17 // //////////// //
18 // //////////// //
18 // FuzzingState //
19 // FuzzingState //
Show file before | Show file after | Hide comments
@@ -62,6 +62,9 extern const QString VALIDATORS_PROPERTY;
62 /// Min/max number of operations to execute before calling validation of the current test's state
62 /// Min/max number of operations to execute before calling validation of the current test's state
63 extern const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY;
63 extern const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY;
64
64
65 /// Custom operations to execute instead of random operations
66 extern const QString CUSTOM_OPERATIONS_PROPERTY;
67
65 // /////// //
68 // /////// //
66 // Structs //
69 // Structs //
67 // /////// //
70 // /////// //
Show file before | Show file after | Hide comments
@@ -57,6 +57,12 struct OperationProperty {
57 using VariableOperation = std::pair<VariableId, std::shared_ptr<IFuzzingOperation> >;
57 using VariableOperation = std::pair<VariableId, std::shared_ptr<IFuzzingOperation> >;
58 using VariablesOperations = std::vector<VariableOperation>;
58 using VariablesOperations = std::vector<VariableOperation>;
59
59
60 struct CustomVariableOperation {
61 VariableOperation m_VariableOperation{};
62 bool m_WaitAcquisition{false};
63 };
64 using CustomVariablesOperations = std::vector<CustomVariableOperation>;
65
60 using OperationsTypes = std::map<FuzzingOperationType, OperationProperty>;
66 using OperationsTypes = std::map<FuzzingOperationType, OperationProperty>;
61 using OperationsPool = std::map<std::shared_ptr<IFuzzingOperation>, OperationProperty>;
67 using OperationsPool = std::map<std::shared_ptr<IFuzzingOperation>, OperationProperty>;
62
68
@@ -70,27 +76,161 using Validators = std::vector<std::shared_ptr<IFuzzingValidator> >;
70 const auto ACQUISITION_TIMEOUT_DEFAULT_VALUE = 30000;
76 const auto ACQUISITION_TIMEOUT_DEFAULT_VALUE = 30000;
71 const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100;
77 const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100;
72 const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1;
78 const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1;
73 const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 1;
79 const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 2;
74 const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue(
80 const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue(
75 OperationsTypes{{FuzzingOperationType::CREATE, {1., true}},
81 OperationsTypes{{FuzzingOperationType::CREATE, {400., true}},
76 {FuzzingOperationType::DELETE, {0.1}}, // Delete operation is less frequent
82 {FuzzingOperationType::DELETE, {0.0}}, // Delete operation is less frequent
77 {FuzzingOperationType::PAN_LEFT, {1.}},
83 {FuzzingOperationType::PAN_LEFT, {1.}},
78 {FuzzingOperationType::PAN_RIGHT, {1.}},
84 {FuzzingOperationType::PAN_RIGHT, {1.}},
79 {FuzzingOperationType::ZOOM_IN, {1.}},
85 {FuzzingOperationType::ZOOM_IN, {1.}},
80 {FuzzingOperationType::ZOOM_OUT, {1.}},
86 {FuzzingOperationType::ZOOM_OUT, {1.}},
81 {FuzzingOperationType::SYNCHRONIZE, {0.8}},
87 {FuzzingOperationType::SYNCHRONIZE, {4000.0}},
82 {FuzzingOperationType::DESYNCHRONIZE, {0.4}}});
88 {FuzzingOperationType::DESYNCHRONIZE, {0.4}}});
83 const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2;
89 const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2;
84
90
85 /// Min/max delays between each operation (in ms)
91 /// Min/max delays between each operation (in ms)
86 const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(100, 3000));
92 const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(3000, 3000));
87
93
88 /// Validators for the tests (executed in the order in which they're defined)
94 /// Validators for the tests (executed in the order in which they're defined)
89 const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue(
95 const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue(
90 ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}});
96 ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}});
91
97
92 /// Min/max number of operations to execute before calling validation
98 /// Min/max number of operations to execute before calling validation
93 const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(1, 10));
99 const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(8, 8));
100
101
102 // /////// //////
103 // CUSTOM CASE //
104 // /////// //////
105
106 auto op = [](auto type){
107 return FuzzingOperationFactory::create(type);
108 };
109
110
111 const QVariant CUSTOM_CASE_ONE =QVariant::fromValue(CustomVariablesOperations{{{0, op(FuzzingOperationType::CREATE)}, true},
112 {{0, op(FuzzingOperationType::SYNCHRONIZE)}},
113 {{1, op(FuzzingOperationType::CREATE)}, true},
114 {{1, op(FuzzingOperationType::SYNCHRONIZE)}},
115 {{0, op(FuzzingOperationType::PAN_LEFT)}},
116 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
117 {{0, op(FuzzingOperationType::PAN_LEFT)}},
118 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
119 {{0, op(FuzzingOperationType::PAN_LEFT)}},
120 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
121 {{0, op(FuzzingOperationType::PAN_LEFT)}},
122 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
123 {{0, op(FuzzingOperationType::PAN_LEFT)}},
124 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
125 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
126 {{1, op(FuzzingOperationType::PAN_LEFT)}},
127 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
128 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
129 {{0, op(FuzzingOperationType::PAN_LEFT)}},
130 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
131 {{0, op(FuzzingOperationType::PAN_LEFT)}},
132 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
133 {{0, op(FuzzingOperationType::PAN_LEFT)}},
134 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
135 {{0, op(FuzzingOperationType::PAN_LEFT)}},
136 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
137 {{0, op(FuzzingOperationType::PAN_LEFT)}},
138 {{1, op(FuzzingOperationType::PAN_LEFT)}},
139 {{0, op(FuzzingOperationType::PAN_LEFT)}},
140 {{1, op(FuzzingOperationType::PAN_LEFT)}},
141 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
142 {{1, op(FuzzingOperationType::PAN_LEFT)}},
143 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
144 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
145 {{0, op(FuzzingOperationType::PAN_LEFT)}},
146 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
147 {{0, op(FuzzingOperationType::PAN_LEFT)}},
148 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
149 {{0, op(FuzzingOperationType::PAN_LEFT)}},
150 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
151 {{0, op(FuzzingOperationType::PAN_LEFT)}},
152 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
153 {{0, op(FuzzingOperationType::PAN_LEFT)}},
154 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
155 {{0, op(FuzzingOperationType::PAN_LEFT)}},
156 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
157 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
158 {{1, op(FuzzingOperationType::PAN_LEFT)}},
159 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
160 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
161 {{0, op(FuzzingOperationType::PAN_LEFT)}},
162 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
163 {{0, op(FuzzingOperationType::PAN_LEFT)}},
164 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
165 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
166 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
167 {{0, op(FuzzingOperationType::PAN_LEFT)}},
168 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
169 {{0, op(FuzzingOperationType::PAN_LEFT)}},
170 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
171 {{0, op(FuzzingOperationType::PAN_LEFT)}},
172 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
173 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
174 {{1, op(FuzzingOperationType::PAN_LEFT)}},
175 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
176 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
177 {{0, op(FuzzingOperationType::PAN_LEFT)}},
178 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
179 {{0, op(FuzzingOperationType::PAN_LEFT)}},
180 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
181 {{0, op(FuzzingOperationType::PAN_LEFT)}},
182 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
183 {{0, op(FuzzingOperationType::PAN_LEFT)}},
184 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
185 {{0, op(FuzzingOperationType::PAN_LEFT)}},
186 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
187 {{0, op(FuzzingOperationType::PAN_LEFT)}},
188 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
189 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
190 {{1, op(FuzzingOperationType::PAN_LEFT)}},
191 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
192 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
193 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
194 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
195 {{0, op(FuzzingOperationType::PAN_LEFT)}},
196 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
197 {{0, op(FuzzingOperationType::PAN_LEFT)}},
198 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
199 {{0, op(FuzzingOperationType::PAN_LEFT)}},
200 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
201 {{0, op(FuzzingOperationType::PAN_LEFT)}},
202 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
203 {{0, op(FuzzingOperationType::PAN_LEFT)}},
204 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
205 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
206 {{1, op(FuzzingOperationType::PAN_LEFT)}},
207 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
208 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
209 {{0, op(FuzzingOperationType::PAN_LEFT)}},
210 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
211 });
212
213 const QVariant CUSTOM_CASE_TWO =QVariant::fromValue(CustomVariablesOperations{{{0, op(FuzzingOperationType::CREATE)}, true},
214 {{0, op(FuzzingOperationType::SYNCHRONIZE)}},
215 {{1, op(FuzzingOperationType::CREATE)}, true},
216 {{1, op(FuzzingOperationType::SYNCHRONIZE)}},
217 {{0, op(FuzzingOperationType::ZOOM_IN)}},
218 {{1, op(FuzzingOperationType::ZOOM_IN)}},
219 {{0, op(FuzzingOperationType::ZOOM_IN)}},
220 {{1, op(FuzzingOperationType::ZOOM_OUT)}},
221 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
222 {{1, op(FuzzingOperationType::ZOOM_OUT)}},
223 {{0, op(FuzzingOperationType::ZOOM_IN)}},
224 {{1, op(FuzzingOperationType::ZOOM_OUT)}},
225 {{0, op(FuzzingOperationType::ZOOM_IN)}},
226 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
227 {{1, op(FuzzingOperationType::ZOOM_IN)}},
228 {{0, op(FuzzingOperationType::ZOOM_IN)}},
229 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
230 {{1, op(FuzzingOperationType::ZOOM_IN)}},
231 {{0, op(FuzzingOperationType::ZOOM_IN)}},
232 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
233 });
94
234
95 // /////// //
235 // /////// //
96 // Methods //
236 // Methods //
@@ -193,9 +333,6 public:
193
333
194 void execute()
334 void execute()
195 {
335 {
196 qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbMaxOperations() << "operations on"
197 << nbMaxVariables() << "variable(s)...";
198
199
336
200 // Inits the count of the number of operations before the next validation
337 // Inits the count of the number of operations before the next validation
201 int nextValidationCounter = 0;
338 int nextValidationCounter = 0;
@@ -207,27 +344,49 public:
207 };
344 };
208 updateValidationCounter();
345 updateValidationCounter();
209
346
347 // Get custom operations
348 auto customOperations = m_Properties.value(CUSTOM_OPERATIONS_PROPERTY).value<CustomVariablesOperations>();
349 auto isCustomTest = !customOperations.empty();
350
351 auto nbOperations = isCustomTest ? customOperations.size() : nbMaxOperations();
352
353 qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbOperations << "operations on"
354 << nbMaxVariables() << "variable(s)...";
355
210 auto canExecute = true;
356 auto canExecute = true;
211 for (auto i = 0; i < nbMaxOperations() && canExecute; ++i) {
357 for (auto i = 0; i < nbOperations && canExecute; ++i) {
358 VariableOperation variableOperation{};
359 auto forceWait = false;
360
361 if(isCustomTest){
362 auto customOperation = customOperations.front();
363 variableOperation = customOperation.m_VariableOperation;
364 customOperations.erase(customOperations.begin());
365
366 canExecute = variableOperation.second->canExecute(variableOperation.first, m_FuzzingState);
367 forceWait = customOperation.m_WaitAcquisition;
368 } else {
212 // Retrieves all operations that can be executed in the current context
369 // Retrieves all operations that can be executed in the current context
213 VariablesOperations variableOperations{};
370 VariablesOperations variableOperations{};
214 Weights weights{};
371 Weights weights{};
215 std::tie(variableOperations, weights)
372 std::tie(variableOperations, weights)
216 = availableOperations(m_FuzzingState, operationsPool());
373 = availableOperations(m_FuzzingState, operationsPool());
217
374
375 // Of the operations available, chooses a random operation and executes it
376 variableOperation
377 = RandomGenerator::instance().randomChoice(variableOperations, weights);
218 canExecute = !variableOperations.empty();
378 canExecute = !variableOperations.empty();
379 forceWait = operationsPool().at(variableOperation.second).m_WaitAcquisition;
380 }
381
219 if (canExecute) {
382 if (canExecute) {
220 --nextValidationCounter;
383 --nextValidationCounter;
221
384
222 // Of the operations available, chooses a random operation and executes it
223 auto variableOperation
224 = RandomGenerator::instance().randomChoice(variableOperations, weights);
225
226 auto variableId = variableOperation.first;
385 auto variableId = variableOperation.first;
227 auto fuzzingOperation = variableOperation.second;
386 auto fuzzingOperation = variableOperation.second;
228
387
229 auto waitAcquisition = nextValidationCounter == 0
388 auto waitAcquisition = nextValidationCounter == 0
230 || operationsPool().at(fuzzingOperation).m_WaitAcquisition;
389 || forceWait;
231
390
232 fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController,
391 fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController,
233 m_Properties);
392 m_Properties);
@@ -293,6 +452,7 private:
293 } // namespace
452 } // namespace
294
453
295 Q_DECLARE_METATYPE(OperationsTypes)
454 Q_DECLARE_METATYPE(OperationsTypes)
455 Q_DECLARE_METATYPE(CustomVariablesOperations)
296
456
297 class TestAmdaFuzzing : public QObject {
457 class TestAmdaFuzzing : public QObject {
298 Q_OBJECT
458 Q_OBJECT
@@ -306,7 +466,7 private slots:
306 void TestAmdaFuzzing::testFuzzing_data()
466 void TestAmdaFuzzing::testFuzzing_data()
307 {
467 {
308 // Note: Comment this line to run fuzzing tests
468 // Note: Comment this line to run fuzzing tests
309 QSKIP("Fuzzing tests are disabled by default");
469 // QSKIP("Fuzzing tests are disabled by default");
310
470
311 // ////////////// //
471 // ////////////// //
312 // Test structure //
472 // Test structure //
@@ -319,16 +479,33 void TestAmdaFuzzing::testFuzzing_data()
319 // ////////// //
479 // ////////// //
320
480
321 auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0});
481 auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0});
322 MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "imf"}}};
482 MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "c1_b"}}};
323
483
324 // Note: we don't use auto here as we want to pass std::shared_ptr<IDataProvider> as is in the
484 // Note: we don't use auto here as we want to pass std::shared_ptr<IDataProvider> as is in the
325 // QVariant
485 // QVariant
326 std::shared_ptr<IDataProvider> provider = std::make_shared<AmdaProvider>();
486 std::shared_ptr<IDataProvider> provider = std::make_shared<AmdaProvider>();
327
487
328 QTest::newRow("fuzzingTest") << Properties{
488
489 // Case Custom one : Only lot of pan on two variables synchronized
490 // QTest::newRow("fuzzingTestPan") << Properties{
491 // {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
492 // {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
493 // {PROVIDER_PROPERTY, QVariant::fromValue(provider)},
494 // {CUSTOM_OPERATIONS_PROPERTY, CUSTOM_CASE_ONE}};
495
496 QTest::newRow("fuzzingTestZoom") << Properties{
329 {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
497 {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
330 {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
498 {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
331 {PROVIDER_PROPERTY, QVariant::fromValue(provider)}};
499 {PROVIDER_PROPERTY, QVariant::fromValue(provider)},
500 {CUSTOM_OPERATIONS_PROPERTY, CUSTOM_CASE_TWO}};
501
502
503 // Fuzzing
504 // QTest::newRow("fuzzingTest") << Properties{
505 // {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
506 // {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
507 // {PROVIDER_PROPERTY, QVariant::fromValue(provider)}};
508
332 }
509 }
333
510
334 void TestAmdaFuzzing::testFuzzing()
511 void TestAmdaFuzzing::testFuzzing()
General Comments 0
You need to be logged in to leave comments. Login now