##// END OF EJS Templates
GIT_REPOSITORIES » LPP » Users » mperrinel » SciQLop-fork
RSS

Fork of HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop

Modif test fuzzing to enable custom test on specific operation
perrinel -
r1396:115d3bfd69ec
parent child
Show More
Side by Side Unified
Context file:
r1396:115d3bfd69ec Loading diff...:
Show file before | Show file after | Hide comments
@@ -13,6 +13,7 const QString PROVIDER_PROPERTY = QStringLiteral("provider");
13 13 const QString OPERATION_DELAY_BOUNDS_PROPERTY = QStringLiteral("operationDelays");
14 14 const QString VALIDATORS_PROPERTY = QStringLiteral("validators");
15 15 const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY = QStringLiteral("validationFrequencyBounds");
16 const QString CUSTOM_OPERATIONS_PROPERTY = QStringLiteral("customOperations");
16 17
17 18 // //////////// //
18 19 // FuzzingState //
Show file before | Show file after | Hide comments
@@ -62,6 +62,9 extern const QString VALIDATORS_PROPERTY;
62 62 /// Min/max number of operations to execute before calling validation of the current test's state
63 63 extern const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY;
64 64
65 /// Custom operations to execute instead of random operations
66 extern const QString CUSTOM_OPERATIONS_PROPERTY;
67
65 68 // /////// //
66 69 // Structs //
67 70 // /////// //
Show file before | Show file after | Hide comments
@@ -57,6 +57,12 struct OperationProperty {
57 57 using VariableOperation = std::pair<VariableId, std::shared_ptr<IFuzzingOperation> >;
58 58 using VariablesOperations = std::vector<VariableOperation>;
59 59
60 struct CustomVariableOperation {
61 VariableOperation m_VariableOperation{};
62 bool m_WaitAcquisition{false};
63 };
64 using CustomVariablesOperations = std::vector<CustomVariableOperation>;
65
60 66 using OperationsTypes = std::map<FuzzingOperationType, OperationProperty>;
61 67 using OperationsPool = std::map<std::shared_ptr<IFuzzingOperation>, OperationProperty>;
62 68
@@ -70,27 +76,161 using Validators = std::vector<std::shared_ptr<IFuzzingValidator> >;
70 76 const auto ACQUISITION_TIMEOUT_DEFAULT_VALUE = 30000;
71 77 const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100;
72 78 const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1;
73 const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 1;
79 const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 2;
74 80 const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue(
75 OperationsTypes{{FuzzingOperationType::CREATE, {1., true}},
76 {FuzzingOperationType::DELETE, {0.1}}, // Delete operation is less frequent
81 OperationsTypes{{FuzzingOperationType::CREATE, {400., true}},
82 {FuzzingOperationType::DELETE, {0.0}}, // Delete operation is less frequent
77 83 {FuzzingOperationType::PAN_LEFT, {1.}},
78 84 {FuzzingOperationType::PAN_RIGHT, {1.}},
79 85 {FuzzingOperationType::ZOOM_IN, {1.}},
80 86 {FuzzingOperationType::ZOOM_OUT, {1.}},
81 {FuzzingOperationType::SYNCHRONIZE, {0.8}},
87 {FuzzingOperationType::SYNCHRONIZE, {4000.0}},
82 88 {FuzzingOperationType::DESYNCHRONIZE, {0.4}}});
83 89 const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2;
84 90
85 91 /// Min/max delays between each operation (in ms)
86 const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(100, 3000));
92 const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(3000, 3000));
87 93
88 94 /// Validators for the tests (executed in the order in which they're defined)
89 95 const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue(
90 96 ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}});
91 97
92 98 /// Min/max number of operations to execute before calling validation
93 const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(1, 10));
99 const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(8, 8));
100
101
102 // /////// //////
103 // CUSTOM CASE //
104 // /////// //////
105
106 auto op = [](auto type){
107 return FuzzingOperationFactory::create(type);
108 };
109
110
111 const QVariant CUSTOM_CASE_ONE =QVariant::fromValue(CustomVariablesOperations{{{0, op(FuzzingOperationType::CREATE)}, true},
112 {{0, op(FuzzingOperationType::SYNCHRONIZE)}},
113 {{1, op(FuzzingOperationType::CREATE)}, true},
114 {{1, op(FuzzingOperationType::SYNCHRONIZE)}},
115 {{0, op(FuzzingOperationType::PAN_LEFT)}},
116 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
117 {{0, op(FuzzingOperationType::PAN_LEFT)}},
118 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
119 {{0, op(FuzzingOperationType::PAN_LEFT)}},
120 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
121 {{0, op(FuzzingOperationType::PAN_LEFT)}},
122 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
123 {{0, op(FuzzingOperationType::PAN_LEFT)}},
124 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
125 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
126 {{1, op(FuzzingOperationType::PAN_LEFT)}},
127 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
128 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
129 {{0, op(FuzzingOperationType::PAN_LEFT)}},
130 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
131 {{0, op(FuzzingOperationType::PAN_LEFT)}},
132 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
133 {{0, op(FuzzingOperationType::PAN_LEFT)}},
134 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
135 {{0, op(FuzzingOperationType::PAN_LEFT)}},
136 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
137 {{0, op(FuzzingOperationType::PAN_LEFT)}},
138 {{1, op(FuzzingOperationType::PAN_LEFT)}},
139 {{0, op(FuzzingOperationType::PAN_LEFT)}},
140 {{1, op(FuzzingOperationType::PAN_LEFT)}},
141 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
142 {{1, op(FuzzingOperationType::PAN_LEFT)}},
143 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
144 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
145 {{0, op(FuzzingOperationType::PAN_LEFT)}},
146 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
147 {{0, op(FuzzingOperationType::PAN_LEFT)}},
148 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
149 {{0, op(FuzzingOperationType::PAN_LEFT)}},
150 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
151 {{0, op(FuzzingOperationType::PAN_LEFT)}},
152 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
153 {{0, op(FuzzingOperationType::PAN_LEFT)}},
154 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
155 {{0, op(FuzzingOperationType::PAN_LEFT)}},
156 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
157 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
158 {{1, op(FuzzingOperationType::PAN_LEFT)}},
159 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
160 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
161 {{0, op(FuzzingOperationType::PAN_LEFT)}},
162 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
163 {{0, op(FuzzingOperationType::PAN_LEFT)}},
164 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
165 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
166 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
167 {{0, op(FuzzingOperationType::PAN_LEFT)}},
168 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
169 {{0, op(FuzzingOperationType::PAN_LEFT)}},
170 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
171 {{0, op(FuzzingOperationType::PAN_LEFT)}},
172 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
173 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
174 {{1, op(FuzzingOperationType::PAN_LEFT)}},
175 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
176 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
177 {{0, op(FuzzingOperationType::PAN_LEFT)}},
178 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
179 {{0, op(FuzzingOperationType::PAN_LEFT)}},
180 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
181 {{0, op(FuzzingOperationType::PAN_LEFT)}},
182 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
183 {{0, op(FuzzingOperationType::PAN_LEFT)}},
184 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
185 {{0, op(FuzzingOperationType::PAN_LEFT)}},
186 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
187 {{0, op(FuzzingOperationType::PAN_LEFT)}},
188 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
189 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
190 {{1, op(FuzzingOperationType::PAN_LEFT)}},
191 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
192 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
193 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
194 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
195 {{0, op(FuzzingOperationType::PAN_LEFT)}},
196 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
197 {{0, op(FuzzingOperationType::PAN_LEFT)}},
198 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
199 {{0, op(FuzzingOperationType::PAN_LEFT)}},
200 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
201 {{0, op(FuzzingOperationType::PAN_LEFT)}},
202 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
203 {{0, op(FuzzingOperationType::PAN_LEFT)}},
204 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
205 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
206 {{1, op(FuzzingOperationType::PAN_LEFT)}},
207 {{0, op(FuzzingOperationType::PAN_RIGHT)}},
208 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
209 {{0, op(FuzzingOperationType::PAN_LEFT)}},
210 {{1, op(FuzzingOperationType::PAN_RIGHT)}},
211 });
212
213 const QVariant CUSTOM_CASE_TWO =QVariant::fromValue(CustomVariablesOperations{{{0, op(FuzzingOperationType::CREATE)}, true},
214 {{0, op(FuzzingOperationType::SYNCHRONIZE)}},
215 {{1, op(FuzzingOperationType::CREATE)}, true},
216 {{1, op(FuzzingOperationType::SYNCHRONIZE)}},
217 {{0, op(FuzzingOperationType::ZOOM_IN)}},
218 {{1, op(FuzzingOperationType::ZOOM_IN)}},
219 {{0, op(FuzzingOperationType::ZOOM_IN)}},
220 {{1, op(FuzzingOperationType::ZOOM_OUT)}},
221 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
222 {{1, op(FuzzingOperationType::ZOOM_OUT)}},
223 {{0, op(FuzzingOperationType::ZOOM_IN)}},
224 {{1, op(FuzzingOperationType::ZOOM_OUT)}},
225 {{0, op(FuzzingOperationType::ZOOM_IN)}},
226 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
227 {{1, op(FuzzingOperationType::ZOOM_IN)}},
228 {{0, op(FuzzingOperationType::ZOOM_IN)}},
229 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
230 {{1, op(FuzzingOperationType::ZOOM_IN)}},
231 {{0, op(FuzzingOperationType::ZOOM_IN)}},
232 {{0, op(FuzzingOperationType::ZOOM_OUT)}},
233 });
94 234
95 235 // /////// //
96 236 // Methods //
@@ -193,9 +333,6 public:
193 333
194 334 void execute()
195 335 {
196 qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbMaxOperations() << "operations on"
197 << nbMaxVariables() << "variable(s)...";
198
199 336
200 337 // Inits the count of the number of operations before the next validation
201 338 int nextValidationCounter = 0;
@@ -207,27 +344,49 public:
207 344 };
208 345 updateValidationCounter();
209 346
347 // Get custom operations
348 auto customOperations = m_Properties.value(CUSTOM_OPERATIONS_PROPERTY).value<CustomVariablesOperations>();
349 auto isCustomTest = !customOperations.empty();
350
351 auto nbOperations = isCustomTest ? customOperations.size() : nbMaxOperations();
352
353 qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbOperations << "operations on"
354 << nbMaxVariables() << "variable(s)...";
355
210 356 auto canExecute = true;
211 for (auto i = 0; i < nbMaxOperations() && canExecute; ++i) {
357 for (auto i = 0; i < nbOperations && canExecute; ++i) {
358 VariableOperation variableOperation{};
359 auto forceWait = false;
360
361 if(isCustomTest){
362 auto customOperation = customOperations.front();
363 variableOperation = customOperation.m_VariableOperation;
364 customOperations.erase(customOperations.begin());
365
366 canExecute = variableOperation.second->canExecute(variableOperation.first, m_FuzzingState);
367 forceWait = customOperation.m_WaitAcquisition;
368 } else {
212 369 // Retrieves all operations that can be executed in the current context
213 370 VariablesOperations variableOperations{};
214 371 Weights weights{};
215 372 std::tie(variableOperations, weights)
216 373 = availableOperations(m_FuzzingState, operationsPool());
217 374
375 // Of the operations available, chooses a random operation and executes it
376 variableOperation
377 = RandomGenerator::instance().randomChoice(variableOperations, weights);
218 378 canExecute = !variableOperations.empty();
379 forceWait = operationsPool().at(variableOperation.second).m_WaitAcquisition;
380 }
381
219 382 if (canExecute) {
220 383 --nextValidationCounter;
221 384
222 // Of the operations available, chooses a random operation and executes it
223 auto variableOperation
224 = RandomGenerator::instance().randomChoice(variableOperations, weights);
225
226 385 auto variableId = variableOperation.first;
227 386 auto fuzzingOperation = variableOperation.second;
228 387
229 388 auto waitAcquisition = nextValidationCounter == 0
230 || operationsPool().at(fuzzingOperation).m_WaitAcquisition;
389 || forceWait;
231 390
232 391 fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController,
233 392 m_Properties);
@@ -293,6 +452,7 private:
293 452 } // namespace
294 453
295 454 Q_DECLARE_METATYPE(OperationsTypes)
455 Q_DECLARE_METATYPE(CustomVariablesOperations)
296 456
297 457 class TestAmdaFuzzing : public QObject {
298 458 Q_OBJECT
@@ -306,7 +466,7 private slots:
306 466 void TestAmdaFuzzing::testFuzzing_data()
307 467 {
308 468 // Note: Comment this line to run fuzzing tests
309 QSKIP("Fuzzing tests are disabled by default");
469 // QSKIP("Fuzzing tests are disabled by default");
310 470
311 471 // ////////////// //
312 472 // Test structure //
@@ -319,16 +479,33 void TestAmdaFuzzing::testFuzzing_data()
319 479 // ////////// //
320 480
321 481 auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0});
322 MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "imf"}}};
482 MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "c1_b"}}};
323 483
324 484 // Note: we don't use auto here as we want to pass std::shared_ptr<IDataProvider> as is in the
325 485 // QVariant
326 486 std::shared_ptr<IDataProvider> provider = std::make_shared<AmdaProvider>();
327 487
328 QTest::newRow("fuzzingTest") << Properties{
488
489 // Case Custom one : Only lot of pan on two variables synchronized
490 // QTest::newRow("fuzzingTestPan") << Properties{
491 // {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
492 // {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
493 // {PROVIDER_PROPERTY, QVariant::fromValue(provider)},
494 // {CUSTOM_OPERATIONS_PROPERTY, CUSTOM_CASE_ONE}};
495
496 QTest::newRow("fuzzingTestZoom") << Properties{
329 497 {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
330 498 {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
331 {PROVIDER_PROPERTY, QVariant::fromValue(provider)}};
499 {PROVIDER_PROPERTY, QVariant::fromValue(provider)},
500 {CUSTOM_OPERATIONS_PROPERTY, CUSTOM_CASE_TWO}};
501
502
503 // Fuzzing
504 // QTest::newRow("fuzzingTest") << Properties{
505 // {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
506 // {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
507 // {PROVIDER_PROPERTY, QVariant::fromValue(provider)}};
508
332 509 }
333 510
334 511 void TestAmdaFuzzing::testFuzzing()
General Comments 0
You need to be logged in to leave comments. Login now