From 115d3bfd69ec6b520fc49a8fa6ea17edb9c4fe57 2018-01-16 08:15:09 From: mperrinel Date: 2018-01-16 08:15:09 Subject: [PATCH] Modif test fuzzing to enable custom test on specific operation --- diff --git a/plugins/amda/tests/FuzzingDefs.cpp b/plugins/amda/tests/FuzzingDefs.cpp index 5d2502e..f7b7524 100644 --- a/plugins/amda/tests/FuzzingDefs.cpp +++ b/plugins/amda/tests/FuzzingDefs.cpp @@ -13,6 +13,7 @@ const QString PROVIDER_PROPERTY = QStringLiteral("provider"); const QString OPERATION_DELAY_BOUNDS_PROPERTY = QStringLiteral("operationDelays"); const QString VALIDATORS_PROPERTY = QStringLiteral("validators"); const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY = QStringLiteral("validationFrequencyBounds"); +const QString CUSTOM_OPERATIONS_PROPERTY = QStringLiteral("customOperations"); // //////////// // // FuzzingState // diff --git a/plugins/amda/tests/FuzzingDefs.h b/plugins/amda/tests/FuzzingDefs.h index 6b66508..42b3656 100644 --- a/plugins/amda/tests/FuzzingDefs.h +++ b/plugins/amda/tests/FuzzingDefs.h @@ -62,6 +62,9 @@ extern const QString VALIDATORS_PROPERTY; /// Min/max number of operations to execute before calling validation of the current test's state extern const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY; +/// Custom operations to execute instead of random operations +extern const QString CUSTOM_OPERATIONS_PROPERTY; + // /////// // // Structs // // /////// // diff --git a/plugins/amda/tests/TestAmdaFuzzing.cpp b/plugins/amda/tests/TestAmdaFuzzing.cpp index b2d299c..7ed642c 100644 --- a/plugins/amda/tests/TestAmdaFuzzing.cpp +++ b/plugins/amda/tests/TestAmdaFuzzing.cpp @@ -57,6 +57,12 @@ struct OperationProperty { using VariableOperation = std::pair >; using VariablesOperations = std::vector; +struct CustomVariableOperation { + VariableOperation m_VariableOperation{}; + bool m_WaitAcquisition{false}; +}; +using CustomVariablesOperations = std::vector; + using OperationsTypes = std::map; using OperationsPool = std::map, OperationProperty>; @@ -70,27 +76,161 @@ using Validators = std::vector >; const auto ACQUISITION_TIMEOUT_DEFAULT_VALUE = 30000; const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100; const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1; -const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 1; +const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 2; const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue( - OperationsTypes{{FuzzingOperationType::CREATE, {1., true}}, - {FuzzingOperationType::DELETE, {0.1}}, // Delete operation is less frequent + OperationsTypes{{FuzzingOperationType::CREATE, {400., true}}, + {FuzzingOperationType::DELETE, {0.0}}, // Delete operation is less frequent {FuzzingOperationType::PAN_LEFT, {1.}}, {FuzzingOperationType::PAN_RIGHT, {1.}}, {FuzzingOperationType::ZOOM_IN, {1.}}, {FuzzingOperationType::ZOOM_OUT, {1.}}, - {FuzzingOperationType::SYNCHRONIZE, {0.8}}, + {FuzzingOperationType::SYNCHRONIZE, {4000.0}}, {FuzzingOperationType::DESYNCHRONIZE, {0.4}}}); const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2; /// Min/max delays between each operation (in ms) -const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(100, 3000)); +const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(3000, 3000)); /// Validators for the tests (executed in the order in which they're defined) const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue( ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}}); /// Min/max number of operations to execute before calling validation -const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(1, 10)); +const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(8, 8)); + + +// /////// ////// +// CUSTOM CASE // +// /////// ////// + +auto op = [](auto type){ + return FuzzingOperationFactory::create(type); +}; + + +const QVariant CUSTOM_CASE_ONE =QVariant::fromValue(CustomVariablesOperations{{{0, op(FuzzingOperationType::CREATE)}, true}, + {{0, op(FuzzingOperationType::SYNCHRONIZE)}}, + {{1, op(FuzzingOperationType::CREATE)}, true}, + {{1, op(FuzzingOperationType::SYNCHRONIZE)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_LEFT)}}, + {{0, op(FuzzingOperationType::PAN_RIGHT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + {{0, op(FuzzingOperationType::PAN_LEFT)}}, + {{1, op(FuzzingOperationType::PAN_RIGHT)}}, + }); + +const QVariant CUSTOM_CASE_TWO =QVariant::fromValue(CustomVariablesOperations{{{0, op(FuzzingOperationType::CREATE)}, true}, + {{0, op(FuzzingOperationType::SYNCHRONIZE)}}, + {{1, op(FuzzingOperationType::CREATE)}, true}, + {{1, op(FuzzingOperationType::SYNCHRONIZE)}}, + {{0, op(FuzzingOperationType::ZOOM_IN)}}, + {{1, op(FuzzingOperationType::ZOOM_IN)}}, + {{0, op(FuzzingOperationType::ZOOM_IN)}}, + {{1, op(FuzzingOperationType::ZOOM_OUT)}}, + {{0, op(FuzzingOperationType::ZOOM_OUT)}}, + {{1, op(FuzzingOperationType::ZOOM_OUT)}}, + {{0, op(FuzzingOperationType::ZOOM_IN)}}, + {{1, op(FuzzingOperationType::ZOOM_OUT)}}, + {{0, op(FuzzingOperationType::ZOOM_IN)}}, + {{0, op(FuzzingOperationType::ZOOM_OUT)}}, + {{1, op(FuzzingOperationType::ZOOM_IN)}}, + {{0, op(FuzzingOperationType::ZOOM_IN)}}, + {{0, op(FuzzingOperationType::ZOOM_OUT)}}, + {{1, op(FuzzingOperationType::ZOOM_IN)}}, + {{0, op(FuzzingOperationType::ZOOM_IN)}}, + {{0, op(FuzzingOperationType::ZOOM_OUT)}}, + }); // /////// // // Methods // @@ -193,9 +333,6 @@ public: void execute() { - qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbMaxOperations() << "operations on" - << nbMaxVariables() << "variable(s)..."; - // Inits the count of the number of operations before the next validation int nextValidationCounter = 0; @@ -207,27 +344,49 @@ public: }; updateValidationCounter(); + // Get custom operations + auto customOperations = m_Properties.value(CUSTOM_OPERATIONS_PROPERTY).value(); + auto isCustomTest = !customOperations.empty(); + + auto nbOperations = isCustomTest ? customOperations.size() : nbMaxOperations(); + + qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbOperations << "operations on" + << nbMaxVariables() << "variable(s)..."; + auto canExecute = true; - for (auto i = 0; i < nbMaxOperations() && canExecute; ++i) { - // Retrieves all operations that can be executed in the current context - VariablesOperations variableOperations{}; - Weights weights{}; - std::tie(variableOperations, weights) - = availableOperations(m_FuzzingState, operationsPool()); - - canExecute = !variableOperations.empty(); - if (canExecute) { - --nextValidationCounter; + for (auto i = 0; i < nbOperations && canExecute; ++i) { + VariableOperation variableOperation{}; + auto forceWait = false; + + if(isCustomTest){ + auto customOperation = customOperations.front(); + variableOperation = customOperation.m_VariableOperation; + customOperations.erase(customOperations.begin()); + + canExecute = variableOperation.second->canExecute(variableOperation.first, m_FuzzingState); + forceWait = customOperation.m_WaitAcquisition; + } else { + // Retrieves all operations that can be executed in the current context + VariablesOperations variableOperations{}; + Weights weights{}; + std::tie(variableOperations, weights) + = availableOperations(m_FuzzingState, operationsPool()); // Of the operations available, chooses a random operation and executes it - auto variableOperation + variableOperation = RandomGenerator::instance().randomChoice(variableOperations, weights); + canExecute = !variableOperations.empty(); + forceWait = operationsPool().at(variableOperation.second).m_WaitAcquisition; + } + + if (canExecute) { + --nextValidationCounter; auto variableId = variableOperation.first; auto fuzzingOperation = variableOperation.second; auto waitAcquisition = nextValidationCounter == 0 - || operationsPool().at(fuzzingOperation).m_WaitAcquisition; + || forceWait; fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController, m_Properties); @@ -293,6 +452,7 @@ private: } // namespace Q_DECLARE_METATYPE(OperationsTypes) +Q_DECLARE_METATYPE(CustomVariablesOperations) class TestAmdaFuzzing : public QObject { Q_OBJECT @@ -306,7 +466,7 @@ private slots: void TestAmdaFuzzing::testFuzzing_data() { // Note: Comment this line to run fuzzing tests - QSKIP("Fuzzing tests are disabled by default"); +// QSKIP("Fuzzing tests are disabled by default"); // ////////////// // // Test structure // @@ -319,16 +479,33 @@ void TestAmdaFuzzing::testFuzzing_data() // ////////// // auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0}); - MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "imf"}}}; + MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "c1_b"}}}; // Note: we don't use auto here as we want to pass std::shared_ptr as is in the // QVariant std::shared_ptr provider = std::make_shared(); - QTest::newRow("fuzzingTest") << Properties{ - {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)}, - {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)}, - {PROVIDER_PROPERTY, QVariant::fromValue(provider)}}; + + // Case Custom one : Only lot of pan on two variables synchronized +// QTest::newRow("fuzzingTestPan") << Properties{ +// {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)}, +// {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)}, +// {PROVIDER_PROPERTY, QVariant::fromValue(provider)}, +// {CUSTOM_OPERATIONS_PROPERTY, CUSTOM_CASE_ONE}}; + + QTest::newRow("fuzzingTestZoom") << Properties{ + {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)}, + {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)}, + {PROVIDER_PROPERTY, QVariant::fromValue(provider)}, + {CUSTOM_OPERATIONS_PROPERTY, CUSTOM_CASE_TWO}}; + + + // Fuzzing +// QTest::newRow("fuzzingTest") << Properties{ +// {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)}, +// {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)}, +// {PROVIDER_PROPERTY, QVariant::fromValue(provider)}}; + } void TestAmdaFuzzing::testFuzzing()