TestAmdaFuzzing.cpp
395 lines
| 14.7 KiB
| text/x-c
|
CppLexer
Alexandre Leroux
|
r1201 | #include "FuzzingDefs.h" | ||
|
Alexandre Leroux
|
r1202 | #include "FuzzingOperations.h" | ||
|
Alexandre Leroux
|
r1206 | #include "FuzzingUtils.h" | ||
|
Alexandre Leroux
|
r1228 | #include "FuzzingValidators.h" | ||
|
Alexandre Leroux
|
r1206 | |||
|
Alexandre Leroux
|
r1210 | #include "AmdaProvider.h" | ||
|
Alexandre Leroux
|
r1246 | #include <Common/SignalWaiter.h> | ||
|
Alexandre Leroux
|
r1199 | #include <Network/NetworkController.h> | ||
|
Alexandre Leroux
|
r1224 | #include <Settings/SqpSettingsDefs.h> | ||
|
Alexandre Leroux
|
r1199 | #include <SqpApplication.h> | ||
| #include <Time/TimeController.h> | ||||
|
Alexandre Leroux
|
r1222 | #include <Variable/Variable.h> | ||
|
Alexandre Leroux
|
r1199 | #include <Variable/VariableController.h> | ||
| #include <QLoggingCategory> | ||||
| #include <QObject> | ||||
| #include <QtTest> | ||||
| #include <memory> | ||||
| Q_LOGGING_CATEGORY(LOG_TestAmdaFuzzing, "TestAmdaFuzzing") | ||||
|
Alexandre Leroux
|
r1245 | /** | ||
| * Macro used to generate a getter for a property in @sa FuzzingTest. The macro generates a static | ||||
| * attribute that is initialized by searching in properties the property and use a default value if | ||||
| * it's not present. Macro arguments are: | ||||
| * - GETTER_NAME : name of the getter | ||||
| * - PROPERTY_NAME: used to generate constants for property's name ({PROPERTY_NAME}_PROPERTY) and | ||||
| * default value ({PROPERTY_NAME}_DEFAULT_VALUE) | ||||
| * - TYPE : return type of the getter | ||||
| */ | ||||
| // clang-format off | ||||
| #define DECLARE_PROPERTY_GETTER(GETTER_NAME, PROPERTY_NAME, TYPE) \ | ||||
| TYPE GETTER_NAME() const \ | ||||
| { \ | ||||
| static auto result = m_Properties.value(PROPERTY_NAME##_PROPERTY, PROPERTY_NAME##_DEFAULT_VALUE).value<TYPE>(); \ | ||||
| return result; \ | ||||
| } \ | ||||
| // clang-format on | ||||
|
Alexandre Leroux
|
r1199 | namespace { | ||
|
Alexandre Leroux
|
r1202 | // /////// // | ||
| // Aliases // | ||||
| // /////// // | ||||
|
Alexandre Leroux
|
r1245 | using IntPair = std::pair<int, int>; | ||
|
Alexandre Leroux
|
r1216 | using Weight = double; | ||
| using Weights = std::vector<Weight>; | ||||
|
Alexandre Leroux
|
r1203 | |||
|
Alexandre Leroux
|
r1249 | struct OperationProperty { | ||
| Weight m_Weight{1.}; | ||||
| bool m_WaitAcquisition{false}; | ||||
| }; | ||||
|
Alexandre Leroux
|
r1205 | using VariableOperation = std::pair<VariableId, std::shared_ptr<IFuzzingOperation> >; | ||
| using VariablesOperations = std::vector<VariableOperation>; | ||||
|
Alexandre Leroux
|
r1249 | using OperationsTypes = std::map<FuzzingOperationType, OperationProperty>; | ||
| using OperationsPool = std::map<std::shared_ptr<IFuzzingOperation>, OperationProperty>; | ||||
|
Alexandre Leroux
|
r1228 | using Validators = std::vector<std::shared_ptr<IFuzzingValidator> >; | ||
|
Alexandre Leroux
|
r1203 | |||
|
Alexandre Leroux
|
r1201 | // ///////// // | ||
| // Constants // | ||||
| // ///////// // | ||||
| // Defaults values used when the associated properties have not been set for the test | ||||
|
Alexandre Leroux
|
r1248 | const auto ACQUISITION_TIMEOUT_DEFAULT_VALUE = 30000; | ||
|
Alexandre Leroux
|
r1201 | const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100; | ||
|
Alexandre Leroux
|
r1237 | const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1; | ||
|
Alexandre Leroux
|
r1201 | const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 1; | ||
|
Alexandre Leroux
|
r1251 | const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue( | ||
| OperationsTypes{{FuzzingOperationType::CREATE, {1., true}}, | ||||
| {FuzzingOperationType::DELETE, {0.1}}, // Delete operation is less frequent | ||||
| {FuzzingOperationType::PAN_LEFT, {1.}}, | ||||
| {FuzzingOperationType::PAN_RIGHT, {1.}}, | ||||
| {FuzzingOperationType::ZOOM_IN, {1.}}, | ||||
| {FuzzingOperationType::ZOOM_OUT, {1.}}, | ||||
| {FuzzingOperationType::SYNCHRONIZE, {0.8}}, | ||||
| {FuzzingOperationType::DESYNCHRONIZE, {0.4}}}); | ||||
|
Alexandre Leroux
|
r1224 | const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2; | ||
|
Alexandre Leroux
|
r1219 | |||
|
Alexandre Leroux
|
r1243 | /// Min/max delays between each operation (in ms) | ||
| const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(100, 3000)); | ||||
|
Alexandre Leroux
|
r1209 | |||
|
Alexandre Leroux
|
r1228 | /// Validators for the tests (executed in the order in which they're defined) | ||
| const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue( | ||||
| ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}}); | ||||
|
Alexandre Leroux
|
r1244 | /// Min/max number of operations to execute before calling validation | ||
| const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(1, 10)); | ||||
|
Alexandre Leroux
|
r1205 | // /////// // | ||
| // Methods // | ||||
| // /////// // | ||||
| /// Goes through the variables pool and operations pool to determine the set of {variable/operation} | ||||
| /// pairs that are valid (i.e. operation that can be executed on variable) | ||||
|
Alexandre Leroux
|
r1249 | std::pair<VariablesOperations, Weights> availableOperations(const FuzzingState &fuzzingState, | ||
| const OperationsPool &operationsPool) | ||||
|
Alexandre Leroux
|
r1205 | { | ||
| VariablesOperations result{}; | ||||
|
Alexandre Leroux
|
r1216 | Weights weights{}; | ||
|
Alexandre Leroux
|
r1205 | |||
|
Alexandre Leroux
|
r1236 | for (const auto &variablesPoolEntry : fuzzingState.m_VariablesPool) { | ||
|
Alexandre Leroux
|
r1205 | auto variableId = variablesPoolEntry.first; | ||
|
Alexandre Leroux
|
r1216 | for (const auto &operationsPoolEntry : operationsPool) { | ||
| auto operation = operationsPoolEntry.first; | ||||
|
Alexandre Leroux
|
r1249 | auto operationProperty = operationsPoolEntry.second; | ||
|
Alexandre Leroux
|
r1216 | |||
|
Alexandre Leroux
|
r1205 | // A pair is valid if the current operation can be executed on the current variable | ||
|
Alexandre Leroux
|
r1236 | if (operation->canExecute(variableId, fuzzingState)) { | ||
|
Alexandre Leroux
|
r1205 | result.push_back({variableId, operation}); | ||
|
Alexandre Leroux
|
r1249 | weights.push_back(operationProperty.m_Weight); | ||
|
Alexandre Leroux
|
r1205 | } | ||
| } | ||||
| } | ||||
|
Alexandre Leroux
|
r1216 | return {result, weights}; | ||
|
Alexandre Leroux
|
r1205 | } | ||
|
Alexandre Leroux
|
r1249 | OperationsPool createOperationsPool(const OperationsTypes &types) | ||
|
Alexandre Leroux
|
r1204 | { | ||
|
Alexandre Leroux
|
r1249 | OperationsPool result{}; | ||
|
Alexandre Leroux
|
r1204 | |||
|
Alexandre Leroux
|
r1216 | std::transform( | ||
| types.cbegin(), types.cend(), std::inserter(result, result.end()), [](const auto &type) { | ||||
| return std::make_pair(FuzzingOperationFactory::create(type.first), type.second); | ||||
| }); | ||||
|
Alexandre Leroux
|
r1204 | |||
| return result; | ||||
| } | ||||
|
Alexandre Leroux
|
r1228 | Validators createValidators(const ValidatorsTypes &types) | ||
| { | ||||
| Validators result{}; | ||||
| std::transform(types.cbegin(), types.cend(), std::inserter(result, result.end()), | ||||
| [](const auto &type) { return FuzzingValidatorFactory::create(type); }); | ||||
| return result; | ||||
| } | ||||
|
Alexandre Leroux
|
r1229 | /** | ||
| * Validates all the variables' states passed in parameter, according to a set of validators | ||||
| * @param variablesPool the variables' states | ||||
| * @param validators the validators used for validation | ||||
| */ | ||||
| void validate(const VariablesPool &variablesPool, const Validators &validators) | ||||
| { | ||||
| for (const auto &variablesPoolEntry : variablesPool) { | ||||
| auto variableId = variablesPoolEntry.first; | ||||
| const auto &variableState = variablesPoolEntry.second; | ||||
| auto variableMessage = variableState.m_Variable ? variableState.m_Variable->name() | ||||
| : QStringLiteral("null variable"); | ||||
| qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Validating state of variable at index" | ||||
| << variableId << "(" << variableMessage << ")..."; | ||||
| for (const auto &validator : validators) { | ||||
| validator->validate(VariableState{variableState}); | ||||
| } | ||||
| qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Validation completed."; | ||||
| } | ||||
| } | ||||
|
Alexandre Leroux
|
r1199 | /** | ||
| * Class to run random tests | ||||
| */ | ||||
| class FuzzingTest { | ||||
| public: | ||||
|
Alexandre Leroux
|
r1200 | explicit FuzzingTest(VariableController &variableController, Properties properties) | ||
| : m_VariableController{variableController}, | ||||
| m_Properties{std::move(properties)}, | ||||
|
Alexandre Leroux
|
r1236 | m_FuzzingState{} | ||
|
Alexandre Leroux
|
r1200 | { | ||
|
Alexandre Leroux
|
r1203 | // Inits variables pool: at init, all variables are null | ||
| for (auto variableId = 0; variableId < nbMaxVariables(); ++variableId) { | ||||
|
Alexandre Leroux
|
r1236 | m_FuzzingState.m_VariablesPool[variableId] = VariableState{}; | ||
|
Alexandre Leroux
|
r1203 | } | ||
|
Alexandre Leroux
|
r1237 | |||
| // Inits sync groups and registers them into the variable controller | ||||
| for (auto i = 0; i < nbMaxSyncGroups(); ++i) { | ||||
| auto syncGroupId = SyncGroupId::createUuid(); | ||||
| variableController.onAddSynchronizationGroupId(syncGroupId); | ||||
| m_FuzzingState.m_SyncGroupsPool[syncGroupId] = SyncGroup{}; | ||||
| } | ||||
|
Alexandre Leroux
|
r1200 | } | ||
|
Alexandre Leroux
|
r1199 | void execute() | ||
| { | ||||
|
Alexandre Leroux
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbMaxOperations() << "operations on" | ||
| << nbMaxVariables() << "variable(s)..."; | ||||
|
Alexandre Leroux
|
r1201 | |||
|
Alexandre Leroux
|
r1244 | |||
| // Inits the count of the number of operations before the next validation | ||||
| int nextValidationCounter = 0; | ||||
| auto updateValidationCounter = [this, &nextValidationCounter]() { | ||||
| nextValidationCounter = RandomGenerator::instance().generateInt( | ||||
| validationFrequencies().first, validationFrequencies().second); | ||||
| qCInfo(LOG_TestAmdaFuzzing()).noquote() | ||||
|
Alexandre Leroux
|
r1248 | << "Next validation in " << nextValidationCounter << "operation(s)..."; | ||
|
Alexandre Leroux
|
r1244 | }; | ||
| updateValidationCounter(); | ||||
|
Alexandre Leroux
|
r1205 | auto canExecute = true; | ||
| for (auto i = 0; i < nbMaxOperations() && canExecute; ++i) { | ||||
| // Retrieves all operations that can be executed in the current context | ||||
|
Alexandre Leroux
|
r1216 | VariablesOperations variableOperations{}; | ||
| Weights weights{}; | ||||
| std::tie(variableOperations, weights) | ||||
|
Alexandre Leroux
|
r1236 | = availableOperations(m_FuzzingState, operationsPool()); | ||
|
Alexandre Leroux
|
r1205 | |||
| canExecute = !variableOperations.empty(); | ||||
| if (canExecute) { | ||||
|
Alexandre Leroux
|
r1244 | --nextValidationCounter; | ||
|
Alexandre Leroux
|
r1205 | // Of the operations available, chooses a random operation and executes it | ||
| auto variableOperation | ||||
|
Alexandre Leroux
|
r1216 | = RandomGenerator::instance().randomChoice(variableOperations, weights); | ||
|
Alexandre Leroux
|
r1205 | |||
| auto variableId = variableOperation.first; | ||||
| auto fuzzingOperation = variableOperation.second; | ||||
|
Alexandre Leroux
|
r1250 | auto waitAcquisition = nextValidationCounter == 0 | ||
| || operationsPool().at(fuzzingOperation).m_WaitAcquisition; | ||||
|
Alexandre Leroux
|
r1248 | |||
|
Alexandre Leroux
|
r1236 | fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController, | ||
| m_Properties); | ||||
|
Alexandre Leroux
|
r1243 | |||
|
Alexandre Leroux
|
r1248 | if (waitAcquisition) { | ||
| qCDebug(LOG_TestAmdaFuzzing()) << "Waiting for acquisition to finish..."; | ||||
| SignalWaiter{m_VariableController, SIGNAL(acquisitionFinished())}.wait( | ||||
| acquisitionTimeout()); | ||||
|
Alexandre Leroux
|
r1229 | |||
|
Alexandre Leroux
|
r1248 | // Validates variables | ||
|
Alexandre Leroux
|
r1250 | if (nextValidationCounter == 0) { | ||
| validate(m_FuzzingState.m_VariablesPool, validators()); | ||||
| updateValidationCounter(); | ||||
| } | ||||
|
Alexandre Leroux
|
r1244 | } | ||
|
Alexandre Leroux
|
r1248 | else { | ||
| // Delays the next operation with a randomly generated time | ||||
| auto delay = RandomGenerator::instance().generateInt(operationDelays().first, | ||||
| operationDelays().second); | ||||
| qCDebug(LOG_TestAmdaFuzzing()) | ||||
| << "Waiting " << delay << "ms before the next operation..."; | ||||
| QTest::qWait(delay); | ||||
| } | ||||
|
Alexandre Leroux
|
r1205 | } | ||
| else { | ||||
|
Alexandre Leroux
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() | ||
|
Alexandre Leroux
|
r1205 | << "No more operations are available, the execution of the test will stop..."; | ||
| } | ||||
| } | ||||
|
Alexandre Leroux
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Execution of the test completed."; | ||
|
Alexandre Leroux
|
r1199 | } | ||
|
Alexandre Leroux
|
r1200 | |||
| private: | ||||
|
Alexandre Leroux
|
r1249 | OperationsPool operationsPool() const | ||
|
Alexandre Leroux
|
r1204 | { | ||
| static auto result = createOperationsPool( | ||||
| m_Properties.value(AVAILABLE_OPERATIONS_PROPERTY, AVAILABLE_OPERATIONS_DEFAULT_VALUE) | ||||
|
Alexandre Leroux
|
r1249 | .value<OperationsTypes>()); | ||
|
Alexandre Leroux
|
r1204 | return result; | ||
| } | ||||
|
Alexandre Leroux
|
r1228 | Validators validators() const | ||
| { | ||||
| static auto result | ||||
| = createValidators(m_Properties.value(VALIDATORS_PROPERTY, VALIDATORS_DEFAULT_VALUE) | ||||
| .value<ValidatorsTypes>()); | ||||
| return result; | ||||
| } | ||||
|
Alexandre Leroux
|
r1245 | DECLARE_PROPERTY_GETTER(nbMaxOperations, NB_MAX_OPERATIONS, int) | ||
| DECLARE_PROPERTY_GETTER(nbMaxSyncGroups, NB_MAX_SYNC_GROUPS, int) | ||||
| DECLARE_PROPERTY_GETTER(nbMaxVariables, NB_MAX_VARIABLES, int) | ||||
| DECLARE_PROPERTY_GETTER(operationDelays, OPERATION_DELAY_BOUNDS, IntPair) | ||||
| DECLARE_PROPERTY_GETTER(validationFrequencies, VALIDATION_FREQUENCY_BOUNDS, IntPair) | ||||
| DECLARE_PROPERTY_GETTER(acquisitionTimeout, ACQUISITION_TIMEOUT, int) | ||||
|
Alexandre Leroux
|
r1244 | |||
|
Alexandre Leroux
|
r1200 | VariableController &m_VariableController; | ||
| Properties m_Properties; | ||||
|
Alexandre Leroux
|
r1236 | FuzzingState m_FuzzingState; | ||
|
Alexandre Leroux
|
r1199 | }; | ||
| } // namespace | ||||
|
Alexandre Leroux
|
r1249 | Q_DECLARE_METATYPE(OperationsTypes) | ||
|
Alexandre Leroux
|
r1199 | class TestAmdaFuzzing : public QObject { | ||
| Q_OBJECT | ||||
| private slots: | ||||
| /// Input data for @sa testFuzzing() | ||||
| void testFuzzing_data(); | ||||
| void testFuzzing(); | ||||
| }; | ||||
| void TestAmdaFuzzing::testFuzzing_data() | ||||
| { | ||||
|
Alexandre Leroux
|
r1255 | // Note: Comment this line to run fuzzing tests | ||
| QSKIP("Fuzzing tests are disabled by default"); | ||||
|
Alexandre Leroux
|
r1199 | // ////////////// // | ||
| // Test structure // | ||||
| // ////////////// // | ||||
|
Alexandre Leroux
|
r1200 | QTest::addColumn<Properties>("properties"); // Properties for random test | ||
|
Alexandre Leroux
|
r1199 | |||
| // ////////// // | ||||
| // Test cases // | ||||
| // ////////// // | ||||
|
Alexandre Leroux
|
r1210 | auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0}); | ||
| MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "imf"}}}; | ||||
| // Note: we don't use auto here as we want to pass std::shared_ptr<IDataProvider> as is in the | ||||
| // QVariant | ||||
| std::shared_ptr<IDataProvider> provider = std::make_shared<AmdaProvider>(); | ||||
| QTest::newRow("fuzzingTest") << Properties{ | ||||
| {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)}, | ||||
| {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)}, | ||||
| {PROVIDER_PROPERTY, QVariant::fromValue(provider)}}; | ||||
|
Alexandre Leroux
|
r1199 | } | ||
| void TestAmdaFuzzing::testFuzzing() | ||||
| { | ||||
|
Alexandre Leroux
|
r1200 | QFETCH(Properties, properties); | ||
|
Alexandre Leroux
|
r1224 | // Sets cache property | ||
| QSettings settings{}; | ||||
| auto cacheTolerance = properties.value(CACHE_TOLERANCE_PROPERTY, CACHE_TOLERANCE_DEFAULT_VALUE); | ||||
| settings.setValue(GENERAL_TOLERANCE_AT_INIT_KEY, cacheTolerance); | ||||
| settings.setValue(GENERAL_TOLERANCE_AT_UPDATE_KEY, cacheTolerance); | ||||
|
Alexandre Leroux
|
r1199 | auto &variableController = sqpApp->variableController(); | ||
| auto &timeController = sqpApp->timeController(); | ||||
|
Alexandre Leroux
|
r1209 | // Generates random initial range (bounded to max range) | ||
| auto maxRange = properties.value(MAX_RANGE_PROPERTY, QVariant::fromValue(INVALID_RANGE)) | ||||
| .value<SqpRange>(); | ||||
| QVERIFY(maxRange != INVALID_RANGE); | ||||
| auto initialRangeStart | ||||
| = RandomGenerator::instance().generateDouble(maxRange.m_TStart, maxRange.m_TEnd); | ||||
| auto initialRangeEnd | ||||
| = RandomGenerator::instance().generateDouble(maxRange.m_TStart, maxRange.m_TEnd); | ||||
| if (initialRangeStart > initialRangeEnd) { | ||||
| std::swap(initialRangeStart, initialRangeEnd); | ||||
| } | ||||
| // Sets initial range on time controller | ||||
| SqpRange initialRange{initialRangeStart, initialRangeEnd}; | ||||
|
Alexandre Leroux
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Setting initial range to" << initialRange << "..."; | ||
|
Alexandre Leroux
|
r1209 | timeController.onTimeToUpdate(initialRange); | ||
|
Alexandre Leroux
|
r1223 | properties.insert(INITIAL_RANGE_PROPERTY, QVariant::fromValue(initialRange)); | ||
|
Alexandre Leroux
|
r1209 | |||
|
Alexandre Leroux
|
r1200 | FuzzingTest test{variableController, properties}; | ||
|
Alexandre Leroux
|
r1199 | test.execute(); | ||
| } | ||||
| int main(int argc, char *argv[]) | ||||
| { | ||||
|
Alexandre Leroux
|
r1328 | // Increases the test function timeout (which is 5 minutes by default) to 12 hours | ||
| // https://stackoverflow.com/questions/42655932/setting-timeout-to-qt-test | ||||
| qputenv("QTEST_FUNCTION_TIMEOUT", QByteArray::number(12*60*60*1000)); | ||||
|
Alexandre Leroux
|
r1199 | QLoggingCategory::setFilterRules( | ||
| "*.warning=false\n" | ||||
| "*.info=false\n" | ||||
| "*.debug=false\n" | ||||
| "FuzzingOperations.info=true\n" | ||||
|
Alexandre Leroux
|
r1228 | "FuzzingValidators.info=true\n" | ||
|
Alexandre Leroux
|
r1199 | "TestAmdaFuzzing.info=true\n"); | ||
| SqpApplication app{argc, argv}; | ||||
|
Alexandre Leroux
|
r1224 | SqpApplication::setOrganizationName("LPP"); | ||
| SqpApplication::setOrganizationDomain("lpp.fr"); | ||||
| SqpApplication::setApplicationName("SciQLop-TestFuzzing"); | ||||
|
Alexandre Leroux
|
r1199 | app.setAttribute(Qt::AA_Use96Dpi, true); | ||
| TestAmdaFuzzing testObject{}; | ||||
| QTEST_SET_MAIN_SOURCE_PATH | ||||
| return QTest::qExec(&testObject, argc, argv); | ||||
| } | ||||
| #include "TestAmdaFuzzing.moc" | ||||