TestAmdaFuzzing.cpp
388 lines
| 14.3 KiB
| text/x-c
|
CppLexer
|
r1201 | #include "FuzzingDefs.h" | ||
|
r1202 | #include "FuzzingOperations.h" | ||
|
r1206 | #include "FuzzingUtils.h" | ||
|
r1228 | #include "FuzzingValidators.h" | ||
|
r1206 | |||
|
r1210 | #include "AmdaProvider.h" | ||
|
r1246 | #include <Common/SignalWaiter.h> | ||
|
r1199 | #include <Network/NetworkController.h> | ||
|
r1224 | #include <Settings/SqpSettingsDefs.h> | ||
|
r1199 | #include <SqpApplication.h> | ||
#include <Time/TimeController.h> | ||||
|
r1222 | #include <Variable/Variable.h> | ||
|
r1199 | #include <Variable/VariableController.h> | ||
#include <QLoggingCategory> | ||||
#include <QObject> | ||||
#include <QtTest> | ||||
#include <memory> | ||||
Q_LOGGING_CATEGORY(LOG_TestAmdaFuzzing, "TestAmdaFuzzing") | ||||
|
r1245 | /** | ||
* Macro used to generate a getter for a property in @sa FuzzingTest. The macro generates a static | ||||
* attribute that is initialized by searching in properties the property and use a default value if | ||||
* it's not present. Macro arguments are: | ||||
* - GETTER_NAME : name of the getter | ||||
* - PROPERTY_NAME: used to generate constants for property's name ({PROPERTY_NAME}_PROPERTY) and | ||||
* default value ({PROPERTY_NAME}_DEFAULT_VALUE) | ||||
* - TYPE : return type of the getter | ||||
*/ | ||||
// clang-format off | ||||
#define DECLARE_PROPERTY_GETTER(GETTER_NAME, PROPERTY_NAME, TYPE) \ | ||||
TYPE GETTER_NAME() const \ | ||||
{ \ | ||||
static auto result = m_Properties.value(PROPERTY_NAME##_PROPERTY, PROPERTY_NAME##_DEFAULT_VALUE).value<TYPE>(); \ | ||||
return result; \ | ||||
} \ | ||||
// clang-format on | ||||
|
r1199 | namespace { | ||
|
r1202 | // /////// // | ||
// Aliases // | ||||
// /////// // | ||||
|
r1245 | using IntPair = std::pair<int, int>; | ||
|
r1216 | using Weight = double; | ||
using Weights = std::vector<Weight>; | ||||
|
r1203 | |||
|
r1249 | struct OperationProperty { | ||
Weight m_Weight{1.}; | ||||
bool m_WaitAcquisition{false}; | ||||
}; | ||||
|
r1205 | using VariableOperation = std::pair<VariableId, std::shared_ptr<IFuzzingOperation> >; | ||
using VariablesOperations = std::vector<VariableOperation>; | ||||
|
r1249 | using OperationsTypes = std::map<FuzzingOperationType, OperationProperty>; | ||
using OperationsPool = std::map<std::shared_ptr<IFuzzingOperation>, OperationProperty>; | ||||
|
r1228 | using Validators = std::vector<std::shared_ptr<IFuzzingValidator> >; | ||
|
r1203 | |||
|
r1201 | // ///////// // | ||
// Constants // | ||||
// ///////// // | ||||
// Defaults values used when the associated properties have not been set for the test | ||||
|
r1248 | const auto ACQUISITION_TIMEOUT_DEFAULT_VALUE = 30000; | ||
|
r1201 | const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100; | ||
|
r1237 | const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1; | ||
|
r1201 | const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 1; | ||
|
r1225 | const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue(WeightedOperationsTypes{ | ||
{FuzzingOperationType::CREATE, 1.}, | ||||
{FuzzingOperationType::DELETE, 0.1}, // Delete operation is less frequent | ||||
{FuzzingOperationType::PAN_LEFT, 1.}, | ||||
{FuzzingOperationType::PAN_RIGHT, 1.}, | ||||
{FuzzingOperationType::ZOOM_IN, 1.}, | ||||
|
r1238 | {FuzzingOperationType::ZOOM_OUT, 1.}, | ||
{FuzzingOperationType::SYNCHRONIZE, 0.8}, | ||||
{FuzzingOperationType::DESYNCHRONIZE, 0.4}}); | ||||
|
r1224 | const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2; | ||
|
r1219 | |||
|
r1243 | /// Min/max delays between each operation (in ms) | ||
const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(100, 3000)); | ||||
|
r1209 | |||
|
r1228 | /// Validators for the tests (executed in the order in which they're defined) | ||
const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue( | ||||
ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}}); | ||||
|
r1244 | /// Min/max number of operations to execute before calling validation | ||
const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(1, 10)); | ||||
|
r1205 | // /////// // | ||
// Methods // | ||||
// /////// // | ||||
/// Goes through the variables pool and operations pool to determine the set of {variable/operation} | ||||
/// pairs that are valid (i.e. operation that can be executed on variable) | ||||
|
r1249 | std::pair<VariablesOperations, Weights> availableOperations(const FuzzingState &fuzzingState, | ||
const OperationsPool &operationsPool) | ||||
|
r1205 | { | ||
VariablesOperations result{}; | ||||
|
r1216 | Weights weights{}; | ||
|
r1205 | |||
|
r1236 | for (const auto &variablesPoolEntry : fuzzingState.m_VariablesPool) { | ||
|
r1205 | auto variableId = variablesPoolEntry.first; | ||
|
r1216 | for (const auto &operationsPoolEntry : operationsPool) { | ||
auto operation = operationsPoolEntry.first; | ||||
|
r1249 | auto operationProperty = operationsPoolEntry.second; | ||
|
r1216 | |||
|
r1205 | // A pair is valid if the current operation can be executed on the current variable | ||
|
r1236 | if (operation->canExecute(variableId, fuzzingState)) { | ||
|
r1205 | result.push_back({variableId, operation}); | ||
|
r1249 | weights.push_back(operationProperty.m_Weight); | ||
|
r1205 | } | ||
} | ||||
} | ||||
|
r1216 | return {result, weights}; | ||
|
r1205 | } | ||
|
r1249 | OperationsPool createOperationsPool(const OperationsTypes &types) | ||
|
r1204 | { | ||
|
r1249 | OperationsPool result{}; | ||
|
r1204 | |||
|
r1216 | std::transform( | ||
types.cbegin(), types.cend(), std::inserter(result, result.end()), [](const auto &type) { | ||||
return std::make_pair(FuzzingOperationFactory::create(type.first), type.second); | ||||
}); | ||||
|
r1204 | |||
return result; | ||||
} | ||||
|
r1228 | Validators createValidators(const ValidatorsTypes &types) | ||
{ | ||||
Validators result{}; | ||||
std::transform(types.cbegin(), types.cend(), std::inserter(result, result.end()), | ||||
[](const auto &type) { return FuzzingValidatorFactory::create(type); }); | ||||
return result; | ||||
} | ||||
|
r1229 | /** | ||
* Validates all the variables' states passed in parameter, according to a set of validators | ||||
* @param variablesPool the variables' states | ||||
* @param validators the validators used for validation | ||||
*/ | ||||
void validate(const VariablesPool &variablesPool, const Validators &validators) | ||||
{ | ||||
for (const auto &variablesPoolEntry : variablesPool) { | ||||
auto variableId = variablesPoolEntry.first; | ||||
const auto &variableState = variablesPoolEntry.second; | ||||
auto variableMessage = variableState.m_Variable ? variableState.m_Variable->name() | ||||
: QStringLiteral("null variable"); | ||||
qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Validating state of variable at index" | ||||
<< variableId << "(" << variableMessage << ")..."; | ||||
for (const auto &validator : validators) { | ||||
validator->validate(VariableState{variableState}); | ||||
} | ||||
qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Validation completed."; | ||||
} | ||||
} | ||||
|
r1199 | /** | ||
* Class to run random tests | ||||
*/ | ||||
class FuzzingTest { | ||||
public: | ||||
|
r1200 | explicit FuzzingTest(VariableController &variableController, Properties properties) | ||
: m_VariableController{variableController}, | ||||
m_Properties{std::move(properties)}, | ||||
|
r1236 | m_FuzzingState{} | ||
|
r1200 | { | ||
|
r1203 | // Inits variables pool: at init, all variables are null | ||
for (auto variableId = 0; variableId < nbMaxVariables(); ++variableId) { | ||||
|
r1236 | m_FuzzingState.m_VariablesPool[variableId] = VariableState{}; | ||
|
r1203 | } | ||
|
r1237 | |||
// Inits sync groups and registers them into the variable controller | ||||
for (auto i = 0; i < nbMaxSyncGroups(); ++i) { | ||||
auto syncGroupId = SyncGroupId::createUuid(); | ||||
variableController.onAddSynchronizationGroupId(syncGroupId); | ||||
m_FuzzingState.m_SyncGroupsPool[syncGroupId] = SyncGroup{}; | ||||
} | ||||
|
r1200 | } | ||
|
r1199 | void execute() | ||
{ | ||||
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbMaxOperations() << "operations on" | ||
<< nbMaxVariables() << "variable(s)..."; | ||||
|
r1201 | |||
|
r1244 | |||
// Inits the count of the number of operations before the next validation | ||||
int nextValidationCounter = 0; | ||||
auto updateValidationCounter = [this, &nextValidationCounter]() { | ||||
nextValidationCounter = RandomGenerator::instance().generateInt( | ||||
validationFrequencies().first, validationFrequencies().second); | ||||
qCInfo(LOG_TestAmdaFuzzing()).noquote() | ||||
|
r1248 | << "Next validation in " << nextValidationCounter << "operation(s)..."; | ||
|
r1244 | }; | ||
updateValidationCounter(); | ||||
|
r1205 | auto canExecute = true; | ||
for (auto i = 0; i < nbMaxOperations() && canExecute; ++i) { | ||||
// Retrieves all operations that can be executed in the current context | ||||
|
r1216 | VariablesOperations variableOperations{}; | ||
Weights weights{}; | ||||
std::tie(variableOperations, weights) | ||||
|
r1236 | = availableOperations(m_FuzzingState, operationsPool()); | ||
|
r1205 | |||
canExecute = !variableOperations.empty(); | ||||
if (canExecute) { | ||||
|
r1244 | --nextValidationCounter; | ||
|
r1205 | // Of the operations available, chooses a random operation and executes it | ||
auto variableOperation | ||||
|
r1216 | = RandomGenerator::instance().randomChoice(variableOperations, weights); | ||
|
r1205 | |||
auto variableId = variableOperation.first; | ||||
auto fuzzingOperation = variableOperation.second; | ||||
|
r1250 | auto waitAcquisition = nextValidationCounter == 0 | ||
|| operationsPool().at(fuzzingOperation).m_WaitAcquisition; | ||||
|
r1248 | |||
|
r1236 | fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController, | ||
m_Properties); | ||||
|
r1243 | |||
|
r1248 | if (waitAcquisition) { | ||
qCDebug(LOG_TestAmdaFuzzing()) << "Waiting for acquisition to finish..."; | ||||
SignalWaiter{m_VariableController, SIGNAL(acquisitionFinished())}.wait( | ||||
acquisitionTimeout()); | ||||
|
r1229 | |||
|
r1248 | // Validates variables | ||
|
r1250 | if (nextValidationCounter == 0) { | ||
validate(m_FuzzingState.m_VariablesPool, validators()); | ||||
updateValidationCounter(); | ||||
} | ||||
|
r1244 | } | ||
|
r1248 | else { | ||
// Delays the next operation with a randomly generated time | ||||
auto delay = RandomGenerator::instance().generateInt(operationDelays().first, | ||||
operationDelays().second); | ||||
qCDebug(LOG_TestAmdaFuzzing()) | ||||
<< "Waiting " << delay << "ms before the next operation..."; | ||||
QTest::qWait(delay); | ||||
} | ||||
|
r1205 | } | ||
else { | ||||
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() | ||
|
r1205 | << "No more operations are available, the execution of the test will stop..."; | ||
} | ||||
} | ||||
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Execution of the test completed."; | ||
|
r1199 | } | ||
|
r1200 | |||
private: | ||||
|
r1249 | OperationsPool operationsPool() const | ||
|
r1204 | { | ||
static auto result = createOperationsPool( | ||||
m_Properties.value(AVAILABLE_OPERATIONS_PROPERTY, AVAILABLE_OPERATIONS_DEFAULT_VALUE) | ||||
|
r1249 | .value<OperationsTypes>()); | ||
|
r1204 | return result; | ||
} | ||||
|
r1228 | Validators validators() const | ||
{ | ||||
static auto result | ||||
= createValidators(m_Properties.value(VALIDATORS_PROPERTY, VALIDATORS_DEFAULT_VALUE) | ||||
.value<ValidatorsTypes>()); | ||||
return result; | ||||
} | ||||
|
r1245 | DECLARE_PROPERTY_GETTER(nbMaxOperations, NB_MAX_OPERATIONS, int) | ||
DECLARE_PROPERTY_GETTER(nbMaxSyncGroups, NB_MAX_SYNC_GROUPS, int) | ||||
DECLARE_PROPERTY_GETTER(nbMaxVariables, NB_MAX_VARIABLES, int) | ||||
DECLARE_PROPERTY_GETTER(operationDelays, OPERATION_DELAY_BOUNDS, IntPair) | ||||
DECLARE_PROPERTY_GETTER(validationFrequencies, VALIDATION_FREQUENCY_BOUNDS, IntPair) | ||||
DECLARE_PROPERTY_GETTER(acquisitionTimeout, ACQUISITION_TIMEOUT, int) | ||||
|
r1244 | |||
|
r1200 | VariableController &m_VariableController; | ||
Properties m_Properties; | ||||
|
r1236 | FuzzingState m_FuzzingState; | ||
|
r1199 | }; | ||
} // namespace | ||||
|
r1249 | Q_DECLARE_METATYPE(OperationsTypes) | ||
|
r1199 | class TestAmdaFuzzing : public QObject { | ||
Q_OBJECT | ||||
private slots: | ||||
/// Input data for @sa testFuzzing() | ||||
void testFuzzing_data(); | ||||
void testFuzzing(); | ||||
}; | ||||
void TestAmdaFuzzing::testFuzzing_data() | ||||
{ | ||||
// ////////////// // | ||||
// Test structure // | ||||
// ////////////// // | ||||
|
r1200 | QTest::addColumn<Properties>("properties"); // Properties for random test | ||
|
r1199 | |||
// ////////// // | ||||
// Test cases // | ||||
// ////////// // | ||||
|
r1210 | auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0}); | ||
MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "imf"}}}; | ||||
// Note: we don't use auto here as we want to pass std::shared_ptr<IDataProvider> as is in the | ||||
// QVariant | ||||
std::shared_ptr<IDataProvider> provider = std::make_shared<AmdaProvider>(); | ||||
QTest::newRow("fuzzingTest") << Properties{ | ||||
{MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)}, | ||||
{METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)}, | ||||
{PROVIDER_PROPERTY, QVariant::fromValue(provider)}}; | ||||
|
r1199 | } | ||
void TestAmdaFuzzing::testFuzzing() | ||||
{ | ||||
|
r1200 | QFETCH(Properties, properties); | ||
|
r1224 | // Sets cache property | ||
QSettings settings{}; | ||||
auto cacheTolerance = properties.value(CACHE_TOLERANCE_PROPERTY, CACHE_TOLERANCE_DEFAULT_VALUE); | ||||
settings.setValue(GENERAL_TOLERANCE_AT_INIT_KEY, cacheTolerance); | ||||
settings.setValue(GENERAL_TOLERANCE_AT_UPDATE_KEY, cacheTolerance); | ||||
|
r1199 | auto &variableController = sqpApp->variableController(); | ||
auto &timeController = sqpApp->timeController(); | ||||
|
r1209 | // Generates random initial range (bounded to max range) | ||
auto maxRange = properties.value(MAX_RANGE_PROPERTY, QVariant::fromValue(INVALID_RANGE)) | ||||
.value<SqpRange>(); | ||||
QVERIFY(maxRange != INVALID_RANGE); | ||||
auto initialRangeStart | ||||
= RandomGenerator::instance().generateDouble(maxRange.m_TStart, maxRange.m_TEnd); | ||||
auto initialRangeEnd | ||||
= RandomGenerator::instance().generateDouble(maxRange.m_TStart, maxRange.m_TEnd); | ||||
if (initialRangeStart > initialRangeEnd) { | ||||
std::swap(initialRangeStart, initialRangeEnd); | ||||
} | ||||
// Sets initial range on time controller | ||||
SqpRange initialRange{initialRangeStart, initialRangeEnd}; | ||||
|
r1223 | qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Setting initial range to" << initialRange << "..."; | ||
|
r1209 | timeController.onTimeToUpdate(initialRange); | ||
|
r1223 | properties.insert(INITIAL_RANGE_PROPERTY, QVariant::fromValue(initialRange)); | ||
|
r1209 | |||
|
r1200 | FuzzingTest test{variableController, properties}; | ||
|
r1199 | test.execute(); | ||
} | ||||
int main(int argc, char *argv[]) | ||||
{ | ||||
QLoggingCategory::setFilterRules( | ||||
"*.warning=false\n" | ||||
"*.info=false\n" | ||||
"*.debug=false\n" | ||||
"FuzzingOperations.info=true\n" | ||||
|
r1228 | "FuzzingValidators.info=true\n" | ||
|
r1199 | "TestAmdaFuzzing.info=true\n"); | ||
SqpApplication app{argc, argv}; | ||||
|
r1224 | SqpApplication::setOrganizationName("LPP"); | ||
SqpApplication::setOrganizationDomain("lpp.fr"); | ||||
SqpApplication::setApplicationName("SciQLop-TestFuzzing"); | ||||
|
r1199 | app.setAttribute(Qt::AA_Use96Dpi, true); | ||
TestAmdaFuzzing testObject{}; | ||||
QTEST_SET_MAIN_SOURCE_PATH | ||||
return QTest::qExec(&testObject, argc, argv); | ||||
} | ||||
#include "TestAmdaFuzzing.moc" | ||||