GIT_REPOSITORIES/LPP/Users/mperrinel/SciQLop-fork Commit - r1244:7202dffeda15

Randomizes the number of operations to wait between calling validation (defines min/max frequencies)

Alexandre Leroux -

r1244:7202dffeda15

parent child

Context file:

r1244:7202dffeda15

Collapse all files

plugins/amda/tests/FuzzingDefs.cpp +1 0

             #include "FuzzingDefs.h"
             const QString NB_MAX_OPERATIONS_PROPERTY = QStringLiteral("component");
             const QString NB_MAX_SYNC_GROUPS_PROPERTY = QStringLiteral("nbSyncGroups");
             const QString NB_MAX_VARIABLES_PROPERTY = QStringLiteral("nbMaxVariables");
             const QString AVAILABLE_OPERATIONS_PROPERTY = QStringLiteral("availableOperations");
             const QString CACHE_TOLERANCE_PROPERTY = QStringLiteral("cacheTolerance");
             const QString INITIAL_RANGE_PROPERTY = QStringLiteral("initialRange");
             const QString MAX_RANGE_PROPERTY = QStringLiteral("maxRange");
             const QString METADATA_POOL_PROPERTY = QStringLiteral("metadataPool");
             const QString PROVIDER_PROPERTY = QStringLiteral("provider");
             const QString OPERATION_DELAY_BOUNDS_PROPERTY = QStringLiteral("operationDelays");
             const QString VALIDATORS_PROPERTY = QStringLiteral("validators");
+            const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY = QStringLiteral("validationFrequencyBounds");
             // //////////// //
             // FuzzingState //
             // //////////// //
             const SyncGroup &FuzzingState::syncGroup(SyncGroupId id) const
             {
                 return m_SyncGroupsPool.at(id);
             }
             SyncGroup &FuzzingState::syncGroup(SyncGroupId id)
             {
                 return m_SyncGroupsPool.at(id);
             }
             const VariableState &FuzzingState::variableState(VariableId id) const
             {
                 return m_VariablesPool.at(id);
             }
             VariableState &FuzzingState::variableState(VariableId id)
             {
                 return m_VariablesPool.at(id);
             }
             SyncGroupId FuzzingState::syncGroupId(VariableId variableId) const
             {
                 auto end = m_SyncGroupsPool.cend();
                 auto it
                     = std::find_if(m_SyncGroupsPool.cbegin(), end, [&variableId](const auto &syncGroupEntry) {
                           const auto &syncGroup = syncGroupEntry.second;
                           return syncGroup.m_Variables.find(variableId) != syncGroup.m_Variables.end();
                       });
                 return it != end ? it->first : SyncGroupId{};
             }
             std::vector<SyncGroupId> FuzzingState::syncGroupsIds() const
             {
                 std::vector<SyncGroupId> result{};
                 for (const auto &entry : m_SyncGroupsPool) {
                     result.push_back(entry.first);
                 }
                 return result;
             }
             void FuzzingState::synchronizeVariable(VariableId variableId, SyncGroupId syncGroupId)
             {
                 if (syncGroupId.isNull()) {
                     return;
                 }
                 // Registers variable into sync group: if it's the first variable, sets the variable range as
                 // the sync group range
                 auto &syncGroup = m_SyncGroupsPool.at(syncGroupId);
                 syncGroup.m_Variables.insert(variableId);
                 if (syncGroup.m_Variables.size() == 1) {
                     auto &variableState = m_VariablesPool.at(variableId);
                     syncGroup.m_Range = variableState.m_Range;
                 }
             }
             void FuzzingState::desynchronizeVariable(VariableId variableId, SyncGroupId syncGroupId)
             {
                 if (syncGroupId.isNull()) {
                     return;
                 }
                 // Unregisters variable from sync group: if there is no more variable in the group, resets the
                 // range
                 auto &syncGroup = m_SyncGroupsPool.at(syncGroupId);
                 syncGroup.m_Variables.erase(variableId);
                 if (syncGroup.m_Variables.empty()) {
                     syncGroup.m_Range = INVALID_RANGE;
                 }
             }
             void FuzzingState::updateRanges(VariableId variableId, const SqpRange &newRange)
             {
                 auto syncGroupId = this->syncGroupId(variableId);
                 // Retrieves the variables to update:
                 // - if the variable is synchronized to others, updates all synchronized variables
                 // - otherwise, updates only the variable
                 auto variablesToUpdate = syncGroupId.isNull() ? std::set<VariableId>{variableId}
                                                               : m_SyncGroupsPool.at(syncGroupId).m_Variables;
                 // Sets new range
                 for (const auto &variableId : variablesToUpdate) {
                     m_VariablesPool.at(variableId).m_Range = newRange;
                 }
             }

plugins/amda/tests/FuzzingDefs.h +3 0

             #ifndef SCIQLOP_FUZZINGDEFS_H
             #define SCIQLOP_FUZZINGDEFS_H
             #include <Data/SqpRange.h>
             #include <QString>
             #include <QUuid>
             #include <QVariantHash>
             #include <memory>
             #include <set>
             // /////// //
             // Aliases //
             // /////// //
             using MetadataPool = std::vector<QVariantHash>;
             Q_DECLARE_METATYPE(MetadataPool)
             using Properties = QVariantHash;
             // ///////// //
             // Constants //
             // ///////// //
             /// Max number of operations to generate
             extern const QString NB_MAX_OPERATIONS_PROPERTY;
             /// Max number of sync groups to create through operations
             extern const QString NB_MAX_SYNC_GROUPS_PROPERTY;
             /// Max number of variables to manipulate through operations
             extern const QString NB_MAX_VARIABLES_PROPERTY;
             /// Set of operations available for the test
             extern const QString AVAILABLE_OPERATIONS_PROPERTY;
             /// Tolerance used for variable's cache (in ratio)
             extern const QString CACHE_TOLERANCE_PROPERTY;
             /// Range with which the timecontroller is initialized
             extern const QString INITIAL_RANGE_PROPERTY;
             /// Max range that an operation can reach
             extern const QString MAX_RANGE_PROPERTY;
             /// Set of metadata that can be associated to a variable
             extern const QString METADATA_POOL_PROPERTY;
             /// Provider used to retrieve data
             extern const QString PROVIDER_PROPERTY;
             /// Min/max times left for an operation to execute
             extern const QString OPERATION_DELAY_BOUNDS_PROPERTY;
             /// Validators used to validate an operation
             extern const QString VALIDATORS_PROPERTY;
+            /// Min/max number of operations to execute before calling validation of the current test's state
+            extern const QString VALIDATION_FREQUENCY_BOUNDS_PROPERTY;
             // /////// //
             // Structs //
             // /////// //
             class Variable;
             struct VariableState {
                 std::shared_ptr<Variable> m_Variable{nullptr};
                 SqpRange m_Range{INVALID_RANGE};
             };
             using VariableId = int;
             using VariablesPool = std::map<VariableId, VariableState>;
             /**
              * Defines a synchronization group for a fuzzing state. A group reports the variables synchronized
              * with each other, and the current range of the group (i.e. range of the last synchronized variable
              * that has been moved)
              */
             struct SyncGroup {
                 std::set<VariableId> m_Variables{};
                 SqpRange m_Range{INVALID_RANGE};
             };
             using SyncGroupId = QUuid;
             using SyncGroupsPool = std::map<SyncGroupId, SyncGroup>;
             /**
              * Defines a current state during a fuzzing state. It contains all the variables manipulated during
              * the test, as well as the synchronization status of these variables.
              */
             struct FuzzingState {
                 const SyncGroup &syncGroup(SyncGroupId id) const;
                 SyncGroup &syncGroup(SyncGroupId id);
                 const VariableState &variableState(VariableId id) const;
                 VariableState &variableState(VariableId id);
                 /// @return the identifier of the synchronization group in which the variable passed in
                 /// parameter is located. If the variable is not in any group, returns an invalid identifier
                 SyncGroupId syncGroupId(VariableId variableId) const;
                 /// @return the set of synchronization group identifiers
                 std::vector<SyncGroupId> syncGroupsIds() const;
                 /// Updates fuzzing state according to a variable synchronization
                 /// @param variableId the variable that is synchronized
                 /// @param syncGroupId the synchronization group
                 void synchronizeVariable(VariableId variableId, SyncGroupId syncGroupId);
                 /// Updates fuzzing state according to a variable desynchronization
                 /// @param variableId the variable that is desynchronized
                 /// @param syncGroupId the synchronization group from which to remove the variable
                 void desynchronizeVariable(VariableId variableId, SyncGroupId syncGroupId);
                 /// Updates the range of a variable and all variables to which it is synchronized
                 /// @param the variable for which to affect the range
                 /// @param the range to affect
                 void updateRanges(VariableId variableId, const SqpRange &newRange);
                 VariablesPool m_VariablesPool;
                 SyncGroupsPool m_SyncGroupsPool;
             };
             #endif // SCIQLOP_FUZZINGDEFS_H

plugins/amda/tests/TestAmdaFuzzing.cpp +28 0

             #include "FuzzingDefs.h"
             #include "FuzzingOperations.h"
             #include "FuzzingUtils.h"
             #include "FuzzingValidators.h"
             #include "AmdaProvider.h"
             #include <Network/NetworkController.h>
             #include <Settings/SqpSettingsDefs.h>
             #include <SqpApplication.h>
             #include <Time/TimeController.h>
             #include <Variable/Variable.h>
             #include <Variable/VariableController.h>
             #include <QLoggingCategory>
             #include <QObject>
             #include <QtTest>
             #include <memory>
             Q_LOGGING_CATEGORY(LOG_TestAmdaFuzzing, "TestAmdaFuzzing")
             namespace {
             // /////// //
             // Aliases //
             // /////// //
             using Weight = double;
             using Weights = std::vector<Weight>;
             using VariableOperation = std::pair<VariableId, std::shared_ptr<IFuzzingOperation> >;
             using VariablesOperations = std::vector<VariableOperation>;
             using WeightedOperationsPool = std::map<std::shared_ptr<IFuzzingOperation>, Weight>;
             using Validators = std::vector<std::shared_ptr<IFuzzingValidator> >;
             // ///////// //
             // Constants //
             // ///////// //
             // Defaults values used when the associated properties have not been set for the test
             const auto NB_MAX_OPERATIONS_DEFAULT_VALUE = 100;
             const auto NB_MAX_SYNC_GROUPS_DEFAULT_VALUE = 1;
             const auto NB_MAX_VARIABLES_DEFAULT_VALUE = 1;
             const auto AVAILABLE_OPERATIONS_DEFAULT_VALUE = QVariant::fromValue(WeightedOperationsTypes{
                 {FuzzingOperationType::CREATE, 1.},
                 {FuzzingOperationType::DELETE, 0.1}, // Delete operation is less frequent
                 {FuzzingOperationType::PAN_LEFT, 1.},
                 {FuzzingOperationType::PAN_RIGHT, 1.},
                 {FuzzingOperationType::ZOOM_IN, 1.},
                 {FuzzingOperationType::ZOOM_OUT, 1.},
                 {FuzzingOperationType::SYNCHRONIZE, 0.8},
                 {FuzzingOperationType::DESYNCHRONIZE, 0.4}});
             const auto CACHE_TOLERANCE_DEFAULT_VALUE = 0.2;
             /// Min/max delays between each operation (in ms)
             const auto OPERATION_DELAY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(100, 3000));
             /// Validators for the tests (executed in the order in which they're defined)
             const auto VALIDATORS_DEFAULT_VALUE = QVariant::fromValue(
                 ValidatorsTypes{{FuzzingValidatorType::RANGE, FuzzingValidatorType::DATA}});
+            /// Min/max number of operations to execute before calling validation
+            const auto VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE = QVariant::fromValue(std::make_pair(1, 10));
             // /////// //
             // Methods //
             // /////// //
             /// Goes through the variables pool and operations pool to determine the set of {variable/operation}
             /// pairs that are valid (i.e. operation that can be executed on variable)
             std::pair<VariablesOperations, Weights>
             availableOperations(const FuzzingState &fuzzingState, const WeightedOperationsPool &operationsPool)
             {
                 VariablesOperations result{};
                 Weights weights{};
                 for (const auto &variablesPoolEntry : fuzzingState.m_VariablesPool) {
                     auto variableId = variablesPoolEntry.first;
                     for (const auto &operationsPoolEntry : operationsPool) {
                         auto operation = operationsPoolEntry.first;
                         auto weight = operationsPoolEntry.second;
                         // A pair is valid if the current operation can be executed on the current variable
                         if (operation->canExecute(variableId, fuzzingState)) {
                             result.push_back({variableId, operation});
                             weights.push_back(weight);
                         }
                     }
                 }
                 return {result, weights};
             }
             WeightedOperationsPool createOperationsPool(const WeightedOperationsTypes &types)
             {
                 WeightedOperationsPool result{};
                 std::transform(
                     types.cbegin(), types.cend(), std::inserter(result, result.end()), [](const auto &type) {
                         return std::make_pair(FuzzingOperationFactory::create(type.first), type.second);
                     });
                 return result;
             }
             Validators createValidators(const ValidatorsTypes &types)
             {
                 Validators result{};
                 std::transform(types.cbegin(), types.cend(), std::inserter(result, result.end()),
                                [](const auto &type) { return FuzzingValidatorFactory::create(type); });
                 return result;
             }
             /**
              * Validates all the variables' states passed in parameter, according to a set of validators
              * @param variablesPool the variables' states
              * @param validators the validators used for validation
              */
             void validate(const VariablesPool &variablesPool, const Validators &validators)
             {
                 for (const auto &variablesPoolEntry : variablesPool) {
                     auto variableId = variablesPoolEntry.first;
                     const auto &variableState = variablesPoolEntry.second;
                     auto variableMessage = variableState.m_Variable ? variableState.m_Variable->name()
                                                                     : QStringLiteral("null variable");
                     qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Validating state of variable at index"
                                                             << variableId << "(" << variableMessage << ")...";
                     for (const auto &validator : validators) {
                         validator->validate(VariableState{variableState});
                     }
                     qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Validation completed.";
                 }
             }
             /**
              * Class to run random tests
              */
             class FuzzingTest {
             public:
                 explicit FuzzingTest(VariableController &variableController, Properties properties)
                         : m_VariableController{variableController},
                           m_Properties{std::move(properties)},
                           m_FuzzingState{}
                 {
                     // Inits variables pool: at init, all variables are null
                     for (auto variableId = 0; variableId < nbMaxVariables(); ++variableId) {
                         m_FuzzingState.m_VariablesPool[variableId] = VariableState{};
                     }
                     // Inits sync groups and registers them into the variable controller
                     for (auto i = 0; i < nbMaxSyncGroups(); ++i) {
                         auto syncGroupId = SyncGroupId::createUuid();
                         variableController.onAddSynchronizationGroupId(syncGroupId);
                         m_FuzzingState.m_SyncGroupsPool[syncGroupId] = SyncGroup{};
                     }
                 }
                 void execute()
                 {
                     qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Running" << nbMaxOperations() << "operations on"
                                                             << nbMaxVariables() << "variable(s)...";
+                    // Inits the count of the number of operations before the next validation
+                    int nextValidationCounter = 0;
+                    auto updateValidationCounter = [this, &nextValidationCounter]() {
+                        nextValidationCounter = RandomGenerator::instance().generateInt(
+                            validationFrequencies().first, validationFrequencies().second);
+                        qCInfo(LOG_TestAmdaFuzzing()).noquote()
+                            << "Next validation in " << nextValidationCounter << "operations...";
+                    };
+                    updateValidationCounter();
                     auto canExecute = true;
                     for (auto i = 0; i < nbMaxOperations() && canExecute; ++i) {
                         // Retrieves all operations that can be executed in the current context
                         VariablesOperations variableOperations{};
                         Weights weights{};
                         std::tie(variableOperations, weights)
                             = availableOperations(m_FuzzingState, operationsPool());
                         canExecute = !variableOperations.empty();
                         if (canExecute) {
+                            --nextValidationCounter;
                             // Of the operations available, chooses a random operation and executes it
                             auto variableOperation
                                 = RandomGenerator::instance().randomChoice(variableOperations, weights);
                             auto variableId = variableOperation.first;
                             auto fuzzingOperation = variableOperation.second;
                             fuzzingOperation->execute(variableId, m_FuzzingState, m_VariableController,
                                                       m_Properties);
                             // Delays the next operation with a randomly generated time
                             auto delay = RandomGenerator::instance().generateInt(operationDelays().first,
                                                                                  operationDelays().second);
                             qCDebug(LOG_TestAmdaFuzzing())
                                 << "Waiting " << delay << "ms before the next operation...";
                             QTest::qWait(delay);
                             // Validates variables
+                            if (nextValidationCounter == 0) {
                                 validate(m_FuzzingState.m_VariablesPool, validators());
+                                updateValidationCounter();
+                            }
                         }
                         else {
                             qCInfo(LOG_TestAmdaFuzzing()).noquote()
                                 << "No more operations are available, the execution of the test will stop...";
                         }
                     }
                     qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Execution of the test completed.";
                 }
             private:
                 int nbMaxOperations() const
                 {
                     static auto result
                         = m_Properties.value(NB_MAX_OPERATIONS_PROPERTY, NB_MAX_OPERATIONS_DEFAULT_VALUE)
                               .toInt();
                     return result;
                 }
                 int nbMaxSyncGroups() const
                 {
                     static auto result
                         = m_Properties.value(NB_MAX_SYNC_GROUPS_PROPERTY, NB_MAX_SYNC_GROUPS_DEFAULT_VALUE)
                               .toInt();
                     return result;
                 }
                 int nbMaxVariables() const
                 {
                     static auto result
                         = m_Properties.value(NB_MAX_VARIABLES_PROPERTY, NB_MAX_VARIABLES_DEFAULT_VALUE).toInt();
                     return result;
                 }
                 std::pair<int, int> operationDelays() const
                 {
                     static auto result
                         = m_Properties
                               .value(OPERATION_DELAY_BOUNDS_PROPERTY, OPERATION_DELAY_BOUNDS_DEFAULT_VALUE)
                               .value<std::pair<int, int> >();
                     return result;
                 }
                 WeightedOperationsPool operationsPool() const
                 {
                     static auto result = createOperationsPool(
                         m_Properties.value(AVAILABLE_OPERATIONS_PROPERTY, AVAILABLE_OPERATIONS_DEFAULT_VALUE)
                             .value<WeightedOperationsTypes>());
                     return result;
                 }
                 Validators validators() const
                 {
                     static auto result
                         = createValidators(m_Properties.value(VALIDATORS_PROPERTY, VALIDATORS_DEFAULT_VALUE)
                                                .value<ValidatorsTypes>());
                     return result;
                 }
+                std::pair<int, int> validationFrequencies() const
+                {
+                    static auto result = m_Properties
+                                             .value(VALIDATION_FREQUENCY_BOUNDS_PROPERTY,
+                                                    VALIDATION_FREQUENCY_BOUNDS_DEFAULT_VALUE)
+                                             .value<std::pair<int, int> >();
+                    return result;
+                }
                 VariableController &m_VariableController;
                 Properties m_Properties;
                 FuzzingState m_FuzzingState;
             };
             } // namespace
             class TestAmdaFuzzing : public QObject {
                 Q_OBJECT
             private slots:
                 /// Input data for @sa testFuzzing()
                 void testFuzzing_data();
                 void testFuzzing();
             };
             void TestAmdaFuzzing::testFuzzing_data()
             {
                 // ////////////// //
                 // Test structure //
                 // ////////////// //
                 QTest::addColumn<Properties>("properties"); // Properties for random test
                 // ////////// //
                 // Test cases //
                 // ////////// //
                 auto maxRange = SqpRange::fromDateTime({2017, 1, 1}, {0, 0}, {2017, 1, 5}, {0, 0});
                 MetadataPool metadataPool{{{"dataType", "vector"}, {"xml:id", "imf"}}};
                 // Note: we don't use auto here as we want to pass std::shared_ptr<IDataProvider> as is in the
                 // QVariant
                 std::shared_ptr<IDataProvider> provider = std::make_shared<AmdaProvider>();
                 QTest::newRow("fuzzingTest") << Properties{
                     {MAX_RANGE_PROPERTY, QVariant::fromValue(maxRange)},
                     {METADATA_POOL_PROPERTY, QVariant::fromValue(metadataPool)},
                     {PROVIDER_PROPERTY, QVariant::fromValue(provider)}};
             }
             void TestAmdaFuzzing::testFuzzing()
             {
                 QFETCH(Properties, properties);
                 // Sets cache property
                 QSettings settings{};
                 auto cacheTolerance = properties.value(CACHE_TOLERANCE_PROPERTY, CACHE_TOLERANCE_DEFAULT_VALUE);
                 settings.setValue(GENERAL_TOLERANCE_AT_INIT_KEY, cacheTolerance);
                 settings.setValue(GENERAL_TOLERANCE_AT_UPDATE_KEY, cacheTolerance);
                 auto &variableController = sqpApp->variableController();
                 auto &timeController = sqpApp->timeController();
                 // Generates random initial range (bounded to max range)
                 auto maxRange = properties.value(MAX_RANGE_PROPERTY, QVariant::fromValue(INVALID_RANGE))
                                     .value<SqpRange>();
                 QVERIFY(maxRange != INVALID_RANGE);
                 auto initialRangeStart
                     = RandomGenerator::instance().generateDouble(maxRange.m_TStart, maxRange.m_TEnd);
                 auto initialRangeEnd
                     = RandomGenerator::instance().generateDouble(maxRange.m_TStart, maxRange.m_TEnd);
                 if (initialRangeStart > initialRangeEnd) {
                     std::swap(initialRangeStart, initialRangeEnd);
                 }
                 // Sets initial range on time controller
                 SqpRange initialRange{initialRangeStart, initialRangeEnd};
                 qCInfo(LOG_TestAmdaFuzzing()).noquote() << "Setting initial range to" << initialRange << "...";
                 timeController.onTimeToUpdate(initialRange);
                 properties.insert(INITIAL_RANGE_PROPERTY, QVariant::fromValue(initialRange));
                 FuzzingTest test{variableController, properties};
                 test.execute();
             }
             int main(int argc, char *argv[])
             {
                 QLoggingCategory::setFilterRules(
                     "*.warning=false\n"
                     "*.info=false\n"
                     "*.debug=false\n"
                     "FuzzingOperations.info=true\n"
                     "FuzzingValidators.info=true\n"
                     "TestAmdaFuzzing.info=true\n");
                 SqpApplication app{argc, argv};
                 SqpApplication::setOrganizationName("LPP");
                 SqpApplication::setOrganizationDomain("lpp.fr");
                 SqpApplication::setApplicationName("SciQLop-TestFuzzing");
                 app.setAttribute(Qt::AA_Use96Dpi, true);
                 TestAmdaFuzzing testObject{};
                 QTEST_SET_MAIN_SOURCE_PATH
                 return QTest::qExec(&testObject, argc, argv);
             }
             #include "TestAmdaFuzzing.moc"

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages