diff --git a/core/include/Variable/VariableController.h b/core/include/Variable/VariableController.h index 9238f8a..c98eedf 100644 --- a/core/include/Variable/VariableController.h +++ b/core/include/Variable/VariableController.h @@ -55,6 +55,9 @@ public: QList > variablesForMimeData(const QByteArray &mimeData) const; static AcquisitionZoomType getZoomType(const SqpRange &range, const SqpRange &oldRange); + + /// Returns True if there are pending downloads + bool hasPendingDownloads(); signals: /// Signal emitted when a variable is about to be deleted from the controller void variableAboutToBeDeleted(std::shared_ptr variable); @@ -141,4 +144,5 @@ private: spimpl::unique_impl_ptr impl; }; + #endif // SCIQLOP_VARIABLECONTROLLER_H diff --git a/core/src/Variable/VariableController.cpp b/core/src/Variable/VariableController.cpp index 916d0cb..4661869 100644 --- a/core/src/Variable/VariableController.cpp +++ b/core/src/Variable/VariableController.cpp @@ -135,7 +135,7 @@ struct VariableController::VariableControllerPrivate { void updateVariableRequest(QUuid varRequestId); void cancelVariableRequest(QUuid varRequestId); void executeVarRequest(std::shared_ptr var, VariableRequest &varRequest); - + bool hasPendingDownloads(); template void desynchronize(VariableIterator variableIt, const QUuid &syncGroupId); @@ -656,6 +656,11 @@ void VariableController::waitForFinish() QMutexLocker locker{&impl->m_WorkingMutex}; } +bool VariableController::hasPendingDownloads() +{ + return impl->hasPendingDownloads(); +} + AcquisitionZoomType VariableController::getZoomType(const SqpRange &range, const SqpRange &oldRange) { // t1.m_TStart <= t2.m_TStart && t2.m_TEnd <= t1.m_TEnd @@ -1079,6 +1084,11 @@ void VariableController::VariableControllerPrivate::executeVarRequest(std::share } } +bool VariableController::VariableControllerPrivate::hasPendingDownloads() +{ + return !m_VarGroupIdToVarIds.empty(); +} + template void VariableController::VariableControllerPrivate::desynchronize(VariableIterator variableIt, const QUuid &syncGroupId) diff --git a/plugins/amda/CMakeLists.txt b/plugins/amda/CMakeLists.txt index 99b3c33..0df99ec 100644 --- a/plugins/amda/CMakeLists.txt +++ b/plugins/amda/CMakeLists.txt @@ -48,15 +48,26 @@ target_link_libraries(pytestamdalib PUBLIC pybind11::module) target_link_libraries(pytestamdalib PUBLIC pybind11::embed) target_link_libraries(pytestamdalib PUBLIC amdaplugin) -declare_test(TestPytestamda TestPytestamda "tests/PyTestAmdaWrapperExe.cpp" "amdaplugin;pytestamdalib") -target_compile_definitions(TestPytestamda PRIVATE -DPYTESTAMDA_SCRIPT="${CMAKE_CURRENT_LIST_DIR}/tests/pyamdatests.py") +declare_test(TestAmdaFileParserEmbed TestAmdaFileParserEmbed "tests/PyTestAmdaWrapperExe.cpp" "amdaplugin;pytestamdalib") +target_compile_definitions(TestAmdaFileParserEmbed PRIVATE -DPYTESTAMDA_SCRIPT="${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaFileParser.py") + +declare_test(TestAmdaDownloadEmbed TestAmdaDownloadEmbed "tests/PyTestAmdaWrapperExe.cpp" "amdaplugin;pytestamdalib") +target_compile_definitions(TestAmdaDownloadEmbed PRIVATE -DPYTESTAMDA_SCRIPT="${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaDownload.py") + find_package(PythonInterp 3 REQUIRED) -add_test(NAME pyamdatests +add_test(NAME TestAmdaFileParser + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaFileParser.py + TestAmdaFileParser) +set_tests_properties(TestAmdaFileParser PROPERTIES ENVIRONMENT PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}) + + +add_test(NAME TestAmdaDownload COMMAND ${PYTHON_EXECUTABLE} - ${CMAKE_CURRENT_LIST_DIR}/tests/pyamdatests.py - pyamdatests) + ${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaDownload.py + TestAmdaDownload) +set_tests_properties(TestAmdaDownload PROPERTIES ENVIRONMENT PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}) -set_tests_properties(pyamdatests PROPERTIES ENVIRONMENT PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}) diff --git a/plugins/amda/tests/PyTestAmdaWrapper.cpp b/plugins/amda/tests/PyTestAmdaWrapper.cpp index 1b6284d..ecda73e 100644 --- a/plugins/amda/tests/PyTestAmdaWrapper.cpp +++ b/plugins/amda/tests/PyTestAmdaWrapper.cpp @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include @@ -35,6 +37,7 @@ #include #include #include +#include #include #include @@ -45,14 +48,15 @@ #include #include +using namespace std::chrono; namespace py = pybind11; std::ostream &operator <<(std::ostream& os, const Unit& u) { os << "=========================" << std::endl << "Unit:" << std::endl - << "Name:" << std::endl << u.m_Name.toStdString() << std::endl - << "Is_TimeUnit: " << u.m_TimeUnit << std::endl; + << " Name: " << u.m_Name.toStdString() << std::endl + << " Is_TimeUnit: " << u.m_TimeUnit << std::endl; return os; } @@ -60,10 +64,29 @@ std::ostream &operator <<(std::ostream& os, const IDataSeries& ds) { os << "=========================" << std::endl << "DataSerie:" << std::endl - << "Number of points:" << ds.nbPoints() << std::endl - << "X Axis Unit:" << std::endl << ds.xAxisUnit() << std::endl - << "Y Axis Unit:" << std::endl << ds.yAxisUnit()<< std::endl - << "Values Axis Unit:" << std::endl << ds.valuesUnit()<< std::endl; + << " Number of points:" << ds.nbPoints() << std::endl + << " X Axis Unit:" << std::endl << ds.xAxisUnit() << std::endl + << " Y Axis Unit:" << std::endl << ds.yAxisUnit()<< std::endl + << " Values Axis Unit:" << std::endl << ds.valuesUnit()<< std::endl; + return os; +} + +std::ostream &operator <<(std::ostream& os, const SqpRange& range) +{ + os << "=========================" << std::endl + << "SqpRange:" << std::endl + << " Start date: " << DateUtils::dateTime(range.m_TStart).toString().toStdString() << std::endl + << " Stop date: " << DateUtils::dateTime(range.m_TEnd).toString().toStdString() << std::endl; + return os; +} + +std::ostream &operator <<(std::ostream& os, const Variable& variable) +{ + os << "=========================" << std::endl + << "Variable:" << std::endl + << " Name: " << variable.name().toStdString() << std::endl + << " range: " << std::endl << variable.range() << std::endl + << " cache range: " << std::endl << variable.cacheRange() << std::endl; return os; } @@ -76,9 +99,21 @@ std::string __repr__(const T& obj) } + + PYBIND11_MODULE(pytestamda, m){ + int argc = 0; + char ** argv=nullptr; + SqpApplication::setOrganizationName("LPP"); + SqpApplication::setOrganizationDomain("lpp.fr"); + SqpApplication::setApplicationName("SciQLop"); + static SqpApplication app(argc, argv); + m.doc() = "hello"; + auto amda_provider = std::make_shared(); + m.def("amda_provider",[amda_provider](){return amda_provider;}, py::return_value_policy::copy); + py::enum_(m, "DataSeriesType") .value("SCALAR", DataSeriesType::SCALAR) .value("SPECTROGRAM", DataSeriesType::SPECTROGRAM) @@ -92,25 +127,50 @@ PYBIND11_MODULE(pytestamda, m){ .def(py::self != py::self) .def("__repr__",__repr__); + py::class_(m,"DataSeriesIteratorValue") + .def_property_readonly("x", &DataSeriesIteratorValue::x) + .def("value", py::overload_cast<>(&DataSeriesIteratorValue::value, py::const_)) + .def("value", py::overload_cast(&DataSeriesIteratorValue::value, py::const_)); + py::class_>(m, "IDataSeries") .def("nbPoints", &IDataSeries::nbPoints) .def_property_readonly("xAxisUnit", &IDataSeries::xAxisUnit) .def_property_readonly("yAxisUnit", &IDataSeries::yAxisUnit) .def_property_readonly("valuesUnit", &IDataSeries::valuesUnit) + .def("__getitem__", [](IDataSeries& serie, int key) { + return *(serie.begin()+key); + }, py::is_operator()) + .def("__len__", &IDataSeries::nbPoints) + .def("__iter__", [](IDataSeries& serie) { + return py::make_iterator(serie.begin(), serie.end()); + }, py::keep_alive<0, 1>()) .def("__repr__",__repr__); - - py::class_, IDataSeries>(m, "ScalarSeries") .def("nbPoints", &ScalarSeries::nbPoints); + py::class_, IDataSeries>(m, "VectorSeries") + .def("nbPoints", &VectorSeries::nbPoints); + py::class_(m, "QString") .def(py::init([](const std::string& value){return QString::fromStdString(value);})) .def("__repr__", &QString::toStdString); - py::class_(m, "VariableController"); + py::class_(m, "VariableController") + .def_static("createVariable",[](const QString &name, + std::shared_ptr provider){ + return sqpApp->variableController().createVariable(name, {{"dataType", "vector"}, {"xml:id", "c1_b"}}, provider); + }) + .def_static("hasPendingDownloads", + [](){return sqpApp->variableController().hasPendingDownloads();} + ); - py::class_(m, "AmdaProvider"); + py::class_(m,"TimeController") + .def_static("setTime", [](SqpRange range){sqpApp->timeController().onTimeToUpdate(range);}); + + py::class_>(m, "IDataProvider"); + + py::class_, IDataProvider>(m, "AmdaProvider"); py::class_(m, "AmdaResultParser") .def_static("readTxt", AmdaResultParser::readTxt) @@ -118,25 +178,50 @@ PYBIND11_MODULE(pytestamda, m){ return std::dynamic_pointer_cast(AmdaResultParser::readTxt(path, DataSeriesType::SCALAR)); }, py::return_value_policy::copy); - py::class_(m, "Variable") + py::class_>(m, "Variable") .def(py::init()) .def_property("name", &Variable::name, &Variable::setName) .def_property("range", &Variable::range, &Variable::setRange) - .def_property("cacheRange", &Variable::cacheRange, &Variable::setCacheRange); + .def_property("cacheRange", &Variable::cacheRange, &Variable::setCacheRange) + .def_property_readonly("nbPoints", &Variable::nbPoints) + .def_property_readonly("dataSeries", &Variable::dataSeries) + .def("__len__", [](Variable& variable) { + auto rng = variable.dataSeries()->xAxisRange(variable.range().m_TStart,variable.range().m_TEnd); + return std::distance(rng.first,rng.second); + }) + .def("__iter__", [](Variable& variable) { + auto rng = variable.dataSeries()->xAxisRange(variable.range().m_TStart,variable.range().m_TEnd); + return py::make_iterator(rng.first, rng.second); + }, py::keep_alive<0, 1>()) + .def("__getitem__", [](Variable& variable, int key) { + //insane and slow! + auto rng = variable.dataSeries()->xAxisRange(variable.range().m_TStart,variable.range().m_TEnd); + if(key<0) + return *(rng.second+key); + else + return *(rng.first+key); + }) + .def("__repr__",__repr__); py::implicitly_convertible(); - py::class_(m,"TimeController"); py::class_(m,"SqpRange") .def("fromDateTime", &SqpRange::fromDateTime, py::return_value_policy::move) .def(py::init([](double start, double stop){return SqpRange{start, stop};})) - .def("__repr__", [](const SqpRange& range){ - QString repr = QString("SqpRange:\n Start date: %1\n Stop date: %2") - .arg(DateUtils::dateTime(range.m_TStart).toString()) - .arg(DateUtils::dateTime(range.m_TEnd).toString()); - return repr.toStdString(); - }); + .def(py::init([](system_clock::time_point start, system_clock::time_point stop) + { + double start_ = 0.001 * duration_cast(start.time_since_epoch()).count(); + double stop_ = 0.001 * duration_cast(stop.time_since_epoch()).count(); + return SqpRange{start_, stop_}; + })) + .def_property_readonly("start", [](const SqpRange& range){ + return system_clock::from_time_t(range.m_TStart); + }) + .def_property_readonly("stop", [](const SqpRange& range){ + return system_clock::from_time_t(range.m_TEnd); + }) + .def("__repr__", __repr__); py::class_(m,"QUuid"); @@ -149,14 +234,9 @@ PYBIND11_MODULE(pytestamda, m){ } -int pytestamda_test(int argc, char** argv, const char* testScriptPath ) +int pytestamda_test(const char* testScriptPath ) { - SqpApplication::setOrganizationName("LPP"); - SqpApplication::setOrganizationDomain("lpp.fr"); - SqpApplication::setApplicationName("SciQLop"); - SqpApplication app(argc, argv); py::scoped_interpreter guard{}; - py::globals()["__file__"] = py::str(testScriptPath); py::eval_file(testScriptPath); return 0; diff --git a/plugins/amda/tests/PyTestAmdaWrapperExe.cpp b/plugins/amda/tests/PyTestAmdaWrapperExe.cpp index 06c7f8c..edd6832 100644 --- a/plugins/amda/tests/PyTestAmdaWrapperExe.cpp +++ b/plugins/amda/tests/PyTestAmdaWrapperExe.cpp @@ -20,11 +20,11 @@ -- Mail : alexis.jeandet@member.fsf.org ----------------------------------------------------------------------------*/ #include -extern int pytestamda_test(int argc, char** argv, const char* testScriptPath ); +extern int pytestamda_test(const char* testScriptPath ); int main(int argc, char** argv) { - pytestamda_test(argc, argv, PYTESTAMDA_SCRIPT); + pytestamda_test(PYTESTAMDA_SCRIPT); return 0; } diff --git a/plugins/amda/tests/TestAmdaDownload.py b/plugins/amda/tests/TestAmdaDownload.py new file mode 100644 index 0000000..c944f2a --- /dev/null +++ b/plugins/amda/tests/TestAmdaDownload.py @@ -0,0 +1,54 @@ +import sys +import os +if not hasattr(sys, 'argv'): + sys.argv = [''] +current_script_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(current_script_path) +import amda +import pytestamda + +import numpy as np +import datetime +import time +import unittest +import ddt + +def wait_for_downloads(): + while pytestamda.VariableController.hasPendingDownloads(): + time.sleep(0.1) + +def extract_vector(variable): + return zip(*[(pt.x, pt.value(0), pt.value(1), pt.value(2)) for pt in variable]) + +def compare_with_ref(var, ref): + t_ref, x_ref, y_ref, z_ref = ref + t,x,y,z = extract_vector(var) + return all([ + all([t_ref[i].astype(float)/1000000 == t[i] for i in range(len(t))]), + all([x_ref[i] == x[i] for i in range(len(x))]), + all([y_ref[i] == y[i] for i in range(len(y))]), + all([z_ref[i] == z[i] for i in range(len(z))]) + ]) + +@ddt.ddt +class FunctionalTests(unittest.TestCase): + def setUp(self): + pass + + @ddt.data( + (datetime.datetime(2012,10,20,8,10,00),datetime.datetime(2012,10,20,12,0,0)), + (datetime.datetime(2025,1,1,15,0,0),datetime.datetime(2025,1,1,16,0,0)), + (datetime.datetime(2000,1,1,0,0,0),datetime.datetime(2000,1,1,12,0,0)) + ) + def test_simple_download(self, case): + tstart = case[0] + tstop = case[1] + pytestamda.TimeController.setTime(pytestamda.SqpRange(tstart, tstop)) + variable = pytestamda.VariableController.createVariable("bx_gse",pytestamda.amda_provider()) + wait_for_downloads() + t_ref, x_ref, y_ref, z_ref = amda.generate_data(np.datetime64(tstart), np.datetime64(tstop), 4) + self.assertTrue( compare_with_ref(variable,(t_ref, x_ref, y_ref, z_ref) ) ) + + +if __name__ == '__main__': + unittest.main(exit=False) diff --git a/plugins/amda/tests/TestAmdaFileParser.py b/plugins/amda/tests/TestAmdaFileParser.py new file mode 100644 index 0000000..00de428 --- /dev/null +++ b/plugins/amda/tests/TestAmdaFileParser.py @@ -0,0 +1,39 @@ +import sys +import os +if not hasattr(sys, 'argv'): + sys.argv = [''] +current_script_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(current_script_path) +import pytestamda +import amda + +import numpy as np +import datetime +import time +import unittest +import ddt + +path = current_script_path+'/../tests-resources/TestAmdaResultParser/ValidScalar1.txt' + +@ddt.ddt +class FunctionalTests(unittest.TestCase): + def setUp(self): + pass + + @ddt.data( + current_script_path+'/../tests-resources/TestAmdaResultParser/ValidScalar1.txt' + ) + def test_correct_scalars(self, case): + scalar_sciqlop = pytestamda.AmdaResultParser.readScalarTxt(case) + scalar_ref = amda.load_scalar(case) + self.assertTrue(len(scalar_ref) == len(scalar_sciqlop)) + self.assertTrue(all( + [scalar_ref[i][1] == scalar_sciqlop[i].value() + for i in range(len(scalar_sciqlop))])) + self.assertTrue(all( + [scalar_ref[i][0].timestamp() == scalar_sciqlop[i].x + for i in range(len(scalar_sciqlop))])) + + +if __name__ == '__main__': + unittest.main(exit=False) diff --git a/plugins/amda/tests/amda.py b/plugins/amda/tests/amda.py new file mode 100644 index 0000000..a138944 --- /dev/null +++ b/plugins/amda/tests/amda.py @@ -0,0 +1,29 @@ +import sys +import os +import numpy as np +import datetime +import time + +os.environ['TZ'] = 'UTC' +epoch_2000 = np.datetime64('2000-01-01T00:00:00',tzinfo=datetime.timezone.utc) + +def load_scalar(fname): + with open(fname, "r") as f: + return [[ + datetime.datetime(*(time.strptime(line.split()[0], '%Y-%m-%dT%H:%M:%S.%f')[0:6]), + tzinfo=datetime.timezone.utc), + float(line.split()[1])] + for line in f if "#" not in line] + +""" +Copied from myAMDA should be factored in somehow +""" +def generate_data(tstart, tstop, dt): + delta = np.timedelta64(dt, 's') + vector_size = int(np.round((tstop-tstart)/delta)) + 1 + t = [tstart+i*delta for i in range(vector_size)] + x0 = tstart-epoch_2000 + x = [(x0 + i * delta).astype('float')/1000000 for i in range(vector_size)] + y = [(x0 + (i+1) * delta).astype('float')/1000000 for i in range(vector_size)] + z = [(x0 + (i+2) * delta).astype('float')/1000000 for i in range(vector_size)] + return t,x,y,z diff --git a/plugins/amda/tests/pyamdatests.py b/plugins/amda/tests/pyamdatests.py deleted file mode 100644 index fbed050..0000000 --- a/plugins/amda/tests/pyamdatests.py +++ /dev/null @@ -1,5 +0,0 @@ -import pytestamda -import os -current_script_path = os.path.dirname(os.path.realpath(__file__)) -path = current_script_path+'/../tests-resources/TestAmdaResultParser/ValidScalar1.txt' -c = pytestamda.AmdaResultParser.readScalarTxt(path)