HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop Commit - r1340:02cea38fdcb4

Added basic tests around Amda plugin Python wrapper...

jeandet -

r1340:02cea38fdcb4

parent child

Context file:

r1340:02cea38fdcb4

plugins/amda/tests/TestAmdaDownload.py created 644 +54 0

@@ -0,0 +1,54
	1	import sys
	2	import os
	3	if not hasattr(sys, 'argv'):
	4	sys.argv = ['']
	5	current_script_path = os.path.dirname(os.path.realpath(__file__))
	6	sys.path.append(current_script_path)
	7	import amda
	8	import pytestamda
	9
	10	import numpy as np
	11	import datetime
	12	import time
	13	import unittest
	14	import ddt
	15
	16	def wait_for_downloads():
	17	while pytestamda.VariableController.hasPendingDownloads():
	18	time.sleep(0.1)
	19
	20	def extract_vector(variable):
	21	return zip(*[(pt.x, pt.value(0), pt.value(1), pt.value(2)) for pt in variable])
	22
	23	def compare_with_ref(var, ref):
	24	t_ref, x_ref, y_ref, z_ref = ref
	25	t,x,y,z = extract_vector(var)
	26	return all([
	27	all([t_ref[i].astype(float)/1000000 == t[i] for i in range(len(t))]),
	28	all([x_ref[i] == x[i] for i in range(len(x))]),
	29	all([y_ref[i] == y[i] for i in range(len(y))]),
	30	all([z_ref[i] == z[i] for i in range(len(z))])
	31	])
	32
	33	@ddt.ddt
	34	class FunctionalTests(unittest.TestCase):
	35	def setUp(self):
	36	pass
	37
	38	@ddt.data(
	39	(datetime.datetime(2012,10,20,8,10,00),datetime.datetime(2012,10,20,12,0,0)),
	40	(datetime.datetime(2025,1,1,15,0,0),datetime.datetime(2025,1,1,16,0,0)),
	41	(datetime.datetime(2000,1,1,0,0,0),datetime.datetime(2000,1,1,12,0,0))
	42	)
	43	def test_simple_download(self, case):
	44	tstart = case[0]
	45	tstop = case[1]
	46	pytestamda.TimeController.setTime(pytestamda.SqpRange(tstart, tstop))
	47	variable = pytestamda.VariableController.createVariable("bx_gse",pytestamda.amda_provider())
	48	wait_for_downloads()
	49	t_ref, x_ref, y_ref, z_ref = amda.generate_data(np.datetime64(tstart), np.datetime64(tstop), 4)
	50	self.assertTrue( compare_with_ref(variable,(t_ref, x_ref, y_ref, z_ref) ) )
	51
	52
	53	if __name__ == '__main__':
	54	unittest.main(exit=False)

plugins/amda/tests/TestAmdaFileParser.py created 644 +39 0

@@ -0,0 +1,39
	1	import sys
	2	import os
	3	if not hasattr(sys, 'argv'):
	4	sys.argv = ['']
	5	current_script_path = os.path.dirname(os.path.realpath(__file__))
	6	sys.path.append(current_script_path)
	7	import pytestamda
	8	import amda
	9
	10	import numpy as np
	11	import datetime
	12	import time
	13	import unittest
	14	import ddt
	15
	16	path = current_script_path+'/../tests-resources/TestAmdaResultParser/ValidScalar1.txt'
	17
	18	@ddt.ddt
	19	class FunctionalTests(unittest.TestCase):
	20	def setUp(self):
	21	pass
	22
	23	@ddt.data(
	24	current_script_path+'/../tests-resources/TestAmdaResultParser/ValidScalar1.txt'
	25	)
	26	def test_correct_scalars(self, case):
	27	scalar_sciqlop = pytestamda.AmdaResultParser.readScalarTxt(case)
	28	scalar_ref = amda.load_scalar(case)
	29	self.assertTrue(len(scalar_ref) == len(scalar_sciqlop))
	30	self.assertTrue(all(
	31	[scalar_ref[i][1] == scalar_sciqlop[i].value()
	32	for i in range(len(scalar_sciqlop))]))
	33	self.assertTrue(all(
	34	[scalar_ref[i][0].timestamp() == scalar_sciqlop[i].x
	35	for i in range(len(scalar_sciqlop))]))
	36
	37
	38	if __name__ == '__main__':
	39	unittest.main(exit=False)

plugins/amda/tests/amda.py created 644 +29 0

@@ -0,0 +1,29
	1	import sys
	2	import os
	3	import numpy as np
	4	import datetime
	5	import time
	6
	7	os.environ['TZ'] = 'UTC'
	8	epoch_2000 = np.datetime64('2000-01-01T00:00:00',tzinfo=datetime.timezone.utc)
	9
	10	def load_scalar(fname):
	11	with open(fname, "r") as f:
	12	return [[
	13	datetime.datetime(*(time.strptime(line.split()[0], '%Y-%m-%dT%H:%M:%S.%f')[0:6]),
	14	tzinfo=datetime.timezone.utc),
	15	float(line.split()[1])]
	16	for line in f if "#" not in line]
	17
	18	"""
	19	Copied from myAMDA should be factored in somehow
	20	"""
	21	def generate_data(tstart, tstop, dt):
	22	delta = np.timedelta64(dt, 's')
	23	vector_size = int(np.round((tstop-tstart)/delta)) + 1
	24	t = [tstart+i*delta for i in range(vector_size)]
	25	x0 = tstart-epoch_2000
	26	x = [(x0 + i * delta).astype('float')/1000000 for i in range(vector_size)]
	27	y = [(x0 + (i+1) * delta).astype('float')/1000000 for i in range(vector_size)]
	28	z = [(x0 + (i+2) * delta).astype('float')/1000000 for i in range(vector_size)]
	29	return t,x,y,z

core/include/Variable/VariableController.h +4 0

                 QList<std::shared_ptr<Variable> > variablesForMimeData(const QByteArray &mimeData) const;
                 static AcquisitionZoomType getZoomType(const SqpRange &range, const SqpRange &oldRange);
+                /// Returns True if there are pending downloads
+                bool hasPendingDownloads();
             signals:
                 /// Signal emitted when a variable is about to be deleted from the controller
                 void variableAboutToBeDeleted(std::shared_ptr<Variable> variable);
                 spimpl::unique_impl_ptr<VariableControllerPrivate> impl;
             };
             #endif // SCIQLOP_VARIABLECONTROLLER_H

core/src/Variable/VariableController.cpp +11 -1

                 void updateVariableRequest(QUuid varRequestId);
                 void cancelVariableRequest(QUuid varRequestId);
                 void executeVarRequest(std::shared_ptr<Variable> var, VariableRequest &varRequest);
+                bool hasPendingDownloads();
                 template <typename VariableIterator>
                 void desynchronize(VariableIterator variableIt, const QUuid &syncGroupId);
                 QMutexLocker locker{&impl->m_WorkingMutex};
             }
+            bool VariableController::hasPendingDownloads()
+            {
+                return impl->hasPendingDownloads();
+            }
             AcquisitionZoomType VariableController::getZoomType(const SqpRange &range, const SqpRange &oldRange)
             {
                 // t1.m_TStart <= t2.m_TStart && t2.m_TEnd <= t1.m_TEnd
                 }
             }
+            bool VariableController::VariableControllerPrivate::hasPendingDownloads()
+            {
+                return !m_VarGroupIdToVarIds.empty();
+            }
             template <typename VariableIterator>
             void VariableController::VariableControllerPrivate::desynchronize(VariableIterator variableIt,
                                                                               const QUuid &syncGroupId)

plugins/amda/CMakeLists.txt +17 -6

@@ -48,15 +48,26 target_link_libraries(pytestamdalib PUBLIC pybind11::module)
48	target_link_libraries(pytestamdalib PUBLIC pybind11::embed)	48	target_link_libraries(pytestamdalib PUBLIC pybind11::embed)
49	target_link_libraries(pytestamdalib PUBLIC amdaplugin)	49	target_link_libraries(pytestamdalib PUBLIC amdaplugin)
50		50
51	declare_test(Test~~Pytestamda TestPytestamda~~ "tests/PyTestAmdaWrapperExe.cpp" "amdaplugin;pytestamdalib")	51	declare_test(TestAmdaFileParserEmbed TestAmdaFileParserEmbed "tests/PyTestAmdaWrapperExe.cpp" "amdaplugin;pytestamdalib")
52	target_compile_definitions(Test~~Pytestamda~~ PRIVATE -DPYTESTAMDA_SCRIPT="${CMAKE_CURRENT_LIST_DIR}/tests/~~pyamdatests~~.py")	52	target_compile_definitions(TestAmdaFileParserEmbed PRIVATE -DPYTESTAMDA_SCRIPT="${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaFileParser.py")
		53
		54	declare_test(TestAmdaDownloadEmbed TestAmdaDownloadEmbed "tests/PyTestAmdaWrapperExe.cpp" "amdaplugin;pytestamdalib")
		55	target_compile_definitions(TestAmdaDownloadEmbed PRIVATE -DPYTESTAMDA_SCRIPT="${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaDownload.py")
		56
53		57
54	find_package(PythonInterp 3 REQUIRED)	58	find_package(PythonInterp 3 REQUIRED)
55		59
56	add_test(NAME ~~pyamdatests~~	60	add_test(NAME TestAmdaFileParser
		61	COMMAND ${PYTHON_EXECUTABLE}
		62	${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaFileParser.py
		63	TestAmdaFileParser)
		64	set_tests_properties(TestAmdaFileParser PROPERTIES ENVIRONMENT PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR})
		65
		66
		67	add_test(NAME TestAmdaDownload
57	COMMAND ${PYTHON_EXECUTABLE}	68	COMMAND ${PYTHON_EXECUTABLE}
58	${CMAKE_CURRENT_LIST_DIR}/tests/~~pyamdatests~~.py	69	${CMAKE_CURRENT_LIST_DIR}/tests/TestAmdaDownload.py
59	pyamdatests)	70	TestAmdaDownload)
		71	set_tests_properties(TestAmdaDownload PROPERTIES ENVIRONMENT PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR})
60		72
61	set_tests_properties(pyamdatests PROPERTIES ENVIRONMENT PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR})
62		73

plugins/amda/tests/PyTestAmdaWrapper.cpp +100 -20

             #include <pybind11/pybind11.h>
             #include <pybind11/operators.h>
             #include <pybind11/embed.h>
+            #include <pybind11/numpy.h>
+            #include <pybind11/chrono.h>
             #include <SqpApplication.h>
             #include <Variable/VariableController.h>
             #include <Common/DateUtils.h>
             #include <Variable/Variable.h>
             #include <Data/ScalarSeries.h>
+            #include <Data/VectorSeries.h>
             #include <AmdaProvider.h>
             #include <AmdaResultParser.h>
             #include <QString>
             #include <QFile>
+            using namespace std::chrono;
             namespace py = pybind11;
             std::ostream &operator <<(std::ostream& os, const Unit& u)
             {
                 os << "=========================" << std::endl
                    << "Unit:" << std::endl
-                   << "Name:" << std::endl << u.m_Name.toStdString() << std::endl
+                   << " Name: " << u.m_Name.toStdString() << std::endl
                    << " Is_TimeUnit: " << u.m_TimeUnit << std::endl;
                 return os;
             }
                 return os;
             }
+            std::ostream &operator <<(std::ostream& os, const SqpRange& range)
+            {
+                os << "=========================" << std::endl
+                   << "SqpRange:" << std::endl
+                   << " Start date: " << DateUtils::dateTime(range.m_TStart).toString().toStdString() << std::endl
+                   << " Stop date: "  << DateUtils::dateTime(range.m_TEnd).toString().toStdString() << std::endl;
+                return os;
+            }
+            std::ostream &operator <<(std::ostream& os, const Variable& variable)
+            {
+                os << "=========================" << std::endl
+                   << "Variable:" << std::endl
+                   << " Name: " << variable.name().toStdString() << std::endl
+                   << " range: " << std::endl << variable.range() << std::endl
+                   << " cache range: " << std::endl << variable.cacheRange() << std::endl;
+                return os;
+            }
             template <typename T>
             std::string __repr__(const T& obj)
             {
             }
             PYBIND11_MODULE(pytestamda, m){
+                int argc = 0;
+                char ** argv=nullptr;
+                SqpApplication::setOrganizationName("LPP");
+                SqpApplication::setOrganizationDomain("lpp.fr");
+                SqpApplication::setApplicationName("SciQLop");
+                static SqpApplication app(argc, argv);
                 m.doc() = "hello";
+                auto amda_provider = std::make_shared<AmdaProvider>();
+                m.def("amda_provider",[amda_provider](){return amda_provider;}, py::return_value_policy::copy);
                 py::enum_<DataSeriesType>(m, "DataSeriesType")
                         .value("SCALAR", DataSeriesType::SCALAR)
                         .value("SPECTROGRAM", DataSeriesType::SPECTROGRAM)
                         .def(py::self != py::self)
                         .def("__repr__",__repr__<Unit>);
+                py::class_<DataSeriesIteratorValue>(m,"DataSeriesIteratorValue")
+                        .def_property_readonly("x", &DataSeriesIteratorValue::x)
+                        .def("value", py::overload_cast<>(&DataSeriesIteratorValue::value, py::const_))
+                        .def("value", py::overload_cast<int>(&DataSeriesIteratorValue::value, py::const_));
                 py::class_<IDataSeries, std::shared_ptr<IDataSeries>>(m, "IDataSeries")
                         .def("nbPoints", &IDataSeries::nbPoints)
                         .def_property_readonly("xAxisUnit", &IDataSeries::xAxisUnit)
                         .def_property_readonly("yAxisUnit", &IDataSeries::yAxisUnit)
                         .def_property_readonly("valuesUnit", &IDataSeries::valuesUnit)
+                        .def("__getitem__", [](IDataSeries& serie, int key) {
+                    return *(serie.begin()+key);
+                }, py::is_operator())
+                        .def("__len__", &IDataSeries::nbPoints)
+                        .def("__iter__", [](IDataSeries& serie) {
+                    return py::make_iterator(serie.begin(), serie.end());
+                }, py::keep_alive<0, 1>())
                         .def("__repr__",__repr__<IDataSeries>);
                 py::class_<ScalarSeries, std::shared_ptr<ScalarSeries>, IDataSeries>(m, "ScalarSeries")
                         .def("nbPoints", &ScalarSeries::nbPoints);
+                py::class_<VectorSeries, std::shared_ptr<VectorSeries>, IDataSeries>(m, "VectorSeries")
+                        .def("nbPoints", &VectorSeries::nbPoints);
                 py::class_<QString>(m, "QString")
                         .def(py::init([](const std::string& value){return QString::fromStdString(value);}))
                         .def("__repr__", &QString::toStdString);
-                py::class_<VariableController>(m, "VariableController");
+                py::class_<VariableController>(m, "VariableController")
+                        .def_static("createVariable",[](const QString &name,
+                                    std::shared_ptr<IDataProvider> provider){
+                    return sqpApp->variableController().createVariable(name, {{"dataType", "vector"}, {"xml:id", "c1_b"}}, provider);
+                })
+                        .def_static("hasPendingDownloads",
+                                    [](){return sqpApp->variableController().hasPendingDownloads();}
+                                    );
-                py::class_<AmdaProvider>(m, "AmdaProvider");
+                py::class_<TimeController>(m,"TimeController")
+                        .def_static("setTime", [](SqpRange range){sqpApp->timeController().onTimeToUpdate(range);});
+                py::class_<IDataProvider, std::shared_ptr<IDataProvider>>(m, "IDataProvider");
+                py::class_<AmdaProvider, std::shared_ptr<AmdaProvider>, IDataProvider>(m, "AmdaProvider");
                 py::class_<AmdaResultParser>(m, "AmdaResultParser")
                         .def_static("readTxt", AmdaResultParser::readTxt)
                     return std::dynamic_pointer_cast<ScalarSeries>(AmdaResultParser::readTxt(path, DataSeriesType::SCALAR));
                 }, py::return_value_policy::copy);
-                py::class_<Variable>(m, "Variable")
+                py::class_<Variable,std::shared_ptr<Variable>>(m, "Variable")
                         .def(py::init<const QString&>())
                         .def_property("name", &Variable::name, &Variable::setName)
                         .def_property("range", &Variable::range, &Variable::setRange)
-                        .def_property("cacheRange", &Variable::cacheRange, &Variable::setCacheRange);
+                        .def_property("cacheRange", &Variable::cacheRange, &Variable::setCacheRange)
+                        .def_property_readonly("nbPoints", &Variable::nbPoints)
+                        .def_property_readonly("dataSeries", &Variable::dataSeries)
+                        .def("__len__", [](Variable& variable) {
+                    auto rng = variable.dataSeries()->xAxisRange(variable.range().m_TStart,variable.range().m_TEnd);
+                    return std::distance(rng.first,rng.second);
+                })
+                        .def("__iter__", [](Variable& variable) {
+                    auto rng = variable.dataSeries()->xAxisRange(variable.range().m_TStart,variable.range().m_TEnd);
+                    return py::make_iterator(rng.first, rng.second);
+                }, py::keep_alive<0, 1>())
+                        .def("__getitem__", [](Variable& variable, int key) {
+                    //insane and slow!
+                    auto rng = variable.dataSeries()->xAxisRange(variable.range().m_TStart,variable.range().m_TEnd);
+                    if(key<0)
+                        return *(rng.second+key);
+                    else
+                        return *(rng.first+key);
+                })
+                        .def("__repr__",__repr__<Variable>);
                 py::implicitly_convertible<std::string, QString>();
-                py::class_<TimeController>(m,"TimeController");
                 py::class_<SqpRange>(m,"SqpRange")
                         .def("fromDateTime", &SqpRange::fromDateTime, py::return_value_policy::move)
                         .def(py::init([](double start, double stop){return SqpRange{start, stop};}))
-                        .def("__repr__", [](const SqpRange& range){
+                        .def(py::init([](system_clock::time_point start, system_clock::time_point stop)
-                    QString repr = QString("SqpRange:\n Start date: %1\n Stop date: %2")
-                            .arg(DateUtils::dateTime(range.m_TStart).toString())
+                                 double start_ = 0.001 * duration_cast<milliseconds>(start.time_since_epoch()).count();
-                            .arg(DateUtils::dateTime(range.m_TEnd).toString());
+                                 double stop_ = 0.001 * duration_cast<milliseconds>(stop.time_since_epoch()).count();
-                    return repr.toStdString();
+                                 return SqpRange{start_, stop_};
-                });
+                             }))
+                        .def_property_readonly("start", [](const SqpRange& range){
+                    return  system_clock::from_time_t(range.m_TStart);
+                })
+                        .def_property_readonly("stop", [](const SqpRange& range){
+                    return  system_clock::from_time_t(range.m_TEnd);
+                })
+                        .def("__repr__", __repr__<SqpRange>);
                 py::class_<QUuid>(m,"QUuid");
             }
-            int pytestamda_test(int argc, char** argv, const char* testScriptPath )
+            int pytestamda_test(const char* testScriptPath )
             {
-                SqpApplication::setOrganizationName("LPP");
-                SqpApplication::setOrganizationDomain("lpp.fr");
-                SqpApplication::setApplicationName("SciQLop");
-                SqpApplication app(argc, argv);
                 py::scoped_interpreter guard{};
                 py::globals()["__file__"] = py::str(testScriptPath);
                 py::eval_file(testScriptPath);
                 return 0;

plugins/amda/tests/PyTestAmdaWrapperExe.cpp +2 -2

             --                     Mail : alexis.jeandet@member.fsf.org
             ----------------------------------------------------------------------------*/
             #include <QString>
-            extern int pytestamda_test(int argc, char** argv, const char* testScriptPath );
+            extern int pytestamda_test(const char* testScriptPath );
             int main(int argc, char** argv)
             {
-                pytestamda_test(argc, argv, PYTESTAMDA_SCRIPT);
+                pytestamda_test(PYTESTAMDA_SCRIPT);
                 return 0;
             }

plugins/amda/tests/pyamdatests.py removed 0 -5

NO CONTENT: file was removed

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages