HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop Commit - r1430:ecc8b7b09c3d

Added POC AMDA python impl and CDAWEB bits...

jeandet -

r1430:ecc8b7b09c3d

parent child

Context file:

r1430:ecc8b7b09c3d

plugins/python_providers/resources/cdaweb.py created 644 +44 0

@@ -0,0 +1,44
	1	import sys
	2	sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
	3	import os
	4	import datetime
	5	import PythonProviders
	6	import pysciqlopcore
	7	import numpy as np
	8	import pandas as pds
	9	import requests
	10	from spwc.cdaweb import cdaweb
	11
	12	cd = cdaweb()
	13
	14	def get_sample(name,start,stop):
	15	try:
	16	tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')
	17	tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')
	18	req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"
	19	resp = requests.get(req_url,headers={"Accept":"application/json"})
	20	csv_url = resp.json()['FileDescription'][0]['Name']
	21	df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)
	22	t = np.array([d.timestamp()-7200 for d in df.index])
	23	values = df.values
	24	return pysciqlopcore.VectorTimeSerie(t,values)
	25	except Exception as e:
	26	print("fuck ",str(e))
	27	return pysciqlopcore.VectorTimeSerie(1)
	28
	29	def get_sample(name,start,stop):
	30	try:
	31	tstart=datetime.datetime.fromtimestamp(start)
	32	tend=datetime.datetime.fromtimestamp(stop)
	33	df = cd.get_variable(dataset="MMS2_SCM_SRVY_L2_SCSRVY",variable="mms2_scm_acb_gse_scsrvy_srvy_l2",tstart=tstart,tend=tend)
	34	t = np.array([d.timestamp()-7200 for d in df.index])
	35	values = df.values
	36	return pysciqlopcore.VectorTimeSerie(t,values)
	37	except Exception as e:
	38	print("fuck ",str(e))
	39	return pysciqlopcore.VectorTimeSerie(1)
	40
	41
	42	PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[],[("type","vector")])],get_sample)
	43
	44

plugins/python_providers/include/python_interpreter.h +7 -4

             #include <Data/DateTimeRange.h>
             #include <TimeSeries.h>
             #include <functional>
             #include <iostream>
             #include <memory>
             class PythonInterpreter
             {
             public:
-                using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>;
+                using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(
+                    std::vector<std::tuple<std::string, std::string>>&, double, double)>;
+                using product_t = std::tuple<std::string, std::vector<std::string>,
+                    std::vector<std::pair<std::string, std::string>>>;
                 PythonInterpreter();
-                void add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
+                void add_register_callback(
-                        provider_funct_t)>
+                    std::function<void(const std::vector<product_t>&, provider_funct_t)> callback);
-                        callback);
                 ~PythonInterpreter();
                 void eval(const std::string& file);
                 void release();
             private:
             };

plugins/python_providers/include/python_providers.h +2 -3

             #ifndef PYTHON_PROVIDERS_H
             #define PYTHON_PROVIDERS_H
             #include <Plugin/IPlugin.h>
             #include <QUuid>
             #include <memory>
             #include <python_interpreter.h>
             #ifndef SCIQLOP_PLUGIN_JSON_FILE_PATH
             #define SCIQLOP_PLUGIN_JSON_FILE_PATH "python_providers.json"
             #endif
             class DataSourceItem;
             class PythonProviders : public QObject, public IPlugin
             {
                 Q_OBJECT
                 Q_INTERFACES(IPlugin)
                 Q_PLUGIN_METADATA(IID "sciqlop.plugin.IPlugin" FILE SCIQLOP_PLUGIN_JSON_FILE_PATH)
             public:
                 /// @sa IPlugin::initialize()
                 void initialize() override;
                 ~PythonProviders();
             private:
-                void register_product(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>& product_list,
+                void register_product(const std::vector<PythonInterpreter::product_t>& product_list,
-                    PythonInterpreter::provider_funct_t
+                    PythonInterpreter::provider_funct_t f);
-                        f);
                 PythonInterpreter _interpreter;
             };
             #endif // PYTHON_PROVIDERS_H

plugins/python_providers/resources/amda.py +43 -11

@@ -1,28 +1,60
1	import sys	1	import sys
2	sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")	2	sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
3	import os	3	import os
4	import datetime	4	import datetime
5	import PythonProviders	5	import PythonProviders
6	import pysciqlopcore	6	import pysciqlopcore
7	import numpy as np	7	import numpy as np
8	import pandas as pds	8	import pandas as pds
9	import requests	9	import requests
		10	from spwc.amda import AMDA
10		11
11	def get_sample(name,start,stop):	12	amda = AMDA()
		13
		14	def get_sample(metadata,start,stop):
		15	ts_type = pysciqlopcore.ScalarTimeSerie
12	try:	16	try:
13	tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')	17	param_id = None
14	tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')	18	for key,value in metadata:
15	req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"	19	if key == 'xml:id':
16	resp = requests.get(req_url,headers={"Accept":"application/json"})	20	param_id = value
17	csv_url = resp.json()['FileDescription'][0]['Name']	21	elif key == 'type':
18	df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)	22	if value == 'vector':
		23	ts_type = pysciqlopcore.VectorTimeSerie
		24	tstart=datetime.datetime.fromtimestamp(start)
		25	tend=datetime.datetime.fromtimestamp(stop)
		26	df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id)
19	t = np.array([d.timestamp()-7200 for d in df.index])	27	t = np.array([d.timestamp()-7200 for d in df.index])
20	values = df.values	28	values = df.values
21	return pysciqlopcore.VectorTimeSerie(t,values)	29	print(len(t))
		30	print(len(values))
		31	return ts_type(t,values)
		32	return ts_type(1)
22	except Exception as e:	33	except Exception as e:
23	print("~~fuck~~ ",str(e))	34	print("Error in amda.py ",str(e))
24	return pysciqlopcore.VectorTimeSerie(1)	35	return ts_type(1)
		36
		37
		38	if len(amda.component) is 0:
		39	amda.update_inventory()
		40	parameters = amda.parameter.copy()
		41	for name,component in amda.component.items():
		42	if 'components' in parameters[component['parameter']]:
		43	parameters[component['parameter']]['components'].append(component)
		44	else:
		45	parameters[component['parameter']]['components']=[component]
		46
		47	products = []
		48	for key,parameter in parameters.items():
		49	path = f"/AMDA/{parameter['mission']}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
		50	components = [component['name'] for component in parameter.get('components',[])]
		51	metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ]
		52	if parameter.get('size',0) is '3':
		53	metadata.append(("type","vector"))
		54	else:
		55	metadata.append(("type","scalar"))
		56	products.append( (path, components, metadata))
25		57
26	PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample)	58	PythonProviders.register_product(products, get_sample)
27		59
28		60

plugins/python_providers/src/python_interpreter.cpp +2 -3

             #include "python_interpreter.h"
             #include <Data/DateTimeRange.h>
             #include <TimeSeries.h>
             #include <functional>
             #include <iostream>
             #include <pybind11/embed.h>
             #include <pybind11/functional.h>
             #include <pybind11/stl.h>
             namespace py = pybind11;
             PYBIND11_EMBEDDED_MODULE(PythonProviders, m) {}
             static pybind11::gil_scoped_release* _rel = nullptr;
             PythonInterpreter::PythonInterpreter()
             {
                 py::initialize_interpreter(false);
             }
-            void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
+            void PythonInterpreter::add_register_callback(
-                    provider_funct_t)>
+                std::function<void(const std::vector<product_t>&, provider_funct_t)> callback)
-                    callback)
             {
                 py::module PythonProviders = py::module::import("PythonProviders");
                 PythonProviders.attr("register_product") = callback;
             }
             PythonInterpreter::~PythonInterpreter()
             {
                 if (_rel)
                     delete _rel;
                 py::finalize_interpreter();
             }
             void PythonInterpreter::eval(const std::string& file)
             {
                 try
                 {
                     py::eval_file(file);
                 }
                 catch (py::error_already_set const& pythonErr)
                 {
                     std::cout << pythonErr.what();
                 }
             }
             void PythonInterpreter::release()
             {
                 _rel = new py::gil_scoped_release();
             }

plugins/python_providers/src/python_providers.cpp +14 -14

             #include "python_providers.h"
             #include <Data/DataProviderParameters.h>
             #include <Data/DateTimeRange.h>
             #include <Data/IDataProvider.h>
             #include <Data/ScalarTimeSerie.h>
             #include <Data/SpectrogramTimeSerie.h>
             #include <Data/TimeSeriesUtils.h>
             #include <Data/VectorTimeSerie.h>
             #include <DataSource/DataSourceController.h>
             #include <DataSource/DataSourceItem.h>
             #include <DataSource/DataSourceItemAction.h>
             #include <QDir>
             #include <QStandardPaths>
             #include <QStringList>
             #include <SqpApplication.h>
             #include <TimeSeries.h>
             #include <functional>
             #include <iostream>
             const auto DATA_SOURCE_NAME = QStringLiteral("PythonProviders");
-            struct noop_deleter
-                void operator()(TimeSeries::ITimeSerie*) {}
-            };
             class PythonProvider : public IDataProvider
             {
             public:
                 PythonProvider(PythonInterpreter::provider_funct_t f) : _pythonFunction { f } {}
                 PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {}
                 std::shared_ptr<IDataProvider> clone() const override
                 {
                     return std::make_shared<PythonProvider>(*this);
                 }
                 virtual TimeSeries::ITimeSerie* getData(const DataProviderParameters& parameters) override
                 {
                     auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
                     auto range = parameters.m_Range;
-                    auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
+                    std::vector<std::tuple<std::string, std::string>> metadata;
+                    std::transform(parameters.m_Data.constKeyValueBegin(), parameters.m_Data.constKeyValueEnd(),
+                        std::back_inserter(metadata), [](const auto& item) {
+                            return std::tuple<std::string, std::string> { item.first.toStdString(),
+                                item.second.toString().toStdString() };
+                        });
+                    auto result = _pythonFunction(metadata, range.m_TStart, range.m_TEnd);
                     return TimeSeriesUtils::copy(result);
                 }
             private:
                 PythonInterpreter::provider_funct_t _pythonFunction;
             };
             void PythonProviders::initialize()
             {
                 _interpreter.add_register_callback(
-                    [this](const std::vector<std::pair<std::string,
+                    [this](const std::vector<PythonInterpreter::product_t>& product_list,
-                               std::vector<std::pair<std::string, std::string>>>>& product_list,
                         PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); });
                 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
                 {
                     auto dir = QDir(path + "/python");
                     if (dir.exists())
                     {
                         for (const auto& entry :
                             dir.entryInfoList(QDir::Files | QDir::NoDotAndDotDot, QDir::Name))
                         {
                             if (entry.isFile() && entry.suffix() == "py")
                             {
                                 _interpreter.eval(entry.absoluteFilePath().toStdString());
                             }
                         }
                     }
                 }
                 _interpreter.release();
             }
             PythonProviders::~PythonProviders() {}
             std::unique_ptr<DataSourceItem> make_folder_item(const QString& name)
             {
                 return std::make_unique<DataSourceItem>(DataSourceItemType::NODE, name);
             }
             template <typename T>
             DataSourceItem* make_path_items(
                 const T& path_list_begin, const T& path_list_end, DataSourceItem* root)
             {
                 std::for_each(path_list_begin, path_list_end, [&root](const auto& folder_name) mutable {
                     auto folder_ptr = root->findItem(folder_name);
                     if (folder_ptr == nullptr)
                     {
                         auto folder = make_folder_item(folder_name);
                         folder_ptr = folder.get();
                         root->appendChild(std::move(folder));
                     }
                     root = folder_ptr;
                 });
                 return root;
             }
             std::unique_ptr<DataSourceItem> make_product_item(
                 const QVariantHash& metaData, const QUuid& dataSourceUid)
             {
                 auto result = std::make_unique<DataSourceItem>(DataSourceItemType::PRODUCT, metaData);
                 // Adds plugin name to product metadata
                 result->setData(DataSourceItem::PLUGIN_DATA_KEY, DATA_SOURCE_NAME);
                 result->setData(DataSourceItem::ID_DATA_KEY, metaData.value(DataSourceItem::NAME_DATA_KEY));
                 auto productName = metaData.value(DataSourceItem::NAME_DATA_KEY).toString();
                 // Add action to load product from DataSourceController
                 result->addAction(
                     std::make_unique<DataSourceItemAction>(QObject::tr("Load %1 product").arg(productName),
                         [productName, dataSourceUid](DataSourceItem& item) {
                             if (auto app = sqpApp)
                             {
                                 app->dataSourceController().loadProductItem(dataSourceUid, item);
                             }
                         }));
                 return result;
             }
             void PythonProviders::register_product(
-                const std::vector<std::pair<std::string, std::vector<std::pair<std::string, std::string>>>>&
+                const std::vector<PythonInterpreter::product_t>& product_list,
-                    product_list,
                 PythonInterpreter::provider_funct_t f)
             {
                 auto& dataSourceController = sqpApp->dataSourceController();
-                auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
+                QString test = DATA_SOURCE_NAME + QUuid::createUuid().toString();
-                auto root = make_folder_item(DATA_SOURCE_NAME);
+                auto id = dataSourceController.registerDataSource(test);
+                auto root = make_folder_item(test);
                 std::for_each(std::cbegin(product_list), std::cend(product_list),
                     [id, f, root = root.get()](const auto& product) {
-                        const auto& path = product.first;
+                        const auto& path = std::get<0>(product);
                         auto path_list = QString::fromStdString(path).split('/');
                         auto name = *(std::cend(path_list) - 1);
                         auto path_item
                             = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
                         QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } };
-                        std::for_each(std::cbegin(product.second), std::cend(product.second),
+                        std::for_each(std::cbegin(std::get<2>(product)), std::cend(std::get<2>(product)),
                             [&metaData](const auto& mdata) {
                                 metaData[QString::fromStdString(mdata.first)]
                                     = QString::fromStdString(mdata.second);
                             });
                         path_item->appendChild(make_product_item(metaData, id));
                     });
                 dataSourceController.setDataSourceItem(id, std::move(root));
                 dataSourceController.setDataProvider(id, std::make_unique<PythonProvider>(f));
                 std::cout << "Gone there" << std::endl;
             }

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages