##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Some more progress on Python Providers side, works with a simple request on CDAWeb....
jeandet -
r1429:78e9c85d13ee
parent child
Show More
Side by Side Unified
Context file:
r1429:78e9c85d13ee Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,28
1 import sys
2 sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
3 import os
4 import datetime
5 import PythonProviders
6 import pysciqlopcore
7 import numpy as np
8 import pandas as pds
9 import requests
10
11 def get_sample(name,start,stop):
12 try:
13 tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')
14 tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')
15 req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"
16 resp = requests.get(req_url,headers={"Accept":"application/json"})
17 csv_url = resp.json()['FileDescription'][0]['Name']
18 df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)
19 t = np.array([d.timestamp()-7200 for d in df.index])
20 values = df.values
21 return pysciqlopcore.VectorTimeSerie(t,values)
22 except Exception as e:
23 print("fuck ",str(e))
24 return pysciqlopcore.VectorTimeSerie(1)
25
26 PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample)
27
28
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 Subproject commit 68c01155acd227b01fa3c9f7926215e8cedd70df
1 Subproject commit b9379e008899fce30581fbbf2b3fefd6fe36cee2
Show file before | Show file after | Hide comments
@@ -8,12 +8,14
8 class PythonInterpreter
8 class PythonInterpreter
9 {
9 {
10 public:
10 public:
11 using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>;
11 PythonInterpreter();
12 PythonInterpreter();
12 void add_register_callback(std::function<void(const std::vector<std::string>&,
13 void add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
13 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>)>
14 provider_funct_t)>
14 callback);
15 callback);
15 ~PythonInterpreter();
16 ~PythonInterpreter();
16 void eval(const std::string& file);
17 void eval(const std::string& file);
18 void release();
17
19
18 private:
20 private:
19 };
21 };
Show file before | Show file after | Hide comments
@@ -24,8 +24,8 public:
24 ~PythonProviders();
24 ~PythonProviders();
25
25
26 private:
26 private:
27 void register_product(const std::vector<std::string>& path_list,
27 void register_product(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>& product_list,
28 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string& name, double, double)>
28 PythonInterpreter::provider_funct_t
29 f);
29 f);
30 PythonInterpreter _interpreter;
30 PythonInterpreter _interpreter;
31 };
31 };
Show file before | Show file after | Hide comments
@@ -18,8 +18,8 PythonInterpreter::PythonInterpreter()
18 py::initialize_interpreter(false);
18 py::initialize_interpreter(false);
19 }
19 }
20
20
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::string>&,
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
22 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>)>
22 provider_funct_t)>
23 callback)
23 callback)
24 {
24 {
25 py::module PythonProviders = py::module::import("PythonProviders");
25 py::module PythonProviders = py::module::import("PythonProviders");
@@ -43,5 +43,9 void PythonInterpreter::eval(const std::string& file)
43 {
43 {
44 std::cout << pythonErr.what();
44 std::cout << pythonErr.what();
45 }
45 }
46 }
47
48 void PythonInterpreter::release()
49 {
46 _rel = new py::gil_scoped_release();
50 _rel = new py::gil_scoped_release();
47 }
51 }
Show file before | Show file after | Hide comments
@@ -4,6 +4,7
4 #include <Data/IDataProvider.h>
4 #include <Data/IDataProvider.h>
5 #include <Data/ScalarTimeSerie.h>
5 #include <Data/ScalarTimeSerie.h>
6 #include <Data/SpectrogramTimeSerie.h>
6 #include <Data/SpectrogramTimeSerie.h>
7 #include <Data/TimeSeriesUtils.h>
7 #include <Data/VectorTimeSerie.h>
8 #include <Data/VectorTimeSerie.h>
8 #include <DataSource/DataSourceController.h>
9 #include <DataSource/DataSourceController.h>
9 #include <DataSource/DataSourceItem.h>
10 #include <DataSource/DataSourceItem.h>
@@ -27,11 +28,7 struct noop_deleter
27 class PythonProvider : public IDataProvider
28 class PythonProvider : public IDataProvider
28 {
29 {
29 public:
30 public:
30 PythonProvider(
31 PythonProvider(PythonInterpreter::provider_funct_t f) : _pythonFunction { f } {}
31 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)> f)
32 : _pythonFunction { f }
33 {
34 }
35
32
36 PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {}
33 PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {}
37
34
@@ -44,33 +41,20 public:
44 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
41 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
45 auto range = parameters.m_Range;
42 auto range = parameters.m_Range;
46 auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
43 auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
47 if (auto ts = std::dynamic_pointer_cast<VectorTimeSerie>(result))
44 return TimeSeriesUtils::copy(result);
48 {
49 return new VectorTimeSerie(*ts);
50 }
51 if (auto ts = std::dynamic_pointer_cast<ScalarTimeSerie>(result))
52 {
53 return new ScalarTimeSerie(*ts);
54 }
55 if (auto ts = std::dynamic_pointer_cast<SpectrogramTimeSerie>(result))
56 {
57 return new SpectrogramTimeSerie(*ts);
58 }
59 return nullptr;
60 }
45 }
61
46
62 private:
47 private:
63 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>
48 PythonInterpreter::provider_funct_t _pythonFunction;
64 _pythonFunction;
65 };
49 };
66
50
67
51
68 void PythonProviders::initialize()
52 void PythonProviders::initialize()
69 {
53 {
70 _interpreter.add_register_callback(
54 _interpreter.add_register_callback(
71 [this](const std::vector<std::string>& path_list,
55 [this](const std::vector<std::pair<std::string,
72 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>
56 std::vector<std::pair<std::string, std::string>>>>& product_list,
73 f) { this->register_product(path_list, f); });
57 PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); });
74
58
75 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
59 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
76 {
60 {
@@ -87,6 +71,7 void PythonProviders::initialize()
87 }
71 }
88 }
72 }
89 }
73 }
74 _interpreter.release();
90 }
75 }
91
76
92 PythonProviders::~PythonProviders() {}
77 PythonProviders::~PythonProviders() {}
@@ -137,20 +122,28 std::unique_ptr<DataSourceItem> make_product_item(
137 return result;
122 return result;
138 }
123 }
139
124
140 void PythonProviders::register_product(const std::vector<std::string>& path_list,
125 void PythonProviders::register_product(
141 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)> f)
126 const std::vector<std::pair<std::string, std::vector<std::pair<std::string, std::string>>>>&
127 product_list,
128 PythonInterpreter::provider_funct_t f)
142 {
129 {
143 auto& dataSourceController = sqpApp->dataSourceController();
130 auto& dataSourceController = sqpApp->dataSourceController();
144 auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
131 auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
145 auto root = make_folder_item(DATA_SOURCE_NAME);
132 auto root = make_folder_item(DATA_SOURCE_NAME);
146 std::for_each(
133 std::for_each(std::cbegin(product_list), std::cend(product_list),
147 std::cbegin(path_list), std::cend(path_list), [id, f, root = root.get()](const auto& path) {
134 [id, f, root = root.get()](const auto& product) {
135 const auto& path = product.first;
148 auto path_list = QString::fromStdString(path).split('/');
136 auto path_list = QString::fromStdString(path).split('/');
149 auto name = *(std::cend(path_list) - 1);
137 auto name = *(std::cend(path_list) - 1);
150 auto path_item
138 auto path_item
151 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
139 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
152 path_item->appendChild(
140 QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } };
153 make_product_item({ { DataSourceItem::NAME_DATA_KEY, name } }, id));
141 std::for_each(std::cbegin(product.second), std::cend(product.second),
142 [&metaData](const auto& mdata) {
143 metaData[QString::fromStdString(mdata.first)]
144 = QString::fromStdString(mdata.second);
145 });
146 path_item->appendChild(make_product_item(metaData, id));
154 });
147 });
155 dataSourceController.setDataSourceItem(id, std::move(root));
148 dataSourceController.setDataSourceItem(id, std::move(root));
156 dataSourceController.setDataProvider(id, std::make_unique<PythonProvider>(f));
149 dataSourceController.setDataProvider(id, std::make_unique<PythonProvider>(f));
General Comments 0
You need to be logged in to leave comments. Login now