##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Some more progress on Python Providers side, works with a simple request on CDAWeb....
jeandet -
r1429:78e9c85d13ee
parent child
Show More
Side by Side Unified
Context file:
r1429:78e9c85d13ee Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,28
1 import sys
2 sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
3 import os
4 import datetime
5 import PythonProviders
6 import pysciqlopcore
7 import numpy as np
8 import pandas as pds
9 import requests
10
11 def get_sample(name,start,stop):
12 try:
13 tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')
14 tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')
15 req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"
16 resp = requests.get(req_url,headers={"Accept":"application/json"})
17 csv_url = resp.json()['FileDescription'][0]['Name']
18 df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)
19 t = np.array([d.timestamp()-7200 for d in df.index])
20 values = df.values
21 return pysciqlopcore.VectorTimeSerie(t,values)
22 except Exception as e:
23 print("fuck ",str(e))
24 return pysciqlopcore.VectorTimeSerie(1)
25
26 PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample)
27
28
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 Subproject commit 68c01155acd227b01fa3c9f7926215e8cedd70df
1 Subproject commit b9379e008899fce30581fbbf2b3fefd6fe36cee2
Show file before | Show file after | Hide comments
@@ -8,12 +8,14
8 8 class PythonInterpreter
9 9 {
10 10 public:
11 using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>;
11 12 PythonInterpreter();
12 void add_register_callback(std::function<void(const std::vector<std::string>&,
13 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>)>
13 void add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
14 provider_funct_t)>
14 15 callback);
15 16 ~PythonInterpreter();
16 17 void eval(const std::string& file);
18 void release();
17 19
18 20 private:
19 21 };
Show file before | Show file after | Hide comments
@@ -24,8 +24,8 public:
24 24 ~PythonProviders();
25 25
26 26 private:
27 void register_product(const std::vector<std::string>& path_list,
28 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string& name, double, double)>
27 void register_product(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>& product_list,
28 PythonInterpreter::provider_funct_t
29 29 f);
30 30 PythonInterpreter _interpreter;
31 31 };
Show file before | Show file after | Hide comments
@@ -18,8 +18,8 PythonInterpreter::PythonInterpreter()
18 18 py::initialize_interpreter(false);
19 19 }
20 20
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::string>&,
22 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>)>
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
22 provider_funct_t)>
23 23 callback)
24 24 {
25 25 py::module PythonProviders = py::module::import("PythonProviders");
@@ -43,5 +43,9 void PythonInterpreter::eval(const std::string& file)
43 43 {
44 44 std::cout << pythonErr.what();
45 45 }
46 }
47
48 void PythonInterpreter::release()
49 {
46 50 _rel = new py::gil_scoped_release();
47 51 }
Show file before | Show file after | Hide comments
@@ -4,6 +4,7
4 4 #include <Data/IDataProvider.h>
5 5 #include <Data/ScalarTimeSerie.h>
6 6 #include <Data/SpectrogramTimeSerie.h>
7 #include <Data/TimeSeriesUtils.h>
7 8 #include <Data/VectorTimeSerie.h>
8 9 #include <DataSource/DataSourceController.h>
9 10 #include <DataSource/DataSourceItem.h>
@@ -27,11 +28,7 struct noop_deleter
27 28 class PythonProvider : public IDataProvider
28 29 {
29 30 public:
30 PythonProvider(
31 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)> f)
32 : _pythonFunction { f }
33 {
34 }
31 PythonProvider(PythonInterpreter::provider_funct_t f) : _pythonFunction { f } {}
35 32
36 33 PythonProvider(const PythonProvider& other) : _pythonFunction { other._pythonFunction } {}
37 34
@@ -44,33 +41,20 public:
44 41 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
45 42 auto range = parameters.m_Range;
46 43 auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
47 if (auto ts = std::dynamic_pointer_cast<VectorTimeSerie>(result))
48 {
49 return new VectorTimeSerie(*ts);
50 }
51 if (auto ts = std::dynamic_pointer_cast<ScalarTimeSerie>(result))
52 {
53 return new ScalarTimeSerie(*ts);
54 }
55 if (auto ts = std::dynamic_pointer_cast<SpectrogramTimeSerie>(result))
56 {
57 return new SpectrogramTimeSerie(*ts);
58 }
59 return nullptr;
44 return TimeSeriesUtils::copy(result);
60 45 }
61 46
62 47 private:
63 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>
64 _pythonFunction;
48 PythonInterpreter::provider_funct_t _pythonFunction;
65 49 };
66 50
67 51
68 52 void PythonProviders::initialize()
69 53 {
70 54 _interpreter.add_register_callback(
71 [this](const std::vector<std::string>& path_list,
72 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>
73 f) { this->register_product(path_list, f); });
55 [this](const std::vector<std::pair<std::string,
56 std::vector<std::pair<std::string, std::string>>>>& product_list,
57 PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); });
74 58
75 59 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
76 60 {
@@ -87,6 +71,7 void PythonProviders::initialize()
87 71 }
88 72 }
89 73 }
74 _interpreter.release();
90 75 }
91 76
92 77 PythonProviders::~PythonProviders() {}
@@ -137,20 +122,28 std::unique_ptr<DataSourceItem> make_product_item(
137 122 return result;
138 123 }
139 124
140 void PythonProviders::register_product(const std::vector<std::string>& path_list,
141 std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)> f)
125 void PythonProviders::register_product(
126 const std::vector<std::pair<std::string, std::vector<std::pair<std::string, std::string>>>>&
127 product_list,
128 PythonInterpreter::provider_funct_t f)
142 129 {
143 130 auto& dataSourceController = sqpApp->dataSourceController();
144 131 auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
145 132 auto root = make_folder_item(DATA_SOURCE_NAME);
146 std::for_each(
147 std::cbegin(path_list), std::cend(path_list), [id, f, root = root.get()](const auto& path) {
133 std::for_each(std::cbegin(product_list), std::cend(product_list),
134 [id, f, root = root.get()](const auto& product) {
135 const auto& path = product.first;
148 136 auto path_list = QString::fromStdString(path).split('/');
149 137 auto name = *(std::cend(path_list) - 1);
150 138 auto path_item
151 139 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
152 path_item->appendChild(
153 make_product_item({ { DataSourceItem::NAME_DATA_KEY, name } }, id));
140 QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } };
141 std::for_each(std::cbegin(product.second), std::cend(product.second),
142 [&metaData](const auto& mdata) {
143 metaData[QString::fromStdString(mdata.first)]
144 = QString::fromStdString(mdata.second);
145 });
146 path_item->appendChild(make_product_item(metaData, id));
154 147 });
155 148 dataSourceController.setDataSourceItem(id, std::move(root));
156 149 dataSourceController.setDataProvider(id, std::make_unique<PythonProvider>(f));
General Comments 0
You need to be logged in to leave comments. Login now