##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Added POC AMDA python impl and CDAWEB bits...
jeandet -
r1430:ecc8b7b09c3d
parent child
Show More
Side by Side Unified
Context file:
r1430:ecc8b7b09c3d Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,44
1 import sys
2 sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
3 import os
4 import datetime
5 import PythonProviders
6 import pysciqlopcore
7 import numpy as np
8 import pandas as pds
9 import requests
10 from spwc.cdaweb import cdaweb
11
12 cd = cdaweb()
13
14 def get_sample(name,start,stop):
15 try:
16 tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')
17 tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')
18 req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"
19 resp = requests.get(req_url,headers={"Accept":"application/json"})
20 csv_url = resp.json()['FileDescription'][0]['Name']
21 df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)
22 t = np.array([d.timestamp()-7200 for d in df.index])
23 values = df.values
24 return pysciqlopcore.VectorTimeSerie(t,values)
25 except Exception as e:
26 print("fuck ",str(e))
27 return pysciqlopcore.VectorTimeSerie(1)
28
29 def get_sample(name,start,stop):
30 try:
31 tstart=datetime.datetime.fromtimestamp(start)
32 tend=datetime.datetime.fromtimestamp(stop)
33 df = cd.get_variable(dataset="MMS2_SCM_SRVY_L2_SCSRVY",variable="mms2_scm_acb_gse_scsrvy_srvy_l2",tstart=tstart,tend=tend)
34 t = np.array([d.timestamp()-7200 for d in df.index])
35 values = df.values
36 return pysciqlopcore.VectorTimeSerie(t,values)
37 except Exception as e:
38 print("fuck ",str(e))
39 return pysciqlopcore.VectorTimeSerie(1)
40
41
42 PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[],[("type","vector")])],get_sample)
43
44
Show file before | Show file after | Hide comments
@@ -8,11 +8,14
8 class PythonInterpreter
8 class PythonInterpreter
9 {
9 {
10 public:
10 public:
11 using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(std::string&, double, double)>;
11 using provider_funct_t = std::function<std::shared_ptr<TimeSeries::ITimeSerie>(
12 std::vector<std::tuple<std::string, std::string>>&, double, double)>;
13 using product_t = std::tuple<std::string, std::vector<std::string>,
14 std::vector<std::pair<std::string, std::string>>>;
15
12 PythonInterpreter();
16 PythonInterpreter();
13 void add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
17 void add_register_callback(
14 provider_funct_t)>
18 std::function<void(const std::vector<product_t>&, provider_funct_t)> callback);
15 callback);
16 ~PythonInterpreter();
19 ~PythonInterpreter();
17 void eval(const std::string& file);
20 void eval(const std::string& file);
18 void release();
21 void release();
Show file before | Show file after | Hide comments
@@ -24,9 +24,8 public:
24 ~PythonProviders();
24 ~PythonProviders();
25
25
26 private:
26 private:
27 void register_product(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>& product_list,
27 void register_product(const std::vector<PythonInterpreter::product_t>& product_list,
28 PythonInterpreter::provider_funct_t
28 PythonInterpreter::provider_funct_t f);
29 f);
30 PythonInterpreter _interpreter;
29 PythonInterpreter _interpreter;
31 };
30 };
32
31
Show file before | Show file after | Hide comments
@@ -7,22 +7,54 import pysciqlopcore
7 import numpy as np
7 import numpy as np
8 import pandas as pds
8 import pandas as pds
9 import requests
9 import requests
10 from spwc.amda import AMDA
10
11
11 def get_sample(name,start,stop):
12 amda = AMDA()
13
14 def get_sample(metadata,start,stop):
15 ts_type = pysciqlopcore.ScalarTimeSerie
12 try:
16 try:
13 tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ')
17 param_id = None
14 tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ')
18 for key,value in metadata:
15 req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv"
19 if key == 'xml:id':
16 resp = requests.get(req_url,headers={"Accept":"application/json"})
20 param_id = value
17 csv_url = resp.json()['FileDescription'][0]['Name']
21 elif key == 'type':
18 df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True)
22 if value == 'vector':
23 ts_type = pysciqlopcore.VectorTimeSerie
24 tstart=datetime.datetime.fromtimestamp(start)
25 tend=datetime.datetime.fromtimestamp(stop)
26 df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id)
19 t = np.array([d.timestamp()-7200 for d in df.index])
27 t = np.array([d.timestamp()-7200 for d in df.index])
20 values = df.values
28 values = df.values
21 return pysciqlopcore.VectorTimeSerie(t,values)
29 print(len(t))
30 print(len(values))
31 return ts_type(t,values)
32 return ts_type(1)
22 except Exception as e:
33 except Exception as e:
23 print("fuck ",str(e))
34 print("Error in amda.py ",str(e))
24 return pysciqlopcore.VectorTimeSerie(1)
35 return ts_type(1)
36
37
38 if len(amda.component) is 0:
39 amda.update_inventory()
40 parameters = amda.parameter.copy()
41 for name,component in amda.component.items():
42 if 'components' in parameters[component['parameter']]:
43 parameters[component['parameter']]['components'].append(component)
44 else:
45 parameters[component['parameter']]['components']=[component]
46
47 products = []
48 for key,parameter in parameters.items():
49 path = f"/AMDA/{parameter['mission']}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
50 components = [component['name'] for component in parameter.get('components',[])]
51 metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ]
52 if parameter.get('size',0) is '3':
53 metadata.append(("type","vector"))
54 else:
55 metadata.append(("type","scalar"))
56 products.append( (path, components, metadata))
25
57
26 PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample)
58 PythonProviders.register_product(products, get_sample)
27
59
28
60
Show file before | Show file after | Hide comments
@@ -18,9 +18,8 PythonInterpreter::PythonInterpreter()
18 py::initialize_interpreter(false);
18 py::initialize_interpreter(false);
19 }
19 }
20
20
21 void PythonInterpreter::add_register_callback(std::function<void(const std::vector<std::pair<std::string,std::vector<std::pair<std::string,std::string>>>>&,
21 void PythonInterpreter::add_register_callback(
22 provider_funct_t)>
22 std::function<void(const std::vector<product_t>&, provider_funct_t)> callback)
23 callback)
24 {
23 {
25 py::module PythonProviders = py::module::import("PythonProviders");
24 py::module PythonProviders = py::module::import("PythonProviders");
26 PythonProviders.attr("register_product") = callback;
25 PythonProviders.attr("register_product") = callback;
Show file before | Show file after | Hide comments
@@ -20,11 +20,6
20
20
21 const auto DATA_SOURCE_NAME = QStringLiteral("PythonProviders");
21 const auto DATA_SOURCE_NAME = QStringLiteral("PythonProviders");
22
22
23 struct noop_deleter
24 {
25 void operator()(TimeSeries::ITimeSerie*) {}
26 };
27
28 class PythonProvider : public IDataProvider
23 class PythonProvider : public IDataProvider
29 {
24 {
30 public:
25 public:
@@ -40,7 +35,13 public:
40 {
35 {
41 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
36 auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString();
42 auto range = parameters.m_Range;
37 auto range = parameters.m_Range;
43 auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd);
38 std::vector<std::tuple<std::string, std::string>> metadata;
39 std::transform(parameters.m_Data.constKeyValueBegin(), parameters.m_Data.constKeyValueEnd(),
40 std::back_inserter(metadata), [](const auto& item) {
41 return std::tuple<std::string, std::string> { item.first.toStdString(),
42 item.second.toString().toStdString() };
43 });
44 auto result = _pythonFunction(metadata, range.m_TStart, range.m_TEnd);
44 return TimeSeriesUtils::copy(result);
45 return TimeSeriesUtils::copy(result);
45 }
46 }
46
47
@@ -52,8 +53,7 private:
52 void PythonProviders::initialize()
53 void PythonProviders::initialize()
53 {
54 {
54 _interpreter.add_register_callback(
55 _interpreter.add_register_callback(
55 [this](const std::vector<std::pair<std::string,
56 [this](const std::vector<PythonInterpreter::product_t>& product_list,
56 std::vector<std::pair<std::string, std::string>>>>& product_list,
57 PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); });
57 PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); });
58
58
59 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
59 for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation))
@@ -123,22 +123,22 std::unique_ptr<DataSourceItem> make_product_item(
123 }
123 }
124
124
125 void PythonProviders::register_product(
125 void PythonProviders::register_product(
126 const std::vector<std::pair<std::string, std::vector<std::pair<std::string, std::string>>>>&
126 const std::vector<PythonInterpreter::product_t>& product_list,
127 product_list,
128 PythonInterpreter::provider_funct_t f)
127 PythonInterpreter::provider_funct_t f)
129 {
128 {
130 auto& dataSourceController = sqpApp->dataSourceController();
129 auto& dataSourceController = sqpApp->dataSourceController();
131 auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME);
130 QString test = DATA_SOURCE_NAME + QUuid::createUuid().toString();
132 auto root = make_folder_item(DATA_SOURCE_NAME);
131 auto id = dataSourceController.registerDataSource(test);
132 auto root = make_folder_item(test);
133 std::for_each(std::cbegin(product_list), std::cend(product_list),
133 std::for_each(std::cbegin(product_list), std::cend(product_list),
134 [id, f, root = root.get()](const auto& product) {
134 [id, f, root = root.get()](const auto& product) {
135 const auto& path = product.first;
135 const auto& path = std::get<0>(product);
136 auto path_list = QString::fromStdString(path).split('/');
136 auto path_list = QString::fromStdString(path).split('/');
137 auto name = *(std::cend(path_list) - 1);
137 auto name = *(std::cend(path_list) - 1);
138 auto path_item
138 auto path_item
139 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
139 = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root);
140 QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } };
140 QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } };
141 std::for_each(std::cbegin(product.second), std::cend(product.second),
141 std::for_each(std::cbegin(std::get<2>(product)), std::cend(std::get<2>(product)),
142 [&metaData](const auto& mdata) {
142 [&metaData](const auto& mdata) {
143 metaData[QString::fromStdString(mdata.first)]
143 metaData[QString::fromStdString(mdata.first)]
144 = QString::fromStdString(mdata.second);
144 = QString::fromStdString(mdata.second);
General Comments 0
You need to be logged in to leave comments. Login now