diff --git a/plugins/python_providers/include/python_interpreter.h b/plugins/python_providers/include/python_interpreter.h index 92e40f7..792c44c 100644 --- a/plugins/python_providers/include/python_interpreter.h +++ b/plugins/python_providers/include/python_interpreter.h @@ -8,11 +8,14 @@ class PythonInterpreter { public: - using provider_funct_t = std::function(std::string&, double, double)>; + using provider_funct_t = std::function( + std::vector>&, double, double)>; + using product_t = std::tuple, + std::vector>>; + PythonInterpreter(); - void add_register_callback(std::function>>>&, - provider_funct_t)> - callback); + void add_register_callback( + std::function&, provider_funct_t)> callback); ~PythonInterpreter(); void eval(const std::string& file); void release(); diff --git a/plugins/python_providers/include/python_providers.h b/plugins/python_providers/include/python_providers.h index e910000..a20bbc5 100644 --- a/plugins/python_providers/include/python_providers.h +++ b/plugins/python_providers/include/python_providers.h @@ -24,9 +24,8 @@ public: ~PythonProviders(); private: - void register_product(const std::vector>>>& product_list, - PythonInterpreter::provider_funct_t - f); + void register_product(const std::vector& product_list, + PythonInterpreter::provider_funct_t f); PythonInterpreter _interpreter; }; diff --git a/plugins/python_providers/resources/amda.py b/plugins/python_providers/resources/amda.py index 25e9f50..f3cff40 100644 --- a/plugins/python_providers/resources/amda.py +++ b/plugins/python_providers/resources/amda.py @@ -7,22 +7,54 @@ import pysciqlopcore import numpy as np import pandas as pds import requests +from spwc.amda import AMDA -def get_sample(name,start,stop): +amda = AMDA() + +def get_sample(metadata,start,stop): + ts_type = pysciqlopcore.ScalarTimeSerie try: - tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ') - tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ') - req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv" - resp = requests.get(req_url,headers={"Accept":"application/json"}) - csv_url = resp.json()['FileDescription'][0]['Name'] - df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True) + param_id = None + for key,value in metadata: + if key == 'xml:id': + param_id = value + elif key == 'type': + if value == 'vector': + ts_type = pysciqlopcore.VectorTimeSerie + tstart=datetime.datetime.fromtimestamp(start) + tend=datetime.datetime.fromtimestamp(stop) + df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id) t = np.array([d.timestamp()-7200 for d in df.index]) values = df.values - return pysciqlopcore.VectorTimeSerie(t,values) + print(len(t)) + print(len(values)) + return ts_type(t,values) + return ts_type(1) except Exception as e: - print("fuck ",str(e)) - return pysciqlopcore.VectorTimeSerie(1) + print("Error in amda.py ",str(e)) + return ts_type(1) + + +if len(amda.component) is 0: + amda.update_inventory() +parameters = amda.parameter.copy() +for name,component in amda.component.items(): + if 'components' in parameters[component['parameter']]: + parameters[component['parameter']]['components'].append(component) + else: + parameters[component['parameter']]['components']=[component] + +products = [] +for key,parameter in parameters.items(): + path = f"/AMDA/{parameter['mission']}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}" + components = [component['name'] for component in parameter.get('components',[])] + metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ] + if parameter.get('size',0) is '3': + metadata.append(("type","vector")) + else: + metadata.append(("type","scalar")) + products.append( (path, components, metadata)) -PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[("type","vector")])],get_sample) +PythonProviders.register_product(products, get_sample) diff --git a/plugins/python_providers/resources/cdaweb.py b/plugins/python_providers/resources/cdaweb.py new file mode 100644 index 0000000..ac52c90 --- /dev/null +++ b/plugins/python_providers/resources/cdaweb.py @@ -0,0 +1,44 @@ +import sys +sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core") +import os +import datetime +import PythonProviders +import pysciqlopcore +import numpy as np +import pandas as pds +import requests +from spwc.cdaweb import cdaweb + +cd = cdaweb() + +def get_sample(name,start,stop): + try: + tstart=datetime.datetime.fromtimestamp(start).strftime('%Y%m%dT%H%M%SZ') + tend=datetime.datetime.fromtimestamp(stop).strftime('%Y%m%dT%H%M%SZ') + req_url=f"https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/datasets/MMS4_SCM_SRVY_L2_SCSRVY/data/{tstart},{tend}/mms4_scm_acb_gse_scsrvy_srvy_l2?format=csv" + resp = requests.get(req_url,headers={"Accept":"application/json"}) + csv_url = resp.json()['FileDescription'][0]['Name'] + df = pds.read_csv(csv_url,comment='#',index_col=0, infer_datetime_format=True,parse_dates=True) + t = np.array([d.timestamp()-7200 for d in df.index]) + values = df.values + return pysciqlopcore.VectorTimeSerie(t,values) + except Exception as e: + print("fuck ",str(e)) + return pysciqlopcore.VectorTimeSerie(1) + +def get_sample(name,start,stop): + try: + tstart=datetime.datetime.fromtimestamp(start) + tend=datetime.datetime.fromtimestamp(stop) + df = cd.get_variable(dataset="MMS2_SCM_SRVY_L2_SCSRVY",variable="mms2_scm_acb_gse_scsrvy_srvy_l2",tstart=tstart,tend=tend) + t = np.array([d.timestamp()-7200 for d in df.index]) + values = df.values + return pysciqlopcore.VectorTimeSerie(t,values) + except Exception as e: + print("fuck ",str(e)) + return pysciqlopcore.VectorTimeSerie(1) + + +PythonProviders.register_product([("/CDA/mms4_scm_acb_gse_scsrvy_srvy_l2",[],[("type","vector")])],get_sample) + + diff --git a/plugins/python_providers/src/python_interpreter.cpp b/plugins/python_providers/src/python_interpreter.cpp index 3b5d812..662d8d7 100644 --- a/plugins/python_providers/src/python_interpreter.cpp +++ b/plugins/python_providers/src/python_interpreter.cpp @@ -18,9 +18,8 @@ PythonInterpreter::PythonInterpreter() py::initialize_interpreter(false); } -void PythonInterpreter::add_register_callback(std::function>>>&, - provider_funct_t)> - callback) +void PythonInterpreter::add_register_callback( + std::function&, provider_funct_t)> callback) { py::module PythonProviders = py::module::import("PythonProviders"); PythonProviders.attr("register_product") = callback; diff --git a/plugins/python_providers/src/python_providers.cpp b/plugins/python_providers/src/python_providers.cpp index 63957d7..27acf63 100644 --- a/plugins/python_providers/src/python_providers.cpp +++ b/plugins/python_providers/src/python_providers.cpp @@ -20,11 +20,6 @@ const auto DATA_SOURCE_NAME = QStringLiteral("PythonProviders"); -struct noop_deleter -{ - void operator()(TimeSeries::ITimeSerie*) {} -}; - class PythonProvider : public IDataProvider { public: @@ -40,7 +35,13 @@ public: { auto product = parameters.m_Data.value("PRODUCT", "").toString().toStdString(); auto range = parameters.m_Range; - auto result = _pythonFunction(product, range.m_TStart, range.m_TEnd); + std::vector> metadata; + std::transform(parameters.m_Data.constKeyValueBegin(), parameters.m_Data.constKeyValueEnd(), + std::back_inserter(metadata), [](const auto& item) { + return std::tuple { item.first.toStdString(), + item.second.toString().toStdString() }; + }); + auto result = _pythonFunction(metadata, range.m_TStart, range.m_TEnd); return TimeSeriesUtils::copy(result); } @@ -52,8 +53,7 @@ private: void PythonProviders::initialize() { _interpreter.add_register_callback( - [this](const std::vector>>>& product_list, + [this](const std::vector& product_list, PythonInterpreter::provider_funct_t f) { this->register_product(product_list, f); }); for (const auto& path : QStandardPaths::standardLocations(QStandardPaths::AppLocalDataLocation)) @@ -123,22 +123,22 @@ std::unique_ptr make_product_item( } void PythonProviders::register_product( - const std::vector>>>& - product_list, + const std::vector& product_list, PythonInterpreter::provider_funct_t f) { auto& dataSourceController = sqpApp->dataSourceController(); - auto id = dataSourceController.registerDataSource(DATA_SOURCE_NAME); - auto root = make_folder_item(DATA_SOURCE_NAME); + QString test = DATA_SOURCE_NAME + QUuid::createUuid().toString(); + auto id = dataSourceController.registerDataSource(test); + auto root = make_folder_item(test); std::for_each(std::cbegin(product_list), std::cend(product_list), [id, f, root = root.get()](const auto& product) { - const auto& path = product.first; + const auto& path = std::get<0>(product); auto path_list = QString::fromStdString(path).split('/'); auto name = *(std::cend(path_list) - 1); auto path_item = make_path_items(std::cbegin(path_list), std::cend(path_list) - 1, root); QVariantHash metaData { { DataSourceItem::NAME_DATA_KEY, name } }; - std::for_each(std::cbegin(product.second), std::cend(product.second), + std::for_each(std::cbegin(std::get<2>(product)), std::cend(std::get<2>(product)), [&metaData](const auto& mdata) { metaData[QString::fromStdString(mdata.first)] = QString::fromStdString(mdata.second);