@@ -1,134 +1,136 | |||
|
1 | 1 | #pragma once |
|
2 | 2 | #include <Data/DataProviderParameters.h> |
|
3 | 3 | #include <Data/DataSeriesType.h> |
|
4 | 4 | #include <Data/IDataProvider.h> |
|
5 | 5 | #include <DataSource/DataSourceItem.h> |
|
6 | 6 | #include <DataSource/DataSourceItemAction.h> |
|
7 | 7 | #include <DataSource/datasources.h> |
|
8 | 8 | |
|
9 | 9 | #include <QPair> |
|
10 | #include <QList> | |
|
10 | 11 | #include <SqpApplication.h> |
|
11 | 12 | // must be included last because of Python/Qt definition of slots |
|
12 | 13 | #include "numpy_wrappers.h" |
|
13 | 14 | |
|
14 | 15 | struct Product |
|
15 | 16 | { |
|
16 | 17 | QString path; |
|
17 | 18 | std::vector<std::string> components; |
|
18 | 19 | QMap<QString, QString> metadata; |
|
19 | 20 | Product() = default; |
|
20 | 21 | explicit Product(const QString& path, const std::vector<std::string>& components, |
|
21 | 22 | const QMap<QString, QString>& metadata) |
|
22 | 23 | : path { path }, components { components }, metadata { metadata } |
|
23 | 24 | { |
|
24 | 25 | } |
|
25 | 26 | ~Product() = default; |
|
26 | 27 | }; |
|
27 | 28 | |
|
28 | 29 | template <typename T> |
|
29 | 30 | ScalarTimeSerie* make_scalar(T& t, T& y) |
|
30 | 31 | { |
|
31 | 32 | return new ScalarTimeSerie { std::move(t.data), std::move(y.data) }; |
|
32 | 33 | } |
|
33 | 34 | |
|
34 | 35 | template <typename T> |
|
35 | 36 | VectorTimeSerie* make_vector(T& t, T& y) |
|
36 | 37 | { |
|
37 | 38 | return new VectorTimeSerie { std::move(t.data), y.to_std_vect_vect() }; |
|
38 | 39 | } |
|
39 | 40 | |
|
40 | 41 | template <typename T> |
|
41 | 42 | MultiComponentTimeSerie* make_multi_comp(T& t, T& y) |
|
42 | 43 | { |
|
43 | 44 | auto y_size = y.flat_size(); |
|
44 | 45 | auto t_size = t.flat_size(); |
|
45 | 46 | if (t_size && (y_size % t_size) == 0) |
|
46 | 47 | { |
|
47 | 48 | return new MultiComponentTimeSerie { std::move(t.data), std::move(y.data), |
|
48 | 49 | { t_size, y_size / t_size } }; |
|
49 | 50 | } |
|
50 | 51 | return nullptr; |
|
51 | 52 | } |
|
52 | 53 | |
|
53 | 54 | template <typename T> |
|
54 | SpectrogramTimeSerie* make_spectro(T& t, T& y) | |
|
55 | SpectrogramTimeSerie* make_spectro(T& t, T& y, T& z) | |
|
55 | 56 | { |
|
56 |
auto |
|
|
57 | auto z_size = z.flat_size(); | |
|
57 | 58 | auto t_size = t.flat_size(); |
|
58 |
if (t_size && ( |
|
|
59 | if (t_size && (z_size % t_size) == 0) | |
|
59 | 60 | { |
|
60 | return new SpectrogramTimeSerie { std::move(t.data), std::move(y.data), | |
|
61 |
{ t_size, |
|
|
61 | return new SpectrogramTimeSerie { std::move(t.data), std::move(y.data), std::move(z.data), | |
|
62 | { t_size, z_size / t_size }, std::nan("1"), std::nan("1") }; | |
|
62 | 63 | } |
|
63 | 64 | return nullptr; |
|
64 | 65 | } |
|
65 | 66 | |
|
66 | 67 | |
|
67 | 68 | class PyDataProvider : public IDataProvider |
|
68 | 69 | { |
|
69 | 70 | public: |
|
70 | 71 | PyDataProvider() |
|
71 | 72 | { |
|
72 | 73 | auto& dataSources = sqpApp->dataSources(); |
|
73 | 74 | dataSources.addProvider(this); |
|
74 | 75 | } |
|
75 | 76 | |
|
76 | 77 | virtual ~PyDataProvider() {} |
|
77 | 78 | |
|
78 |
virtual QPair<QPair<NpArray, |
|
|
79 | virtual QPair<QPair<QPair<NpArray,NpArray>,NpArray>, DataSeriesType> get_data( | |
|
79 | 80 | const QMap<QString, QString>& key, double start_time, double stop_time) |
|
80 | 81 | { |
|
81 | 82 | (void)key, (void)start_time, (void)stop_time; |
|
82 | 83 | return {}; |
|
83 | 84 | } |
|
84 | 85 | |
|
85 | 86 | virtual TimeSeries::ITimeSerie* getData(const DataProviderParameters& parameters) override |
|
86 | 87 | { |
|
87 | 88 | TimeSeries::ITimeSerie* ts = nullptr; |
|
88 | 89 | if (parameters.m_Data.contains("name")) |
|
89 | 90 | { |
|
90 | 91 | QMap<QString, QString> metadata; |
|
91 | 92 | std::for_each(parameters.m_Data.constKeyValueBegin(), |
|
92 | 93 | parameters.m_Data.constKeyValueEnd(), |
|
93 | 94 | [&metadata](const auto& item) { metadata[item.first] = item.second.toString(); }); |
|
94 | 95 | auto [data, type] |
|
95 | 96 | = get_data(metadata, parameters.m_Range.m_TStart, parameters.m_Range.m_TEnd); |
|
96 | 97 | |
|
97 |
auto& [ |
|
|
98 | auto& [axes, values] = data; | |
|
99 | auto& [x, y] = axes; | |
|
98 | 100 | switch (type) |
|
99 | 101 | { |
|
100 | 102 | case DataSeriesType::SCALAR: |
|
101 |
ts = make_scalar( |
|
|
103 | ts = make_scalar(x, values); | |
|
102 | 104 | break; |
|
103 | 105 | case DataSeriesType::VECTOR: |
|
104 |
ts = make_vector( |
|
|
106 | ts = make_vector(x, values); | |
|
105 | 107 | break; |
|
106 | 108 | case DataSeriesType::MULTICOMPONENT: |
|
107 |
ts = make_multi_comp( |
|
|
109 | ts = make_multi_comp(x, values); | |
|
108 | 110 | break; |
|
109 | 111 | case DataSeriesType::SPECTROGRAM: |
|
110 |
ts = make_spectro( |
|
|
112 | ts = make_spectro(x, y, values); | |
|
111 | 113 | break; |
|
112 | 114 | default: |
|
113 | 115 | break; |
|
114 | 116 | } |
|
115 | 117 | } |
|
116 | 118 | return ts; |
|
117 | 119 | } |
|
118 | 120 | |
|
119 | 121 | inline void set_icon(const QString& path, const QString& name) |
|
120 | 122 | { |
|
121 | 123 | sqpApp->dataSources().setIcon(path, name); |
|
122 | 124 | } |
|
123 | 125 | |
|
124 | 126 | inline void register_products(const QVector<Product*>& products) |
|
125 | 127 | { |
|
126 | 128 | auto& dataSources = sqpApp->dataSources(); |
|
127 | 129 | auto id = this->id(); |
|
128 | 130 | auto data_source_name = this->name(); |
|
129 | 131 | std::for_each(std::cbegin(products), std::cend(products), |
|
130 | 132 | [&id, &dataSources](const Product* product) { |
|
131 | 133 | dataSources.addDataSourceItem(id, product->path, product->metadata); |
|
132 | 134 | }); |
|
133 | 135 | } |
|
134 | 136 | }; |
@@ -1,124 +1,121 | |||
|
1 | 1 | import traceback |
|
2 | 2 | import os |
|
3 | 3 | from datetime import datetime, timedelta, timezone |
|
4 | 4 | from SciQLopBindings import PyDataProvider, Product, VectorTimeSerie, ScalarTimeSerie, DataSeriesType |
|
5 | 5 | import numpy as np |
|
6 | 6 | import requests |
|
7 | 7 | import copy |
|
8 | 8 | from spwc.amda import AMDA |
|
9 | 9 | |
|
10 | 10 | amda = AMDA() |
|
11 | 11 | |
|
12 | ||
|
12 | 13 | def amda_make_scalar(var=None): |
|
13 | 14 | if var is None: |
|
14 | return ((np.array(), np.array()), DataSeriesType.SCALAR) | |
|
15 | return (((np.array([]), np.array([])), np.array([])), DataSeriesType.SCALAR) | |
|
15 | 16 | else: |
|
16 | return ((var.time,var.data), DataSeriesType.SCALAR) | |
|
17 | return (((var.time, np.array([])), var.data), DataSeriesType.SCALAR) | |
|
18 | ||
|
17 | 19 | |
|
18 | 20 | def amda_make_vector(var=None): |
|
19 | 21 | if var is None: |
|
20 | return ((np.array(), np.array()), DataSeriesType.VECTOR) | |
|
22 | return (((np.array([]), np.array([])), np.array([])), DataSeriesType.VECTOR) | |
|
21 | 23 | else: |
|
22 | return ((var.time,var.data), DataSeriesType.VECTOR) | |
|
24 | return (((var.time, np.array([])), var.data), DataSeriesType.VECTOR) | |
|
25 | ||
|
23 | 26 | |
|
24 | 27 | def amda_make_multi_comp(var=None): |
|
25 | 28 | if var is None: |
|
26 | return ((np.array(), np.array()), DataSeriesType.MULTICOMPONENT) | |
|
29 | return (((np.array([]), np.array([])), np.array([])), DataSeriesType.MULTICOMPONENT) | |
|
27 | 30 | else: |
|
28 | return ((var.time,var.data), DataSeriesType.MULTICOMPONENT) | |
|
31 | return (((var.time, np.array([])), var.data), DataSeriesType.MULTICOMPONENT) | |
|
32 | ||
|
29 | 33 | |
|
30 | 34 | def amda_make_spectro(var=None): |
|
31 | 35 | if var is None: |
|
32 | return ((np.array(), np.array()), DataSeriesType.SPECTROGRAM) | |
|
36 | return (((np.array([]), np.array([])), np.array([])), DataSeriesType.SPECTROGRAM) | |
|
33 | 37 | else: |
|
34 | 38 | min_sampling = float(var.meta.get("DATASET_MIN_SAMPLING","nan")) |
|
35 | 39 | max_sampling = float(var.meta.get("DATASET_MAX_SAMPLING","nan")) |
|
36 | if "PARAMETER_TABLE_MIN_VALUES[1]" in var.meta: | |
|
37 | min_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MIN_VALUES[1]"].split(',') ]) | |
|
38 | max_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MAX_VALUES[1]"].split(',') ]) | |
|
39 | y = (max_v + min_v)/2. | |
|
40 | elif "PARAMETER_TABLE_MIN_VALUES[0]" in var.meta: | |
|
41 | min_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MIN_VALUES[0]"].split(',') ]) | |
|
42 | max_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MAX_VALUES[0]"].split(',') ]) | |
|
43 | y = (max_v + min_v)/2. | |
|
44 | else: | |
|
45 | y = np.logspace(1,3,var.data.shape[1])[::-1] | |
|
46 | return ((var.time,var.data), DataSeriesType.SPECTROGRAM) | |
|
40 | if var.y is None and len(var.data): | |
|
41 | var.y = np.logspace(1, 3, var.data.shape[1])[::-1] | |
|
42 | return (((var.time, var.y), var.data), DataSeriesType.SPECTROGRAM) | |
|
47 | 43 | #return pysciqlopcore.SpectrogramTimeSerie(var.time,y,var.data,min_sampling,max_sampling,True) |
|
48 | 44 | |
|
45 | ||
|
49 | 46 | def amda_get_sample(metadata,start,stop): |
|
50 | 47 | ts_type = amda_make_scalar |
|
51 | 48 | try: |
|
52 | 49 | param_id = None |
|
53 | 50 | for key,value in metadata: |
|
54 | 51 | if key == 'xml:id': |
|
55 | 52 | param_id = value |
|
56 | 53 | elif key == 'type': |
|
57 | 54 | if value == 'vector': |
|
58 | 55 | ts_type = amda_make_vector |
|
59 | 56 | elif value == 'multicomponent': |
|
60 | 57 | ts_type = amda_make_multi_comp |
|
61 | 58 | elif value == 'spectrogram': |
|
62 | 59 | ts_type = amda_make_spectro |
|
63 | 60 | tstart=datetime.fromtimestamp(start, tz=timezone.utc) |
|
64 | 61 | tend=datetime.fromtimestamp(stop, tz=timezone.utc) |
|
65 | 62 | var = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST") |
|
66 | 63 | return ts_type(var) |
|
67 | 64 | except Exception as e: |
|
68 | 65 | print(traceback.format_exc()) |
|
69 | 66 | print("Error in amda.py ",str(e)) |
|
70 | 67 | return ts_type() |
|
71 | 68 | |
|
72 | 69 | |
|
73 | 70 | class AmdaProvider(PyDataProvider): |
|
74 | 71 | def __init__(self): |
|
75 | 72 | super(AmdaProvider,self).__init__() |
|
76 | 73 | if len(amda.component) is 0: |
|
77 | 74 | amda.update_inventory() |
|
78 | 75 | parameters = copy.deepcopy(amda.parameter) |
|
79 | 76 | for name,component in amda.component.items(): |
|
80 | 77 | if 'components' in parameters[component['parameter']]: |
|
81 | 78 | parameters[component['parameter']]['components'].append(component) |
|
82 | 79 | else: |
|
83 | 80 | parameters[component['parameter']]['components']=[component] |
|
84 | 81 | |
|
85 | 82 | products = [] |
|
86 | 83 | for key,parameter in parameters.items(): |
|
87 | 84 | path = f"/AMDA/{parameter['mission']}/{parameter.get('observatory','')}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}" |
|
88 | 85 | components = [component['name'] for component in parameter.get('components',[])] |
|
89 | 86 |
metadata = { |
|
90 | 87 | n_components = parameter.get('size',0) |
|
91 | 88 | if n_components == '3': |
|
92 | 89 | metadata["type"]="vector" |
|
93 | 90 | elif parameter.get('display_type','')=="spectrogram": |
|
94 | 91 | metadata["type"]="spectrogram" |
|
95 | 92 | elif n_components !=0: |
|
96 | 93 | metadata["type"]="multicomponent" |
|
97 | 94 | else: |
|
98 | 95 | metadata["type"]="scalar" |
|
99 | 96 |
products.append( |
|
100 | 97 | self.register_products(products) |
|
101 | 98 | for mission in amda.mission: |
|
102 | 99 | self.set_icon(f'/AMDA/{mission}','satellite') |
|
103 | 100 | |
|
104 | 101 | def get_data(self,metadata,start,stop): |
|
105 | 102 | ts_type = amda_make_scalar |
|
106 | 103 | try: |
|
107 | 104 | param_id = metadata['xml:id'] |
|
108 | 105 | ts_type_str = metadata['type'] |
|
109 | 106 | if ts_type_str == 'vector': |
|
110 | 107 | ts_type = amda_make_vector |
|
111 | 108 | elif ts_type_str == 'multicomponent': |
|
112 | 109 | ts_type = amda_make_multi_comp |
|
113 | 110 | elif ts_type_str == 'spectrogram': |
|
114 | 111 | ts_type = amda_make_spectro |
|
115 | 112 | tstart=datetime.fromtimestamp(start, tz=timezone.utc) |
|
116 | 113 | tend=datetime.fromtimestamp(stop, tz=timezone.utc) |
|
117 | 114 | var = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST") |
|
118 | 115 | return ts_type(var) |
|
119 | 116 | except Exception as e: |
|
120 | 117 | print(traceback.format_exc()) |
|
121 | 118 | print("Error in amda.py ",str(e)) |
|
122 | 119 | return ts_type() |
|
123 | 120 | |
|
124 | 121 | _amda = AmdaProvider() |
@@ -1,93 +1,93 | |||
|
1 | 1 | import traceback |
|
2 | 2 | from SciQLopBindings import PyDataProvider, Product, VectorTimeSerie, ScalarTimeSerie, DataSeriesType |
|
3 | 3 | import numpy as np |
|
4 | 4 | import math |
|
5 | 5 | from spwc.cache import _cache |
|
6 | 6 | from spwc.common.datetime_range import DateTimeRange |
|
7 | 7 | from functools import partial |
|
8 | 8 | from datetime import datetime, timedelta, timezone |
|
9 | 9 | from spwc.common.variable import SpwcVariable |
|
10 | 10 | |
|
11 | 11 | |
|
12 | 12 | def make_scalar(x): |
|
13 | 13 | y = np.cos(x/10.) |
|
14 | 14 | return SpwcVariable(time=x, data=y) |
|
15 | 15 | |
|
16 | 16 | def make_vector(x): |
|
17 | 17 | v=np.ones((len(x),3)) |
|
18 | 18 | for i in range(3): |
|
19 | 19 | v.transpose()[:][i] = np.cos(x/10. + float(i)) + (100. * np.cos(x/10000. + float(i))) |
|
20 | 20 | return SpwcVariable(time=x, data=v) |
|
21 | 21 | |
|
22 | 22 | |
|
23 | 23 | def make_multicomponent(x): |
|
24 | 24 | v=np.ones((len(x),4)) |
|
25 | 25 | for i in range(4): |
|
26 | 26 | v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i)) |
|
27 | 27 | return SpwcVariable(time=x, data=v) |
|
28 | 28 | |
|
29 | 29 | def make_spectrogram(x): |
|
30 | 30 | v=np.ones((len(x),32)) |
|
31 | 31 | for i in range(32): |
|
32 | 32 | v.transpose()[:][i] = 100.*(2.+ float(i+1) * np.cos(x/1024. + float(i))) |
|
33 | 33 | return SpwcVariable(time=x, data=v) |
|
34 | 34 | |
|
35 | 35 | |
|
36 | 36 | def _get_data(p_type, start, stop): |
|
37 | 37 | if type(start) is datetime: |
|
38 | 38 | start = start.timestamp() |
|
39 | 39 | stop = stop.timestamp() |
|
40 | 40 | x = np.arange(math.ceil(start), math.floor(stop))*1. |
|
41 | 41 | if p_type == 'scalar': |
|
42 | 42 | return make_scalar(x) |
|
43 | 43 | if p_type == 'vector': |
|
44 | 44 | return make_vector(x) |
|
45 | 45 | if p_type == 'multicomponent': |
|
46 | 46 | return make_multicomponent(x) |
|
47 | 47 | if p_type == 'spectrogram': |
|
48 | 48 | return make_spectrogram(np.arange(math.ceil(start), math.floor(stop),15.)) |
|
49 | 49 | return None |
|
50 | 50 | |
|
51 | 51 | class MyProvider(PyDataProvider): |
|
52 | 52 | def __init__(self): |
|
53 | 53 | super(MyProvider,self).__init__() |
|
54 | 54 | self.register_products([Product("/tests/without_cache/scalar",[],{"type":"scalar"}), |
|
55 | 55 | Product("/tests/without_cache/vector",[],{"type":"vector"}), |
|
56 | 56 | Product("/tests/without_cache/multicomponent",[],{"type":"multicomponent",'size':'4'}), |
|
57 | 57 | Product("/tests/without_cache/spectrogram",[],{"type":"spectrogram",'size':'32'}), |
|
58 | 58 | Product("/tests/with_cache/scalar",[],{"type":"scalar", "cache":"true"}), |
|
59 | 59 | Product("/tests/with_cache/vector",[],{"type":"vector", "cache":"true"}), |
|
60 | 60 | Product("/tests/with_cache/multicomponent",[],{"type":"multicomponent",'size':'4', "cache":"true"}) |
|
61 | 61 | ]) |
|
62 | 62 | |
|
63 | 63 | def get_data(self,metadata,start,stop): |
|
64 | 64 | ts_type = DataSeriesType.SCALAR |
|
65 | 65 | default_ctor_args = 1 |
|
66 | 66 | use_cache = False |
|
67 | 67 | p_type = 'scalar' |
|
68 | 68 | try: |
|
69 | 69 | for key,value in metadata.items(): |
|
70 | 70 | if key == 'type': |
|
71 | 71 | p_type = value |
|
72 | 72 | if value == 'vector': |
|
73 | 73 | ts_type = DataSeriesType.VECTOR |
|
74 | 74 | elif value == 'multicomponent': |
|
75 | 75 | ts_type = DataSeriesType.MULTICOMPONENT |
|
76 | 76 | elif value == 'spectrogram': |
|
77 | 77 | ts_type = DataSeriesType.SPECTROGRAM |
|
78 | 78 | if key == 'cache' and value == 'true': |
|
79 | 79 | use_cache = True |
|
80 | 80 | if use_cache: |
|
81 | 81 | cache_product = f"tests/{p_type}" |
|
82 | 82 | var = _cache.get_data(cache_product, DateTimeRange(datetime.fromtimestamp(start, tz=timezone.utc), datetime.fromtimestamp(stop, tz=timezone.utc)), partial(_get_data, p_type), fragment_hours=24) |
|
83 | 83 | else: |
|
84 | 84 | var = _get_data(p_type, start, stop) |
|
85 | return ((var.time,var.data), ts_type) | |
|
85 | return (((var.time, np.array([])),var.data), ts_type) | |
|
86 | 86 | except Exception as e: |
|
87 | 87 | print(traceback.format_exc()) |
|
88 | 88 | print("Error in test.py ",str(e)) |
|
89 | return ((np.array(), np.array()), ts_type) | |
|
89 | return (((np.array([]), np.array([])), np.array([])), ts_type) | |
|
90 | 90 | |
|
91 | 91 | |
|
92 | 92 | t=MyProvider() |
|
93 | 93 |
General Comments 0
You need to be logged in to leave comments.
Login now