##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Spectrograms works again, needs more polish......
jeandet -
r1498:d1a5badbcf0e
parent child
Show More
Side by Side Unified
Context file:
r1498:d1a5badbcf0e Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,134 +1,136
1 1 #pragma once
2 2 #include <Data/DataProviderParameters.h>
3 3 #include <Data/DataSeriesType.h>
4 4 #include <Data/IDataProvider.h>
5 5 #include <DataSource/DataSourceItem.h>
6 6 #include <DataSource/DataSourceItemAction.h>
7 7 #include <DataSource/datasources.h>
8 8
9 9 #include <QPair>
10 #include <QList>
10 11 #include <SqpApplication.h>
11 12 // must be included last because of Python/Qt definition of slots
12 13 #include "numpy_wrappers.h"
13 14
14 15 struct Product
15 16 {
16 17 QString path;
17 18 std::vector<std::string> components;
18 19 QMap<QString, QString> metadata;
19 20 Product() = default;
20 21 explicit Product(const QString& path, const std::vector<std::string>& components,
21 22 const QMap<QString, QString>& metadata)
22 23 : path { path }, components { components }, metadata { metadata }
23 24 {
24 25 }
25 26 ~Product() = default;
26 27 };
27 28
28 29 template <typename T>
29 30 ScalarTimeSerie* make_scalar(T& t, T& y)
30 31 {
31 32 return new ScalarTimeSerie { std::move(t.data), std::move(y.data) };
32 33 }
33 34
34 35 template <typename T>
35 36 VectorTimeSerie* make_vector(T& t, T& y)
36 37 {
37 38 return new VectorTimeSerie { std::move(t.data), y.to_std_vect_vect() };
38 39 }
39 40
40 41 template <typename T>
41 42 MultiComponentTimeSerie* make_multi_comp(T& t, T& y)
42 43 {
43 44 auto y_size = y.flat_size();
44 45 auto t_size = t.flat_size();
45 46 if (t_size && (y_size % t_size) == 0)
46 47 {
47 48 return new MultiComponentTimeSerie { std::move(t.data), std::move(y.data),
48 49 { t_size, y_size / t_size } };
49 50 }
50 51 return nullptr;
51 52 }
52 53
53 54 template <typename T>
54 SpectrogramTimeSerie* make_spectro(T& t, T& y)
55 SpectrogramTimeSerie* make_spectro(T& t, T& y, T& z)
55 56 {
56 auto y_size = y.flat_size();
57 auto z_size = z.flat_size();
57 58 auto t_size = t.flat_size();
58 if (t_size && (y_size % t_size) == 0)
59 if (t_size && (z_size % t_size) == 0)
59 60 {
60 return new SpectrogramTimeSerie { std::move(t.data), std::move(y.data),
61 { t_size, y_size / t_size } };
61 return new SpectrogramTimeSerie { std::move(t.data), std::move(y.data), std::move(z.data),
62 { t_size, z_size / t_size }, std::nan("1"), std::nan("1") };
62 63 }
63 64 return nullptr;
64 65 }
65 66
66 67
67 68 class PyDataProvider : public IDataProvider
68 69 {
69 70 public:
70 71 PyDataProvider()
71 72 {
72 73 auto& dataSources = sqpApp->dataSources();
73 74 dataSources.addProvider(this);
74 75 }
75 76
76 77 virtual ~PyDataProvider() {}
77 78
78 virtual QPair<QPair<NpArray, NpArray>, DataSeriesType> get_data(
79 virtual QPair<QPair<QPair<NpArray,NpArray>,NpArray>, DataSeriesType> get_data(
79 80 const QMap<QString, QString>& key, double start_time, double stop_time)
80 81 {
81 82 (void)key, (void)start_time, (void)stop_time;
82 83 return {};
83 84 }
84 85
85 86 virtual TimeSeries::ITimeSerie* getData(const DataProviderParameters& parameters) override
86 87 {
87 88 TimeSeries::ITimeSerie* ts = nullptr;
88 89 if (parameters.m_Data.contains("name"))
89 90 {
90 91 QMap<QString, QString> metadata;
91 92 std::for_each(parameters.m_Data.constKeyValueBegin(),
92 93 parameters.m_Data.constKeyValueEnd(),
93 94 [&metadata](const auto& item) { metadata[item.first] = item.second.toString(); });
94 95 auto [data, type]
95 96 = get_data(metadata, parameters.m_Range.m_TStart, parameters.m_Range.m_TEnd);
96 97
97 auto& [t, y] = data;
98 auto& [axes, values] = data;
99 auto& [x, y] = axes;
98 100 switch (type)
99 101 {
100 102 case DataSeriesType::SCALAR:
101 ts = make_scalar(t, y);
103 ts = make_scalar(x, values);
102 104 break;
103 105 case DataSeriesType::VECTOR:
104 ts = make_vector(t, y);
106 ts = make_vector(x, values);
105 107 break;
106 108 case DataSeriesType::MULTICOMPONENT:
107 ts = make_multi_comp(t, y);
109 ts = make_multi_comp(x, values);
108 110 break;
109 111 case DataSeriesType::SPECTROGRAM:
110 ts = make_spectro(t, y);
112 ts = make_spectro(x, y, values);
111 113 break;
112 114 default:
113 115 break;
114 116 }
115 117 }
116 118 return ts;
117 119 }
118 120
119 121 inline void set_icon(const QString& path, const QString& name)
120 122 {
121 123 sqpApp->dataSources().setIcon(path, name);
122 124 }
123 125
124 126 inline void register_products(const QVector<Product*>& products)
125 127 {
126 128 auto& dataSources = sqpApp->dataSources();
127 129 auto id = this->id();
128 130 auto data_source_name = this->name();
129 131 std::for_each(std::cbegin(products), std::cend(products),
130 132 [&id, &dataSources](const Product* product) {
131 133 dataSources.addDataSourceItem(id, product->path, product->metadata);
132 134 });
133 135 }
134 136 };
Show file before | Show file after | Hide comments
@@ -1,124 +1,121
1 1 import traceback
2 2 import os
3 3 from datetime import datetime, timedelta, timezone
4 4 from SciQLopBindings import PyDataProvider, Product, VectorTimeSerie, ScalarTimeSerie, DataSeriesType
5 5 import numpy as np
6 6 import requests
7 7 import copy
8 8 from spwc.amda import AMDA
9 9
10 10 amda = AMDA()
11 11
12
12 13 def amda_make_scalar(var=None):
13 14 if var is None:
14 return ((np.array(), np.array()), DataSeriesType.SCALAR)
15 return (((np.array([]), np.array([])), np.array([])), DataSeriesType.SCALAR)
15 16 else:
16 return ((var.time,var.data), DataSeriesType.SCALAR)
17 return (((var.time, np.array([])), var.data), DataSeriesType.SCALAR)
18
17 19
18 20 def amda_make_vector(var=None):
19 21 if var is None:
20 return ((np.array(), np.array()), DataSeriesType.VECTOR)
22 return (((np.array([]), np.array([])), np.array([])), DataSeriesType.VECTOR)
21 23 else:
22 return ((var.time,var.data), DataSeriesType.VECTOR)
24 return (((var.time, np.array([])), var.data), DataSeriesType.VECTOR)
25
23 26
24 27 def amda_make_multi_comp(var=None):
25 28 if var is None:
26 return ((np.array(), np.array()), DataSeriesType.MULTICOMPONENT)
29 return (((np.array([]), np.array([])), np.array([])), DataSeriesType.MULTICOMPONENT)
27 30 else:
28 return ((var.time,var.data), DataSeriesType.MULTICOMPONENT)
31 return (((var.time, np.array([])), var.data), DataSeriesType.MULTICOMPONENT)
32
29 33
30 34 def amda_make_spectro(var=None):
31 35 if var is None:
32 return ((np.array(), np.array()), DataSeriesType.SPECTROGRAM)
36 return (((np.array([]), np.array([])), np.array([])), DataSeriesType.SPECTROGRAM)
33 37 else:
34 min_sampling = float(var.meta.get("DATASET_MIN_SAMPLING","nan"))
35 max_sampling = float(var.meta.get("DATASET_MAX_SAMPLING","nan"))
36 if "PARAMETER_TABLE_MIN_VALUES[1]" in var.meta:
37 min_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MIN_VALUES[1]"].split(',') ])
38 max_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MAX_VALUES[1]"].split(',') ])
39 y = (max_v + min_v)/2.
40 elif "PARAMETER_TABLE_MIN_VALUES[0]" in var.meta:
41 min_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MIN_VALUES[0]"].split(',') ])
42 max_v = np.array([ float(v) for v in var.meta["PARAMETER_TABLE_MAX_VALUES[0]"].split(',') ])
43 y = (max_v + min_v)/2.
44 else:
45 y = np.logspace(1,3,var.data.shape[1])[::-1]
46 return ((var.time,var.data), DataSeriesType.SPECTROGRAM)
38 min_sampling = float(var.meta.get("DATASET_MIN_SAMPLING", "nan"))
39 max_sampling = float(var.meta.get("DATASET_MAX_SAMPLING", "nan"))
40 if var.y is None and len(var.data):
41 var.y = np.logspace(1, 3, var.data.shape[1])[::-1]
42 return (((var.time, var.y), var.data), DataSeriesType.SPECTROGRAM)
47 43 #return pysciqlopcore.SpectrogramTimeSerie(var.time,y,var.data,min_sampling,max_sampling,True)
48 44
49 def amda_get_sample(metadata,start,stop):
45
46 def amda_get_sample(metadata, start, stop):
50 47 ts_type = amda_make_scalar
51 48 try:
52 49 param_id = None
53 for key,value in metadata:
50 for key, value in metadata:
54 51 if key == 'xml:id':
55 52 param_id = value
56 53 elif key == 'type':
57 54 if value == 'vector':
58 55 ts_type = amda_make_vector
59 56 elif value == 'multicomponent':
60 57 ts_type = amda_make_multi_comp
61 58 elif value == 'spectrogram':
62 59 ts_type = amda_make_spectro
63 tstart=datetime.fromtimestamp(start, tz=timezone.utc)
64 tend=datetime.fromtimestamp(stop, tz=timezone.utc)
60 tstart = datetime.fromtimestamp(start, tz=timezone.utc)
61 tend = datetime.fromtimestamp(stop, tz=timezone.utc)
65 62 var = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST")
66 63 return ts_type(var)
67 64 except Exception as e:
68 65 print(traceback.format_exc())
69 print("Error in amda.py ",str(e))
66 print("Error in amda.py ", str(e))
70 67 return ts_type()
71 68
72 69
73 70 class AmdaProvider(PyDataProvider):
74 71 def __init__(self):
75 super(AmdaProvider,self).__init__()
72 super(AmdaProvider, self).__init__()
76 73 if len(amda.component) is 0:
77 74 amda.update_inventory()
78 75 parameters = copy.deepcopy(amda.parameter)
79 for name,component in amda.component.items():
76 for name, component in amda.component.items():
80 77 if 'components' in parameters[component['parameter']]:
81 78 parameters[component['parameter']]['components'].append(component)
82 79 else:
83 80 parameters[component['parameter']]['components']=[component]
84 81
85 82 products = []
86 for key,parameter in parameters.items():
83 for key, parameter in parameters.items():
87 84 path = f"/AMDA/{parameter['mission']}/{parameter.get('observatory','')}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
88 85 components = [component['name'] for component in parameter.get('components',[])]
89 metadata = { key:item for key,item in parameter.items() if key is not 'components' }
90 n_components = parameter.get('size',0)
86 metadata = {key: item for key, item in parameter.items() if key is not 'components'}
87 n_components = parameter.get('size', 0)
91 88 if n_components == '3':
92 metadata["type"]="vector"
93 elif parameter.get('display_type','')=="spectrogram":
94 metadata["type"]="spectrogram"
95 elif n_components !=0:
96 metadata["type"]="multicomponent"
89 metadata["type"] = "vector"
90 elif parameter.get('display_type', '')=="spectrogram":
91 metadata["type"] = "spectrogram"
92 elif n_components != 0:
93 metadata["type"] = "multicomponent"
97 94 else:
98 metadata["type"]="scalar"
99 products.append( Product(path, components, metadata))
95 metadata["type"] = "scalar"
96 products.append(Product(path, components, metadata))
100 97 self.register_products(products)
101 98 for mission in amda.mission:
102 99 self.set_icon(f'/AMDA/{mission}','satellite')
103 100
104 def get_data(self,metadata,start,stop):
101 def get_data(self, metadata, start, stop):
105 102 ts_type = amda_make_scalar
106 103 try:
107 104 param_id = metadata['xml:id']
108 105 ts_type_str = metadata['type']
109 106 if ts_type_str == 'vector':
110 107 ts_type = amda_make_vector
111 108 elif ts_type_str == 'multicomponent':
112 109 ts_type = amda_make_multi_comp
113 110 elif ts_type_str == 'spectrogram':
114 111 ts_type = amda_make_spectro
115 tstart=datetime.fromtimestamp(start, tz=timezone.utc)
116 tend=datetime.fromtimestamp(stop, tz=timezone.utc)
112 tstart = datetime.fromtimestamp(start, tz=timezone.utc)
113 tend = datetime.fromtimestamp(stop, tz=timezone.utc)
117 114 var = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST")
118 115 return ts_type(var)
119 116 except Exception as e:
120 117 print(traceback.format_exc())
121 print("Error in amda.py ",str(e))
118 print("Error in amda.py ", str(e))
122 119 return ts_type()
123 120
124 121 _amda = AmdaProvider()
Show file before | Show file after | Hide comments
@@ -1,93 +1,93
1 1 import traceback
2 2 from SciQLopBindings import PyDataProvider, Product, VectorTimeSerie, ScalarTimeSerie, DataSeriesType
3 3 import numpy as np
4 4 import math
5 5 from spwc.cache import _cache
6 6 from spwc.common.datetime_range import DateTimeRange
7 7 from functools import partial
8 8 from datetime import datetime, timedelta, timezone
9 9 from spwc.common.variable import SpwcVariable
10 10
11 11
12 12 def make_scalar(x):
13 13 y = np.cos(x/10.)
14 14 return SpwcVariable(time=x, data=y)
15 15
16 16 def make_vector(x):
17 17 v=np.ones((len(x),3))
18 18 for i in range(3):
19 19 v.transpose()[:][i] = np.cos(x/10. + float(i)) + (100. * np.cos(x/10000. + float(i)))
20 20 return SpwcVariable(time=x, data=v)
21 21
22 22
23 23 def make_multicomponent(x):
24 24 v=np.ones((len(x),4))
25 25 for i in range(4):
26 26 v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i))
27 27 return SpwcVariable(time=x, data=v)
28 28
29 29 def make_spectrogram(x):
30 30 v=np.ones((len(x),32))
31 31 for i in range(32):
32 32 v.transpose()[:][i] = 100.*(2.+ float(i+1) * np.cos(x/1024. + float(i)))
33 33 return SpwcVariable(time=x, data=v)
34 34
35 35
36 36 def _get_data(p_type, start, stop):
37 37 if type(start) is datetime:
38 38 start = start.timestamp()
39 39 stop = stop.timestamp()
40 40 x = np.arange(math.ceil(start), math.floor(stop))*1.
41 41 if p_type == 'scalar':
42 42 return make_scalar(x)
43 43 if p_type == 'vector':
44 44 return make_vector(x)
45 45 if p_type == 'multicomponent':
46 46 return make_multicomponent(x)
47 47 if p_type == 'spectrogram':
48 48 return make_spectrogram(np.arange(math.ceil(start), math.floor(stop),15.))
49 49 return None
50 50
51 51 class MyProvider(PyDataProvider):
52 52 def __init__(self):
53 53 super(MyProvider,self).__init__()
54 54 self.register_products([Product("/tests/without_cache/scalar",[],{"type":"scalar"}),
55 55 Product("/tests/without_cache/vector",[],{"type":"vector"}),
56 56 Product("/tests/without_cache/multicomponent",[],{"type":"multicomponent",'size':'4'}),
57 57 Product("/tests/without_cache/spectrogram",[],{"type":"spectrogram",'size':'32'}),
58 58 Product("/tests/with_cache/scalar",[],{"type":"scalar", "cache":"true"}),
59 59 Product("/tests/with_cache/vector",[],{"type":"vector", "cache":"true"}),
60 60 Product("/tests/with_cache/multicomponent",[],{"type":"multicomponent",'size':'4', "cache":"true"})
61 61 ])
62 62
63 63 def get_data(self,metadata,start,stop):
64 64 ts_type = DataSeriesType.SCALAR
65 65 default_ctor_args = 1
66 66 use_cache = False
67 67 p_type = 'scalar'
68 68 try:
69 69 for key,value in metadata.items():
70 70 if key == 'type':
71 71 p_type = value
72 72 if value == 'vector':
73 73 ts_type = DataSeriesType.VECTOR
74 74 elif value == 'multicomponent':
75 75 ts_type = DataSeriesType.MULTICOMPONENT
76 76 elif value == 'spectrogram':
77 77 ts_type = DataSeriesType.SPECTROGRAM
78 78 if key == 'cache' and value == 'true':
79 79 use_cache = True
80 80 if use_cache:
81 81 cache_product = f"tests/{p_type}"
82 82 var = _cache.get_data(cache_product, DateTimeRange(datetime.fromtimestamp(start, tz=timezone.utc), datetime.fromtimestamp(stop, tz=timezone.utc)), partial(_get_data, p_type), fragment_hours=24)
83 83 else:
84 84 var = _get_data(p_type, start, stop)
85 return ((var.time,var.data), ts_type)
85 return (((var.time, np.array([])),var.data), ts_type)
86 86 except Exception as e:
87 87 print(traceback.format_exc())
88 88 print("Error in test.py ",str(e))
89 return ((np.array(), np.array()), ts_type)
89 return (((np.array([]), np.array([])), np.array([])), ts_type)
90 90
91 91
92 92 t=MyProvider()
93 93
Show file before | Show file after | Hide comments
@@ -1,1 +1,1
1 Subproject commit 9f4e10a342eece28498d9b732bf59f8d1908b716
1 Subproject commit 3bce297cb13e8337d71adceae6737c104c66ad52
General Comments 0
You need to be logged in to leave comments. Login now