@@ -1,67 +1,67 | |||
|
1 | 1 | import sys |
|
2 | 2 | sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core") |
|
3 | 3 | import traceback |
|
4 | 4 | import os |
|
5 | 5 | from datetime import datetime, timedelta, timezone |
|
6 | 6 | import PythonProviders |
|
7 | 7 | import pysciqlopcore |
|
8 | 8 | import numpy as np |
|
9 | 9 | import pandas as pds |
|
10 | 10 | import requests |
|
11 | 11 | import copy |
|
12 | 12 | from spwc.amda import AMDA |
|
13 | 13 | |
|
14 | 14 | amda = AMDA() |
|
15 | 15 | |
|
16 | 16 | def get_sample(metadata,start,stop): |
|
17 | 17 | ts_type = pysciqlopcore.ScalarTimeSerie |
|
18 | 18 | default_ctor_args = 1 |
|
19 | 19 | try: |
|
20 | 20 | param_id = None |
|
21 | 21 | for key,value in metadata: |
|
22 | 22 | if key == 'xml:id': |
|
23 | 23 | param_id = value |
|
24 | 24 | elif key == 'type': |
|
25 | 25 | if value == 'vector': |
|
26 | 26 | ts_type = pysciqlopcore.VectorTimeSerie |
|
27 | 27 | elif value == 'multicomponent': |
|
28 | 28 | ts_type = pysciqlopcore.MultiComponentTimeSerie |
|
29 | 29 | default_ctor_args = (0,2) |
|
30 |
tstart= |
|
|
31 |
tend= |
|
|
30 | tstart=datetime.fromtimestamp(start, tz=timezone.utc) | |
|
31 | tend=datetime.fromtimestamp(stop, tz=timezone.utc) | |
|
32 | 32 | df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST") |
|
33 | 33 | t = np.array([d.timestamp() for d in df.index]) |
|
34 | 34 | values = df.values |
|
35 | 35 | return ts_type(t,values) |
|
36 | 36 | except Exception as e: |
|
37 | 37 | print(traceback.format_exc()) |
|
38 | 38 | print("Error in amda.py ",str(e)) |
|
39 | 39 | return ts_type(default_ctor_args) |
|
40 | 40 | |
|
41 | 41 | |
|
42 | 42 | if len(amda.component) is 0: |
|
43 | 43 | amda.update_inventory() |
|
44 | 44 | parameters = copy.deepcopy(amda.parameter) |
|
45 | 45 | for name,component in amda.component.items(): |
|
46 | 46 | if 'components' in parameters[component['parameter']]: |
|
47 | 47 | parameters[component['parameter']]['components'].append(component) |
|
48 | 48 | else: |
|
49 | 49 | parameters[component['parameter']]['components']=[component] |
|
50 | 50 | |
|
51 | 51 | products = [] |
|
52 | 52 | for key,parameter in parameters.items(): |
|
53 |
path = f"/AMDA/{parameter['mission']}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}" |
|
|
53 | path = f"/AMDA/{parameter['mission']}/{parameter.get('observatory','')}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}" | |
|
54 | 54 | components = [component['name'] for component in parameter.get('components',[])] |
|
55 | 55 | metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ] |
|
56 | 56 | n_components = parameter.get('size',0) |
|
57 | 57 | if n_components is '3': |
|
58 | 58 | metadata.append(("type","vector")) |
|
59 | 59 | elif n_components !=0: |
|
60 | 60 | metadata.append(("type","multicomponent")) |
|
61 | 61 | else: |
|
62 | 62 | metadata.append(("type","scalar")) |
|
63 | 63 | products.append( (path, components, metadata)) |
|
64 | 64 | |
|
65 | 65 | PythonProviders.register_product(products, get_sample) |
|
66 | 66 | |
|
67 | 67 |
@@ -1,45 +1,87 | |||
|
1 | 1 | import sys |
|
2 | 2 | sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core") |
|
3 | 3 | import PythonProviders |
|
4 | 4 | import pysciqlopcore |
|
5 | 5 | import numpy as np |
|
6 | 6 | import math |
|
7 | from spwc.cache import _cache | |
|
8 | from spwc.common.datetime_range import DateTimeRange | |
|
9 | from functools import partial | |
|
10 | from datetime import datetime, timedelta, timezone | |
|
7 | 11 | |
|
8 | 12 | someglobal = 1 |
|
9 | 13 | |
|
10 | 14 | def make_scalar(x): |
|
11 | 15 | y = np.cos(x/10.) |
|
12 | return pysciqlopcore.ScalarTimeSerie(x,y) | |
|
16 | return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=y) | |
|
13 | 17 | |
|
14 | 18 | def make_vector(x): |
|
15 | 19 | v=np.ones((len(x),3)) |
|
16 | 20 | for i in range(3): |
|
17 | v.transpose()[:][i] = np.cos(x/10. + float(i)) | |
|
18 | return pysciqlopcore.VectorTimeSerie(x,v) | |
|
21 | v.transpose()[:][i] = np.cos(x/10. + float(i)) + (100. * np.cos(x/10000. + float(i))) | |
|
22 | return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=v) | |
|
19 | 23 | |
|
20 | 24 | |
|
21 | 25 | def make_multicomponent(x): |
|
22 | 26 | v=np.ones((len(x),4)) |
|
23 | 27 | for i in range(4): |
|
24 | 28 | v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i)) |
|
25 | return pysciqlopcore.MultiComponentTimeSerie(x,v) | |
|
29 | return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=v) | |
|
26 | 30 | |
|
27 | 31 | |
|
28 |
def get_data( |
|
|
32 | def _get_data(p_type, start, stop): | |
|
33 | if type(start) is datetime: | |
|
34 | start = start.timestamp() | |
|
35 | stop = stop.timestamp() | |
|
29 | 36 | x = np.arange(math.ceil(start), math.floor(stop)) |
|
30 | for key,value in metadata: | |
|
31 | if key == 'xml:id': | |
|
32 | param_id = value | |
|
33 | elif key == 'type': | |
|
34 | if value == 'vector': | |
|
37 | if p_type == 'scalar': | |
|
38 | return make_scalar(x) | |
|
39 | if p_type == 'vector': | |
|
35 | 40 |
|
|
36 |
|
|
|
41 | if p_type == 'multicomponent': | |
|
37 | 42 |
|
|
38 |
return |
|
|
39 | ||
|
40 | ||
|
43 | return None | |
|
41 | 44 | |
|
45 | def get_data(metadata,start,stop): | |
|
46 | ts_type = pysciqlopcore.ScalarTimeSerie | |
|
47 | default_ctor_args = 1 | |
|
48 | use_cache = False | |
|
49 | p_type = 'scalar' | |
|
50 | try: | |
|
51 | for key,value in metadata: | |
|
52 | if key == 'type': | |
|
53 | p_type = value | |
|
54 | if value == 'vector': | |
|
55 | ts_type = pysciqlopcore.VectorTimeSerie | |
|
56 | elif value == 'multicomponent': | |
|
57 | ts_type = pysciqlopcore.MultiComponentTimeSerie | |
|
58 | default_ctor_args = (0,2) | |
|
59 | if key == 'cache' and value == 'true': | |
|
60 | use_cache = True | |
|
61 | if use_cache: | |
|
62 | cache_product = f"tests/{p_type}" | |
|
63 | df = _cache.get_data(cache_product, DateTimeRange(datetime.fromtimestamp(start, tz=timezone.utc), datetime.fromtimestamp(stop, tz=timezone.utc)), | |
|
64 | partial(_get_data, p_type), | |
|
65 | fragment_hours=24) | |
|
66 | else: | |
|
67 | print("No Cache") | |
|
68 | df = _get_data(p_type, start, stop) | |
|
69 | t = np.array([d.timestamp() for d in df.index]) | |
|
70 | values = df.values | |
|
71 | return ts_type(t,values) | |
|
72 | except Exception as e: | |
|
73 | print(traceback.format_exc()) | |
|
74 | print("Error in test.py ",str(e)) | |
|
75 | return ts_type(default_ctor_args) | |
|
42 | 76 | |
|
43 | PythonProviders.register_product([("/tests/scalar",[],[("type","scalar")]), ("/tests/vector",[],[("type","vector")]), ("/tests/multicomponent",[],[("type","multicomponent"),('size','4')])],get_data) | |
|
77 | products = [ | |
|
78 | ("/tests/without_cache/scalar",[],[("type","scalar")]), | |
|
79 | ("/tests/without_cache/vector",[],[("type","vector")]), | |
|
80 | ("/tests/without_cache/multicomponent",[],[("type","multicomponent"),('size','4')]), | |
|
81 | ("/tests/with_cache/scalar",[],[("type","scalar"), ("cache","true")]), | |
|
82 | ("/tests/with_cache/vector",[],[("type","vector"), ("cache","true")]), | |
|
83 | ("/tests/with_cache/multicomponent",[],[("type","multicomponent"),('size','4'), ("cache","true")]) | |
|
84 | ] | |
|
44 | 85 | |
|
45 | 86 | |
|
87 | PythonProviders.register_product(products ,get_data) |
General Comments 0
You need to be logged in to leave comments.
Login now