##// END OF EJS Templates
HG_REPOSITORIES » LPP » SciQLOP_Repos » SciQLop
RSS
Better AMDA tree and test data cat also go through cache...
jeandet -
r1439:449d899d9309
parent child
Show More
Side by Side Unified
Context file:
r1439:449d899d9309 Loading diff...:
Show file before | Show file after | Hide comments
@@ -27,8 +27,8 def get_sample(metadata,start,stop):
27 elif value == 'multicomponent':
27 elif value == 'multicomponent':
28 ts_type = pysciqlopcore.MultiComponentTimeSerie
28 ts_type = pysciqlopcore.MultiComponentTimeSerie
29 default_ctor_args = (0,2)
29 default_ctor_args = (0,2)
30 tstart=datetime.datetime.fromtimestamp(start, tz=timezone.utc)
30 tstart=datetime.fromtimestamp(start, tz=timezone.utc)
31 tend=datetime.datetime.fromtimestamp(stop, tz=timezone.utc)
31 tend=datetime.fromtimestamp(stop, tz=timezone.utc)
32 df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST")
32 df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST")
33 t = np.array([d.timestamp() for d in df.index])
33 t = np.array([d.timestamp() for d in df.index])
34 values = df.values
34 values = df.values
@@ -50,7 +50,7 for name,component in amda.component.items():
50
50
51 products = []
51 products = []
52 for key,parameter in parameters.items():
52 for key,parameter in parameters.items():
53 path = f"/AMDA/{parameter['mission']}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
53 path = f"/AMDA/{parameter['mission']}/{parameter.get('observatory','')}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
54 components = [component['name'] for component in parameter.get('components',[])]
54 components = [component['name'] for component in parameter.get('components',[])]
55 metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ]
55 metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ]
56 n_components = parameter.get('size',0)
56 n_components = parameter.get('size',0)
Show file before | Show file after | Hide comments
@@ -4,42 +4,84 import PythonProviders
4 import pysciqlopcore
4 import pysciqlopcore
5 import numpy as np
5 import numpy as np
6 import math
6 import math
7 from spwc.cache import _cache
8 from spwc.common.datetime_range import DateTimeRange
9 from functools import partial
10 from datetime import datetime, timedelta, timezone
7
11
8 someglobal = 1
12 someglobal = 1
9
13
10 def make_scalar(x):
14 def make_scalar(x):
11 y = np.cos(x/10.)
15 y = np.cos(x/10.)
12 return pysciqlopcore.ScalarTimeSerie(x,y)
16 return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=y)
13
17
14 def make_vector(x):
18 def make_vector(x):
15 v=np.ones((len(x),3))
19 v=np.ones((len(x),3))
16 for i in range(3):
20 for i in range(3):
17 v.transpose()[:][i] = np.cos(x/10. + float(i))
21 v.transpose()[:][i] = np.cos(x/10. + float(i)) + (100. * np.cos(x/10000. + float(i)))
18 return pysciqlopcore.VectorTimeSerie(x,v)
22 return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=v)
19
23
20
24
21 def make_multicomponent(x):
25 def make_multicomponent(x):
22 v=np.ones((len(x),4))
26 v=np.ones((len(x),4))
23 for i in range(4):
27 for i in range(4):
24 v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i))
28 v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i))
25 return pysciqlopcore.MultiComponentTimeSerie(x,v)
29 return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=v)
26
30
27
31
28 def get_data(metadata,start,stop):
32 def _get_data(p_type, start, stop):
33 if type(start) is datetime:
34 start = start.timestamp()
35 stop = stop.timestamp()
29 x = np.arange(math.ceil(start), math.floor(stop))
36 x = np.arange(math.ceil(start), math.floor(stop))
30 for key,value in metadata:
37 if p_type == 'scalar':
31 if key == 'xml:id':
38 return make_scalar(x)
32 param_id = value
39 if p_type == 'vector':
33 elif key == 'type':
34 if value == 'vector':
35 return make_vector(x)
40 return make_vector(x)
36 elif value == 'multicomponent':
41 if p_type == 'multicomponent':
37 return make_multicomponent(x)
42 return make_multicomponent(x)
38 return make_scalar(x)
43 return None
39
40
41
44
45 def get_data(metadata,start,stop):
46 ts_type = pysciqlopcore.ScalarTimeSerie
47 default_ctor_args = 1
48 use_cache = False
49 p_type = 'scalar'
50 try:
51 for key,value in metadata:
52 if key == 'type':
53 p_type = value
54 if value == 'vector':
55 ts_type = pysciqlopcore.VectorTimeSerie
56 elif value == 'multicomponent':
57 ts_type = pysciqlopcore.MultiComponentTimeSerie
58 default_ctor_args = (0,2)
59 if key == 'cache' and value == 'true':
60 use_cache = True
61 if use_cache:
62 cache_product = f"tests/{p_type}"
63 df = _cache.get_data(cache_product, DateTimeRange(datetime.fromtimestamp(start, tz=timezone.utc), datetime.fromtimestamp(stop, tz=timezone.utc)),
64 partial(_get_data, p_type),
65 fragment_hours=24)
66 else:
67 print("No Cache")
68 df = _get_data(p_type, start, stop)
69 t = np.array([d.timestamp() for d in df.index])
70 values = df.values
71 return ts_type(t,values)
72 except Exception as e:
73 print(traceback.format_exc())
74 print("Error in test.py ",str(e))
75 return ts_type(default_ctor_args)
42
76
43 PythonProviders.register_product([("/tests/scalar",[],[("type","scalar")]), ("/tests/vector",[],[("type","vector")]), ("/tests/multicomponent",[],[("type","multicomponent"),('size','4')])],get_data)
77 products = [
78 ("/tests/without_cache/scalar",[],[("type","scalar")]),
79 ("/tests/without_cache/vector",[],[("type","vector")]),
80 ("/tests/without_cache/multicomponent",[],[("type","multicomponent"),('size','4')]),
81 ("/tests/with_cache/scalar",[],[("type","scalar"), ("cache","true")]),
82 ("/tests/with_cache/vector",[],[("type","vector"), ("cache","true")]),
83 ("/tests/with_cache/multicomponent",[],[("type","multicomponent"),('size','4'), ("cache","true")])
84 ]
44
85
45
86
87 PythonProviders.register_product(products ,get_data)
General Comments 0
You need to be logged in to leave comments. Login now