HG_REPOSITORIES/LPP/SciQLOP_Repos/SciQLop Commit - r1439:449d899d9309

Better AMDA tree and test data cat also go through cache...

jeandet -

r1439:449d899d9309

parent child

Context file:

r1439:449d899d9309

plugins/python_providers/resources/amda.py +3 -3

             import sys
             sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
             import traceback
             import os
             from datetime import datetime, timedelta, timezone
             import PythonProviders
             import pysciqlopcore
             import numpy as np
             import pandas as pds
             import requests
             import copy
             from spwc.amda import AMDA
             amda = AMDA()
             def get_sample(metadata,start,stop):
                 ts_type = pysciqlopcore.ScalarTimeSerie
                 default_ctor_args = 1
                 try:
                     param_id = None
                     for key,value in metadata:
                         if key == 'xml:id':
                             param_id = value
                         elif key == 'type':
                             if value == 'vector':
                                 ts_type = pysciqlopcore.VectorTimeSerie
                             elif value == 'multicomponent':
                                 ts_type = pysciqlopcore.MultiComponentTimeSerie
                                 default_ctor_args = (0,2)
-                    tstart=datetime.datetime.fromtimestamp(start, tz=timezone.utc)
+                    tstart=datetime.fromtimestamp(start, tz=timezone.utc)
-                    tend=datetime.datetime.fromtimestamp(stop, tz=timezone.utc)
+                    tend=datetime.fromtimestamp(stop, tz=timezone.utc)
                     df = amda.get_parameter(start_time=tstart, stop_time=tend, parameter_id=param_id, method="REST")
                     t = np.array([d.timestamp() for d in df.index])
                     values = df.values
                     return ts_type(t,values)
                 except Exception as e:
                     print(traceback.format_exc())
                     print("Error in amda.py ",str(e))
                     return ts_type(default_ctor_args)
             if len(amda.component) is 0:
                 amda.update_inventory()
             parameters = copy.deepcopy(amda.parameter)
             for name,component in amda.component.items():
                 if 'components' in parameters[component['parameter']]:
                     parameters[component['parameter']]['components'].append(component)
                 else:
                     parameters[component['parameter']]['components']=[component]
             products = []
             for key,parameter in parameters.items():
-                path = f"/AMDA/{parameter['mission']}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
+                path = f"/AMDA/{parameter['mission']}/{parameter.get('observatory','')}/{parameter['instrument']}/{parameter['dataset']}/{parameter['name']}"
                 components = [component['name'] for component in parameter.get('components',[])]
                 metadata = [ (key,item) for key,item in parameter.items() if key is not 'components' ]
                 n_components = parameter.get('size',0)
                 if n_components is '3':
                     metadata.append(("type","vector"))
                 elif n_components !=0:
                     metadata.append(("type","multicomponent"))
                 else:
                     metadata.append(("type","scalar"))
                 products.append( (path, components, metadata))
             PythonProviders.register_product(products, get_sample)

plugins/python_providers/resources/test.py +59 -17

@@ -1,45 +1,87
1	import sys	1	import sys
2	sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")	2	sys.path.append("/home/jeandet/Documents/prog/build-SciQLop-Desktop-Debug/core")
3	import PythonProviders	3	import PythonProviders
4	import pysciqlopcore	4	import pysciqlopcore
5	import numpy as np	5	import numpy as np
6	import math	6	import math
		7	from spwc.cache import _cache
		8	from spwc.common.datetime_range import DateTimeRange
		9	from functools import partial
		10	from datetime import datetime, timedelta, timezone
7		11
8	someglobal = 1	12	someglobal = 1
9		13
10	def make_scalar(x):	14	def make_scalar(x):
11	y = np.cos(x/10.)	15	y = np.cos(x/10.)
12	return pysciqlopcore.ScalarTimeSerie(x,y)	16	return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=y)
13		17
14	def make_vector(x):	18	def make_vector(x):
15	v=np.ones((len(x),3))	19	v=np.ones((len(x),3))
16	for i in range(3):	20	for i in range(3):
17	v.transpose()[:][i] = np.cos(x/10. + float(i))	21	v.transpose()[:][i] = np.cos(x/10. + float(i)) + (100. * np.cos(x/10000. + float(i)))
18	return pysciqlopcore.VectorTimeSerie(x,v)	22	return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=v)
19		23
20		24
21	def make_multicomponent(x):	25	def make_multicomponent(x):
22	v=np.ones((len(x),4))	26	v=np.ones((len(x),4))
23	for i in range(4):	27	for i in range(4):
24	v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i))	28	v.transpose()[:][i] = float(i+1) * np.cos(x/10. + float(i))
25	return pysciqlopcore.MultiComponentTimeSerie(x,v)	29	return pds.DataFrame(index=[datetime.fromtimestamp(t, tz=timezone.utc) for t in x], data=v)
26		30
27		31
28	def get_data(~~metadata~~,start,stop):	32	def _get_data(p_type, start, stop):
		33	if type(start) is datetime:
		34	start = start.timestamp()
		35	stop = stop.timestamp()
29	x = np.arange(math.ceil(start), math.floor(stop))	36	x = np.arange(math.ceil(start), math.floor(stop))
30	for key,value in metadata:	37	if p_type == 'scalar':
31	if key == 'xml:id':	38	return make_scalar(x)
32	param_id = value	39	if p_type == 'vector':
33	elif key == 'type':	40	return make_vector(x)
34	if value == 'vector':	41	if p_type == 'multicomponent':
35	return make_~~vector~~(x)	42	return make_multicomponent(x)
36	elif value == 'multicomponent':	43	return None
37	return make_multicomponent(x)
38	return make_scalar(x)
39
40
41		44
		45	def get_data(metadata,start,stop):
		46	ts_type = pysciqlopcore.ScalarTimeSerie
		47	default_ctor_args = 1
		48	use_cache = False
		49	p_type = 'scalar'
		50	try:
		51	for key,value in metadata:
		52	if key == 'type':
		53	p_type = value
		54	if value == 'vector':
		55	ts_type = pysciqlopcore.VectorTimeSerie
		56	elif value == 'multicomponent':
		57	ts_type = pysciqlopcore.MultiComponentTimeSerie
		58	default_ctor_args = (0,2)
		59	if key == 'cache' and value == 'true':
		60	use_cache = True
		61	if use_cache:
		62	cache_product = f"tests/{p_type}"
		63	df = _cache.get_data(cache_product, DateTimeRange(datetime.fromtimestamp(start, tz=timezone.utc), datetime.fromtimestamp(stop, tz=timezone.utc)),
		64	partial(_get_data, p_type),
		65	fragment_hours=24)
		66	else:
		67	print("No Cache")
		68	df = _get_data(p_type, start, stop)
		69	t = np.array([d.timestamp() for d in df.index])
		70	values = df.values
		71	return ts_type(t,values)
		72	except Exception as e:
		73	print(traceback.format_exc())
		74	print("Error in test.py ",str(e))
		75	return ts_type(default_ctor_args)
42		76
43	PythonProviders.register_product([("/tests/scalar",[],[("type","scalar")]), ("/tests/vector",[],[("type","vector")]), ("/tests/multicomponent",[],[("type","multicomponent"),('size','4')])],get_data)	77	products = [
		78	("/tests/without_cache/scalar",[],[("type","scalar")]),
		79	("/tests/without_cache/vector",[],[("type","vector")]),
		80	("/tests/without_cache/multicomponent",[],[("type","multicomponent"),('size','4')]),
		81	("/tests/with_cache/scalar",[],[("type","scalar"), ("cache","true")]),
		82	("/tests/with_cache/vector",[],[("type","vector"), ("cache","true")]),
		83	("/tests/with_cache/multicomponent",[],[("type","multicomponent"),('size','4'), ("cache","true")])
		84	]
44		85
45		86
		87	PythonProviders.register_product(products ,get_data)

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages