diff --git a/requirements.txt b/requirements.txt
index 6b39ddd3859dbf67082222cd3a8915786d6dedfc..8f3d12d3e7e7409ebb738b1d623cfe1d6c83bacf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,4 +12,3 @@ seaborn
vre-middleware >=1.2.4
jupyter_client
ipykernel
-pympler
diff --git a/setup.cfg b/setup.cfg
index 95587fd243c266a9f7573b09d930b5f439eb905b..ebf7b726267f3fb2d3a814d374a6edca666b7967 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -57,7 +57,6 @@ install_requires =
vre-middleware >=1.2.4
jupyter_client
ipykernel
- pympler
[options.extras_require]
test =
diff --git a/src/virtmat/language/interpreter/session_manager.py b/src/virtmat/language/interpreter/session_manager.py
index a0bae94ff2806dfd9fcf20639fabdb203640d248..0f097643f12ae604148cb5cd1e78b6c518a8140f 100644
--- a/src/virtmat/language/interpreter/session_manager.py
+++ b/src/virtmat/language/interpreter/session_manager.py
@@ -110,7 +110,9 @@ def get_prettytable(dataframe):
table = class_(list(dataframe.columns))
for tpl in dataframe.itertuples(index=False, name=None):
table.add_row(tpl)
- return str(table)
+ table.align = 'l'
+ table.max_width = 120
+ return table
class SessionManager(InteractiveConsole):
diff --git a/src/virtmat/language/utilities/fireworks.py b/src/virtmat/language/utilities/fireworks.py
index 65bf4232c549aa1373d3774ceef7b993719badc3..1a0e671c4f646a201dde9c792810c1fbb73a3b99 100644
--- a/src/virtmat/language/utilities/fireworks.py
+++ b/src/virtmat/language/utilities/fireworks.py
@@ -348,14 +348,15 @@ def get_model_nodes(lpad, uuid):
def get_model_history(lpad, uuid):
"""return node history with some node attributes as pandas dataframe"""
- dct = {'state': [], 'updated_on': [], 'source': []}
+ dct = {'State': [], 'Updated on': [], 'Statement': []}
for fwk in get_model_nodes(lpad, uuid):
if fwk['spec']['_source_code']:
- dct['state'].append(fwk['state'])
- dct['updated_on'].append(get_iso_datetime(fwk['updated_on']))
- dct['source'].append('; '.join(fwk['spec']['_source_code']))
- df = pandas.DataFrame(dct).sort_values('updated_on').sort_values('state')
- return df[['state', 'updated_on', 'source']]
+ dct['State'].append(fwk['state'])
+ timestamp = get_iso_datetime(fwk['updated_on'], add_tzinfo=False, sep=' ')
+ dct['Updated on'].append(timestamp)
+ dct['Statement'].append('; '.join(fwk['spec']['_source_code']))
+ df = pandas.DataFrame(dct).sort_values('Updated on')
+ return df[['Updated on', 'State', 'Statement']] # pylint: disable=E1136
def get_model_tag(lpad, uuid):
@@ -398,7 +399,7 @@ def get_models_overview(lpad, uuids):
wf_states = []
for wf in wfs:
hist = get_model_history(lpad, wf['metadata']['uuid'])
- wf_states.append(dict(Counter(hist['state'].tolist())))
+ wf_states.append(dict(Counter(hist['State'].tolist())))
df_2 = pandas.DataFrame(wf_states).fillna(0).astype('int64')
df_2.rename(lambda x: x[0:3], axis='columns', inplace=True)
df_3 = get_models_tags(lpad, uuids)
diff --git a/src/virtmat/language/utilities/serializable.py b/src/virtmat/language/utilities/serializable.py
index 0454578e3ee9c0f3870efabc2d8899978c6428e0..291ccda718f959300e3521abe4b941d2d0f67b84 100644
--- a/src/virtmat/language/utilities/serializable.py
+++ b/src/virtmat/language/utilities/serializable.py
@@ -1,10 +1,11 @@
"""serialization/deserialization code"""
-from dataclasses import dataclass
import typing
+from dataclasses import dataclass
+from json import JSONEncoder
+from itertools import islice
import numpy
import pandas
import pint_pandas
-from pympler import asizeof
from fireworks.utilities.fw_serializers import FWSerializable
from fireworks.utilities.fw_serializers import serialize_fw
from fireworks.utilities.fw_serializers import recursive_serialize
@@ -45,6 +46,18 @@ def versioned_serialize(func):
return decorator
+def get_json_size(obj, max_size):
+ """compute JSON size in bytes of a JSON serializable object up to max_size"""
+ gen = JSONEncoder().iterencode(obj)
+ chunk_size = 1024
+ json_size = 0
+ next_chunk = len(''.join(islice(gen, chunk_size)).encode())
+ while next_chunk and json_size < max_size:
+ json_size += next_chunk
+ next_chunk = len(''.join(islice(gen, chunk_size)).encode())
+ return json_size
+
+
@dataclass
class FWDataObject(FWSerializable):
"""top-level FWSerializable dataclass to hold any FWSerializable objects"""
@@ -57,21 +70,31 @@ class FWDataObject(FWSerializable):
@recursive_serialize
@versioned_serialize
def to_dict(self):
+ f_name = f'{__name__}.{self.__class__.__name__}.to_dict()'
+ logger = get_fw_logger(f_name)
+ logger.debug('%s: starting', f_name)
if self.datastore is None:
- logger = get_fw_logger(__name__)
- mem_size = asizeof.asizeof(self.value)
- logger.debug('%s: size in memory: %s', __name__, mem_size)
- if mem_size < ioops.DATASTORE_CONFIG['inline-threshold']:
+ dct = recursive_dict(self.value)
+ b_thres = ioops.DATASTORE_CONFIG['inline-threshold']
+ b_size = get_json_size(dct, b_thres)
+ logger.debug('%s: data type: %s', f_name, type(self.value))
+ logger.debug('%s: data size [B]: %s', f_name, b_size)
+ logger.debug('%s: inline-threshold [B]: %s', f_name, b_thres)
+ if b_size < b_thres:
self.datastore = {'type': None}
- return {'value': self.value, 'datastore': self.datastore}
- logger.info('%s: inline data limit exceeded: %s', __name__, mem_size)
- self.datastore, self.filename = ioops.offload_data(recursive_dict(self.value))
+ logger.info('%s: data not offloaded', f_name)
+ return {'value': dct, 'datastore': self.datastore}
+ logger.info('%s: inline data limit exceeded: %s', f_name, b_size)
+ self.datastore, self.filename = ioops.offload_data(dct)
if self.datastore['type'] is None:
- logger.info('%s: data not offloaded', __name__)
+ logger.info('%s: data not offloaded', f_name)
else:
- logger.info('%s: data offloaded in %s', __name__, self.filename)
+ logger.info('%s: data offloaded in %s', f_name, self.filename)
if self.datastore['type'] is None:
+ logger.debug('%s: datastore: %s', f_name, self.datastore)
return {'value': self.value, 'datastore': self.datastore}
+ logger.debug('%s: datastore: %s', f_name, self.datastore)
+ logger.debug('%s: data in file: %s', f_name, self.filename)
return {'datastore': self.datastore, 'filename': self.filename}
@classmethod
diff --git a/src/virtmat/language/utilities/textx.py b/src/virtmat/language/utilities/textx.py
index e7d2dfe8b270df8a4aa6ecf214b56326551f9e85..d692e80e4b33fe75043020218d990c97084be9ef 100644
--- a/src/virtmat/language/utilities/textx.py
+++ b/src/virtmat/language/utilities/textx.py
@@ -142,7 +142,7 @@ def display_exception(func):
return func(*args, **kwargs)
except Exception as err:
print('\n', file=sys.stderr)
- traceback.print_exception(err, file=sys.stderr)
+ traceback.print_exception(*sys.exc_info(), file=sys.stderr)
print('\n', file=sys.stderr)
raise err
return decorator