From 20cab1cec2d3791175cccf7c405f68a9aab774a8 Mon Sep 17 00:00:00 2001 From: jankoslavic Date: Mon, 13 May 2024 18:06:11 +0200 Subject: [PATCH 1/3] implemented arbitrary data separator, for now supported Tab and Comma. --- data/comma_separator.lvm | 27 +++++++++++++++++++++++++++ lvm_read.py | 26 +++++++++++++++++++++----- tests/test_all.py | 8 ++++++-- 3 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 data/comma_separator.lvm diff --git a/data/comma_separator.lvm b/data/comma_separator.lvm new file mode 100644 index 0000000..5ed6234 --- /dev/null +++ b/data/comma_separator.lvm @@ -0,0 +1,27 @@ +LabVIEW Measurement, +Writer_Version,2 +Reader_Version,2 +Separator,Comma +Decimal_Separator,. +Multi_Headings,No +X_Columns,One +Time_Pref,Absolute +Operator,photron +Date,2024/04/10 +Time,13:06:46.2639230999998221435 +***End_of_Header***, +, +Channels,4,,,, +Samples,4,4,4,4, +Date,2024/04/10,2024/04/10,2024/04/10,2024/04/10, +Time,13:06:46.2639230999998221435,13:06:46.2639230999998221435,13:06:46.2639230999998221435,13:06:46.2639230999998221435, +Y_Unit_Label,Volts,Volts,Volts,Volts, +X_Dimension,Time,Time,Time,Time, +X0,0.0000000000000000E+0,0.0000000000000000E+0,0.0000000000000000E+0,0.0000000000000000E+0, +Delta_X,2.000000E-6,2.000000E-6,2.000000E-6,2.000000E-6, +***End_of_Header***,,,,, +X_Value,p_st1,p_st2,p_plate,TC1,Comment +0.000000,-0.000424,-0.002187,-0.002505,0.163976 +2.000000E-6,-0.000424,-0.002827,-0.002185,0.163337 +4.000000E-6,-0.000744,-0.003147,-0.003144,0.163657 +6.000000E-6,-0.000424,-0.002827,-0.002505,0.163657 diff --git a/lvm_read.py b/lvm_read.py index 90c05bc..5252ef0 100644 --- a/lvm_read.py +++ b/lvm_read.py @@ -7,7 +7,7 @@ import pickle import numpy as np -__version__ = '1.21' +__version__ = '1.22' def _lvm_pickle(filename): """ Reads pickle file (for local use) @@ -42,6 +42,21 @@ def _lvm_dump(lvm_data, filename, protocol=-1): pickle.dump(lvm_data, output, protocol=protocol) output.close() +def get_separator(file): + separators = {'Tab': '\t', + 'Comma': ',' + } + for line in file: + if line.startswith('Separator'): + separator = line.strip()[10:] + break + file.seek(0) + if separator in separators.keys(): + return separators[separator] + else: + raise Warning('No separator defined, using tabulator!') + return '\t' + def _read_lvm_base(filename): """ Base lvm reader. Should be called from ``read``, only @@ -50,11 +65,12 @@ def _read_lvm_base(filename): :return lvm_data: lvm dict """ with open(filename, 'r', encoding="utf8", errors='ignore') as f: - lvm_data = read_lines(f) + separator = get_separator(f) + lvm_data = read_lines(f, separator=separator) return lvm_data -def read_lines(lines): +def read_lines(lines, separator='\t'): """ Read lines of strings. :param lines: lines of the lvm file @@ -76,10 +92,10 @@ def to_float(a): return np.nan for line in lines: line = line.replace('\r', '') - line_sp = line.replace('\n', '').split('\t') + line_sp = line.replace('\n', '').split(separator) if line_sp[0] in ['***End_of_Header***', 'LabVIEW Measurement']: continue - elif line in ['\n', '\t\n']: + elif line in ['\n', separator+'\n']: # segment finished, new segment follows segment = dict() lvm_data[segment_nr] = segment diff --git a/tests/test_all.py b/tests/test_all.py index a0d5f9d..091cea7 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -10,7 +10,7 @@ from lvm_read import read -def test_short_lvm(): +def test_short_lvm(): data = read('./data/pickle_only.lvm') np.testing.assert_equal(data[0]['data'][0,0],0.914018) @@ -48,9 +48,13 @@ def timing_on_long_short_lvm(): toc = time.time() print(f'Average time: {(toc-tic)/N:3.1f}s') +def test_comma_separator(): + data = read('./data/comma_separator.lvm', read_from_pickle=False, dump_file=False) + np.testing.assert_equal(data[0]['data'][0,1],-0.000424) + if __name__ == '__mains__': np.testing.run_module_suite() if __name__ == '__main__': - test_several_comments() + test_comma_separator() #timing_on_long_short_lvm() \ No newline at end of file From c35c7fbb11ea3461f910b8639547616c0785416f Mon Sep 17 00:00:00 2001 From: jankoslavic Date: Tue, 14 May 2024 06:23:10 +0200 Subject: [PATCH 2/3] some simplifications and improving robustnes --- lvm_read.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/lvm_read.py b/lvm_read.py index 5252ef0..2a95eb8 100644 --- a/lvm_read.py +++ b/lvm_read.py @@ -6,8 +6,9 @@ from os import path import pickle import numpy as np +import io -__version__ = '1.22' +__version__ = '1.23' def _lvm_pickle(filename): """ Reads pickle file (for local use) @@ -42,21 +43,15 @@ def _lvm_dump(lvm_data, filename, protocol=-1): pickle.dump(lvm_data, output, protocol=protocol) output.close() -def get_separator(file): - separators = {'Tab': '\t', - 'Comma': ',' - } +def _get_separator(file): + separator = '\t' for line in file: if line.startswith('Separator'): - separator = line.strip()[10:] + separator = line.strip()[9] break - file.seek(0) - if separator in separators.keys(): - return separators[separator] - else: - raise Warning('No separator defined, using tabulator!') - return '\t' - + if isinstance(file, io.IOBase): + file.seek(0) + return separator def _read_lvm_base(filename): """ Base lvm reader. Should be called from ``read``, only @@ -65,7 +60,7 @@ def _read_lvm_base(filename): :return lvm_data: lvm dict """ with open(filename, 'r', encoding="utf8", errors='ignore') as f: - separator = get_separator(f) + separator = _get_separator(f) lvm_data = read_lines(f, separator=separator) return lvm_data From 9c734d8d15b92781ebe544bf32a172cf18e7854a Mon Sep 17 00:00:00 2001 From: jankoslavic Date: Wed, 15 May 2024 08:10:53 +0200 Subject: [PATCH 3/3] check only first 20 lines for Separator --- lvm_read.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lvm_read.py b/lvm_read.py index 2a95eb8..fd65a7d 100644 --- a/lvm_read.py +++ b/lvm_read.py @@ -45,10 +45,15 @@ def _lvm_dump(lvm_data, filename, protocol=-1): def _get_separator(file): separator = '\t' + i = 0 for line in file: if line.startswith('Separator'): separator = line.strip()[9] break + if i>20: + break + i+=1 + if isinstance(file, io.IOBase): file.seek(0) return separator