Skip to content

Commit 695d75b

Browse files
author
Dilawar Singh
committed
saving data to npy format in streamer is now bug free. Added three tests: 2 python and 1 at cpp level.
1 parent 280dfa5 commit 695d75b

File tree

14 files changed

+487
-297
lines changed

14 files changed

+487
-297
lines changed

‎basecode/Id.cpp

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,18 +76,26 @@ string Id::id2str( Id id )
7676
string Id::path( const string& separator) const
7777
{
7878
string ret = Neutral::path( eref() );
79-
// Trim off trailing []
80-
assert( ret.length() > 0 );
81-
// the 'back' operation is not supported by pre 2011 compilers
8279

83-
while ( ret[ ret.length() - 1 ] == ']' )
80+
#if 0
81+
// NOTE/FIXME: Monday 09 March 2020 12:30:27 PM IST, Dilawar Singh
82+
// This beaks the path comparison. Getting x.path from x returned in a
83+
// list by moose.wildcardFind() and getting path from here doesn't math
84+
// when this is enabled.
85+
// If we want to remove [0] then it should be done globally and not just
86+
// here.
87+
88+
// Trim off trailing []
89+
assert( ret.length() > 0 );
90+
while ( ret.back() == ']' )
8491
{
8592
size_t pos = ret.find_last_of( '[' );
8693
if ( pos != string::npos && pos > 0 )
8794
{
8895
ret = ret.substr( 0, pos );
8996
}
9097
}
98+
#endif
9199

92100
return ret;
93101
}

‎builtins/Streamer.cpp

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -185,10 +185,10 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
185185
if( tick != tableDt_[0] )
186186
{
187187
moose::showWarn( "Table " + tableIds_[i].path() + " has "
188-
" different clock dt. "
189-
" Make sure all tables added to Streamer have the same "
190-
" dt value."
191-
);
188+
" different clock dt. "
189+
" Make sure all tables added to Streamer have the same "
190+
" dt value."
191+
);
192192
}
193193
}
194194
}
@@ -209,12 +209,12 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
209209
if( tableTick_[i] != tableTick_[0] )
210210
{
211211
LOG( moose::warning
212-
, "Table " << tableIds_[i].path()
213-
<< " has tick (dt) which is different than the first table."
214-
<< endl
215-
<< " Got " << tableTick_[i] << " expected " << tableTick_[0]
216-
<< endl << " Disabling this table."
217-
);
212+
, "Table " << tableIds_[i].path()
213+
<< " has tick (dt) which is different than the first table."
214+
<< endl
215+
<< " Got " << tableTick_[i] << " expected " << tableTick_[0]
216+
<< endl << " Disabling this table."
217+
);
218218
invalidTables.push_back( i );
219219
}
220220
}
@@ -236,7 +236,7 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
236236
// write now.
237237
currTime_ = 0.0;
238238
zipWithTime( );
239-
StreamerBase::writeToOutFile(outfilePath_, format_, "w", data_, columns_);
239+
StreamerBase::writeToOutFile(outfilePath_, format_, WRITE, data_, columns_);
240240
data_.clear( );
241241
}
242242

@@ -247,7 +247,7 @@ void Streamer::reinit(const Eref& e, ProcPtr p)
247247
void Streamer::cleanUp( )
248248
{
249249
zipWithTime( );
250-
StreamerBase::writeToOutFile( outfilePath_, format_, "a", data_, columns_ );
250+
StreamerBase::writeToOutFile( outfilePath_, format_, APPEND, data_, columns_ );
251251
data_.clear( );
252252
}
253253

@@ -261,7 +261,7 @@ void Streamer::process(const Eref& e, ProcPtr p)
261261
{
262262
// LOG( moose::debug, "Writing Streamer data to file." );
263263
zipWithTime( );
264-
StreamerBase::writeToOutFile( outfilePath_, format_, "a", data_, columns_ );
264+
StreamerBase::writeToOutFile( outfilePath_, format_, APPEND, data_, columns_ );
265265
data_.clear();
266266
numWriteEvents_ += 1;
267267
}
@@ -284,12 +284,12 @@ void Streamer::addTable( Id table )
284284
tables_.push_back( t );
285285
tableTick_.push_back( table.element()->getTick() );
286286

287-
// NOTE: If user can make sure that names are unique in table, using name is
288-
// better than using the full path.
289-
if( t->getColumnName().size() > 0 )
290-
columns_.push_back( t->getColumnName( ) );
287+
// NOTE: Table can also have name. If name is set by User, use the name to
288+
// for column name, else use full path of the table.
289+
if(t->getColumnName().size() > 0)
290+
columns_.push_back(t->getColumnName());
291291
else
292-
columns_.push_back( moose::moosePathToUserPath( table.path() ) );
292+
columns_.push_back(table.path());
293293
}
294294

295295
/**
@@ -355,7 +355,7 @@ size_t Streamer::getNumTables( void ) const
355355
* @Synopsis Get number of write events in streamer. Useful for debugging and
356356
* performance measuerments.
357357
*
358-
* @Returns
358+
* @Returns
359359
*/
360360
/* ----------------------------------------------------------------------------*/
361361
size_t Streamer::getNumWriteEvents( void ) const
@@ -414,8 +414,8 @@ void Streamer::zipWithTime( )
414414
{
415415
#if 0
416416
LOG( moose::debug
417-
, "Table " << tables_[i]->getName( ) << " is not functional. Filling with zero "
418-
);
417+
, "Table " << tables_[i]->getName( ) << " is not functional. Filling with zero "
418+
);
419419
#endif
420420
tVec.resize( numEntriesInEachTable, 0 );
421421
}

‎builtins/StreamerBase.cpp

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -57,41 +57,49 @@ void StreamerBase::setOutFilepath( string filepath )
5757

5858
void StreamerBase::writeToOutFile( const string& filepath
5959
, const string& outputFormat
60-
, const string& openmode
60+
, const OpenMode openmode
6161
, const vector<double>& data
6262
, const vector<string>& columns
6363
)
6464
{
6565
if( data.size() == 0 )
6666
return;
6767

68-
if( "npy" == outputFormat )
69-
writeToNPYFile( filepath, openmode, data, columns );
68+
if("npy" == outputFormat || "npz" == outputFormat)
69+
{
70+
OpenMode m = (openmode == WRITE)?WRITE_BIN:APPEND_BIN;
71+
writeToNPYFile( filepath, m, data, columns );
72+
}
7073
else if( "csv" == outputFormat or "dat" == outputFormat )
71-
writeToCSVFile( filepath, openmode, data, columns );
74+
{
75+
OpenMode m = (openmode == WRITE)?WRITE_STR:APPEND_STR;
76+
writeToCSVFile( filepath, m, data, columns );
77+
}
7278
else
7379
{
7480
LOG( moose::warning, "Unsupported format " << outputFormat
7581
<< ". Use npy or csv. Falling back to default csv"
7682
);
77-
writeToCSVFile( filepath, openmode, data, columns );
83+
OpenMode m = (openmode == WRITE)?WRITE_STR:APPEND_STR;
84+
writeToCSVFile( filepath, m, data, columns );
7885
}
7986
}
8087

8188
/* Write to a csv file. */
82-
void StreamerBase::writeToCSVFile( const string& filepath, const string& openmode
89+
void StreamerBase::writeToCSVFile( const string& filepath, const OpenMode openmode
8390
, const vector<double>& data, const vector<string>& columns )
8491
{
92+
string m = (openmode == WRITE_STR)?"w":"a";
93+
FILE* fp = fopen( filepath.c_str(), m.c_str());
8594

86-
FILE* fp = fopen( filepath.c_str(), openmode.c_str() );
8795
if( NULL == fp )
8896
{
8997
LOG( moose::warning, "Failed to open " << filepath );
9098
return;
9199
}
92100

93101
// If writing in "w" mode, write the header first.
94-
if(openmode == "w")
102+
if(openmode == WRITE_STR)
95103
{
96104
string headerText = "";
97105
for( vector<string>::const_iterator it = columns.begin();
@@ -116,16 +124,23 @@ void StreamerBase::writeToCSVFile( const string& filepath, const string& openmod
116124
}
117125

118126
/* write data to a numpy file */
119-
void StreamerBase::writeToNPYFile( const string& filepath, const string& openmode
127+
void StreamerBase::writeToNPYFile( const string& filepath, const OpenMode openmode
120128
, const vector<double>& data, const vector<string>& columns )
121129
{
122-
cnpy2::save_numpy<double>( filepath, data, columns, openmode );
130+
//for(auto v: data) cout << v << ' ';
131+
//cout << endl;
132+
133+
if(openmode == APPEND_BIN)
134+
return cnpy2::appendNumpy( filepath, data, columns);
135+
136+
if(openmode == WRITE_BIN)
137+
return cnpy2::writeNumpy( filepath, data, columns);
123138
}
124139

125140
string StreamerBase::vectorToCSV( const vector<double>& ys, const string& fmt )
126141
{
127-
stringstream ss;
142+
string res{""};
128143
for( auto v : ys )
129-
ss << v << ",";
130-
return ss.str();
144+
res += std::to_string(v) + ",";
145+
return res;
131146
}

‎builtins/StreamerBase.h

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ using namespace std;
2828

2929
class TableBase;
3030

31+
enum OpenMode {WRITE, APPEND, WRITE_STR, APPEND_STR, WRITE_BIN, APPEND_BIN};
32+
3133
class StreamerBase : public TableBase
3234
{
3335

@@ -63,7 +65,7 @@ class StreamerBase : public TableBase
6365
*/
6466
static void writeToOutFile(
6567
const string& filepath, const string& format
66-
, const string& openmode
68+
, const OpenMode openmode
6769
, const vector<double>& data
6870
, const vector<string>& columns
6971
);
@@ -72,15 +74,15 @@ class StreamerBase : public TableBase
7274
* @brief Write data to csv file. See the documentation of writeToOutfile
7375
* for details.
7476
*/
75-
static void writeToCSVFile( const string& filepath, const string& openmode
77+
static void writeToCSVFile( const string& filepath, const OpenMode openMode
7678
, const vector<double>& data, const vector<string>& columns
7779
);
7880

7981
/**
8082
* @brief Write to NPY format. See the documentation of
8183
* writeToOutfile for more details.
8284
*/
83-
static void writeToNPYFile( const string& filepath, const string& openmode
85+
static void writeToNPYFile( const string& filepath, const OpenMode openmode
8486
, const vector<double>& data
8587
, const vector<string>& columns
8688
);

‎builtins/Table.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ Table::~Table( )
228228
if( useFileStreamer_ )
229229
{
230230
mergeWithTime( data_ );
231-
StreamerBase::writeToOutFile( outfile_, format_, "a", data_, columns_ );
231+
StreamerBase::writeToOutFile( outfile_, format_, APPEND, data_, columns_ );
232232
clearAllVecs();
233233
}
234234
}
@@ -269,7 +269,7 @@ void Table::process( const Eref& e, ProcPtr p )
269269
if( fmod(lastTime_, 5.0) == 0.0 || getVecSize() >= 10000 )
270270
{
271271
mergeWithTime( data_ );
272-
StreamerBase::writeToOutFile( outfile_, format_, "a", data_, columns_ );
272+
StreamerBase::writeToOutFile( outfile_, format_, APPEND, data_, columns_ );
273273
clearAllVecs();
274274
}
275275
}
@@ -332,7 +332,7 @@ void Table::reinit( const Eref& e, ProcPtr p )
332332
if( useFileStreamer_ )
333333
{
334334
mergeWithTime( data_ );
335-
StreamerBase::writeToOutFile( outfile_, format_, "w", data_, columns_);
335+
StreamerBase::writeToOutFile( outfile_, format_, WRITE, data_, columns_);
336336
clearAllVecs();
337337
}
338338
}

‎pymoose/CMakeLists.txt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ execute_process( COMMAND ${PYTHON_EXECUTABLE}-config --libs
5959

6060
set_target_properties(_moose PROPERTIES
6161
COMPILE_DEFINITIONS "PYMOOSE"
62-
COMPILE_FLAGS "${COMPILE_FLAGS} ${PYTHON_INCLUDE_FLAGS}"
63-
)
62+
COMPILE_FLAGS "${COMPILE_FLAGS} ${PYTHON_INCLUDE_FLAGS}")
6463

6564
# Remove prefix lib from python module.
6665
if(NOT(PYTHON_SO_EXTENSION STREQUAL ""))

‎shell/Neutral.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -764,6 +764,7 @@ string Neutral::path( const Eref& e )
764764
stringstream ss;
765765

766766
pathVec.push_back( curr );
767+
767768
while ( curr.id != Id() )
768769
{
769770
ObjId mid = curr.eref().element()->findCaller( pafid );
@@ -776,8 +777,10 @@ string Neutral::path( const Eref& e )
776777
curr = Msg::getMsg( mid )->findOtherEnd( curr );
777778
pathVec.push_back( curr );
778779
}
780+
779781
if ( pathVec.size() <= 1 )
780782
return "/";
783+
781784
for ( unsigned int i = 1; i < pathVec.size(); ++i )
782785
{
783786
ss << "/";
@@ -792,6 +795,7 @@ string Neutral::path( const Eref& e )
792795
ss << "[" << oid.dataIndex << "]";
793796
*/
794797
}
798+
795799
// Append braces if Eref was for a fieldElement. This should
796800
// work even if it is off-node.
797801
if ( e.element()->hasFields() )

‎shell/Wildcard.cpp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,6 @@ static int innerFind( const string& path, vector< ObjId >& ret)
104104
return wildcardRelativeFind( start, names, 0, ret );
105105
}
106106

107-
/*
108-
static int wildcardRelativeFind( Id start, const vector< string >& path,
109-
unsigned int depth, vector< Id >& ret )
110-
*/
111-
112107
/**
113108
* This is the basic wildcardFind function, working on a single
114109
* tree. It adds entries into the vector 'ret' with Ids found according

0 commit comments

Comments
 (0)