Commit 08e912a3 authored by Samuel Debionne's avatar Samuel Debionne

Merge branch 'statistics' into 'master'

Add saving statistics logfile

See merge request !101
parents 555f66d0 bc0beac3
Pipeline #6679 passed with stages
in 13 minutes and 53 seconds
......@@ -215,6 +215,9 @@ namespace lima {
int stream_idx=0) const;
void setStatisticHistorySize(int aSize, int stream_idx=0);
int getStatisticHistorySize(int stream_idx=0) const;
void setEnableLogStat(bool enable, int stream_idx=0);
void getEnableLogStat(bool& enable, int stream_idx=0) const;
// --- misc
void clear();
......@@ -232,7 +235,7 @@ namespace lima {
class Stream;
class Stream;
class LIMACORE_API SaveContainer
{
......@@ -329,8 +332,14 @@ namespace lima {
void createStatistic(Data&);
void compressionStart(Data&);
void compressionFinished(Data&);
void writeFileStat(Data&, Timestamp start, Timestamp end, long wsize);
void prepareLogStat(const CtSaving::Parameters&);
int getMaxConcurrentWritingTask() const;
void setMaxConcurrentWritingTask(int nb);
void setEnableLogStat(bool enable);
void getEnableLogStat(bool& enable) const;
protected:
virtual void* _open(const std::string &filename,
std::ios_base::openmode flags) = 0;
......@@ -349,6 +358,10 @@ namespace lima {
private:
StatisticsType m_statistic;
int m_statistic_size;
bool m_log_stat_enable;
std::string m_log_stat_directory;
std::string m_log_stat_filename;
FILE* m_log_stat_file;
mutable Cond m_cond;
long m_nb_frames_to_write;
......@@ -434,6 +447,11 @@ namespace lima {
void setMaxConcurrentWritingTask(int nb_threads)
{ m_save_cnt->setMaxConcurrentWritingTask(nb_threads); }
void setEnableLogStat(bool enable)
{ m_save_cnt->setEnableLogStat(enable); }
void getEnableLogStat(bool& enable) const
{ m_save_cnt->getEnableLogStat(enable); }
void clear()
{ m_save_cnt->clear(); }
......
......@@ -252,6 +252,10 @@ using namespace lima;
void setStatisticHistorySize(int aSize, int stream_idx=0);
int getStatisticHistorySize(int stream_idx=0) const;
void setEnableLogStat(bool enable, int stream_idx=0);
void getEnableLogStat(bool &enable /Out/, int stream_idx=0) const;
// --- misc
void clear();
......
......@@ -316,6 +316,7 @@ void CtSaving::Stream::createSaveContainer()
int statistic_size = -1;
int nb_writing_thread = -1;
bool enable_log_stat = false;
switch (m_pars.fileFormat) {
case CBFFormat :
......@@ -389,6 +390,7 @@ void CtSaving::Stream::createSaveContainer()
if (m_save_cnt) {
statistic_size = m_save_cnt->getStatisticSize();
nb_writing_thread = m_save_cnt->getMaxConcurrentWritingTask();
m_save_cnt->getEnableLogStat(enable_log_stat);
m_save_cnt->close();
delete m_save_cnt;
}
......@@ -444,6 +446,7 @@ void CtSaving::Stream::createSaveContainer()
m_save_cnt->setStatisticSize(statistic_size);
if(nb_writing_thread != -1)
m_save_cnt->setMaxConcurrentWritingTask(nb_writing_thread);
m_save_cnt->setEnableLogStat(enable_log_stat);
}
void CtSaving::Stream::writeFile(Data& data, HeaderMap& header)
......@@ -1651,6 +1654,21 @@ void CtSaving::setMaxConcurrentWritingTask(int nb_threads,int stream_idx)
stream.setMaxConcurrentWritingTask(nb_threads);
}
void CtSaving::getEnableLogStat(bool& enable, int stream_idx) const
{
DEB_MEMBER_FUNCT();
const Stream& stream = getStream(stream_idx);
stream.getEnableLogStat(enable);
}
void CtSaving::setEnableLogStat(bool enable, int stream_idx)
{
DEB_MEMBER_FUNCT();
DEB_PARAM() << DEB_VAR2(enable, stream_idx);
Stream& stream = getStream(stream_idx);
stream.setEnableLogStat(enable);
}
/** @brief clear everything.
- all header
- all waiting data to be saved
......@@ -2016,8 +2034,8 @@ CtConfig::ModuleTypeCallback* CtSaving::_getConfigHandler()
CtSaving::SaveContainer::SaveContainer(Stream& stream)
: m_stream(stream), m_statistic_size(16),
m_max_writing_task(1),
m_running_writing_task(0)
m_max_writing_task(1), m_running_writing_task(0),
m_log_stat_enable(false)
{
DEB_CONSTRUCTOR();
}
......@@ -2032,8 +2050,7 @@ void CtSaving::SaveContainer::writeFile(Data &aData,HeaderMap &aHeader)
DEB_MEMBER_FUNCT();
DEB_PARAM() << DEB_VAR2(aData,aHeader);
struct timeval start_write;
gettimeofday(&start_write, NULL);
Timestamp start_write = Timestamp::now();
long frameId = aData.frameNumber;
......@@ -2069,6 +2086,7 @@ void CtSaving::SaveContainer::writeFile(Data &aData,HeaderMap &aHeader)
unsigned long f_fsid; // file system ID
unsigned long f_flag; //mount flags
unsigned long f_namemax; // maximum filename length
}
*/
//Check if disk full
struct statvfs vfs;
......@@ -2122,28 +2140,111 @@ void CtSaving::SaveContainer::writeFile(Data &aData,HeaderMap &aHeader)
}
}
struct timeval end_write;
gettimeofday(&end_write, NULL);
Timestamp end_write = Timestamp::now();
double diff = (end_write.tv_sec - start_write.tv_sec) +
(end_write.tv_usec - start_write.tv_usec) / 1e6;
Timestamp diff = end_write - start_write;
DEB_TRACE() << "Write took : " << diff << "s";
lock.lock();
writeFileStat(aData, start_write, end_write, write_size);
}
void CtSaving::SaveContainer::writeFileStat(Data& aData, Timestamp start, Timestamp end, long wsize)
{
long frameId = aData.frameNumber;
if(long(m_statistic.size()) >= m_statistic_size)
m_statistic.erase(m_statistic.begin());
StatisticsType::iterator i = m_statistic.find(frameId);
if(i != m_statistic.end())
{
i->second.writing_start = start_write.tv_sec + start_write.tv_usec * 1e-6;
i->second.writing_end = end_write.tv_sec + end_write.tv_usec * 1e-6;
i->second.write_size = write_size;
i->second.writing_start = start;
i->second.writing_end = end;
i->second.write_size = wsize;
if (m_log_stat_file) {
double comp_time=0., comp_rate=0., comp_ratio=0.;
double write_time, write_rate, total_time;
if (i->second.compression_end.isSet() && i->second.compression_start.isSet()) {
comp_time= i->second.compression_end - i->second.compression_start;
comp_rate= i->second.incoming_size / comp_time / 1024. / 1024.;
comp_ratio= i->second.incoming_size / i->second.write_size;
}
write_time= i->second.writing_end - i->second.writing_start;
write_rate= i->second.write_size / write_time / 1024. / 1024.;
total_time= i->second.writing_end - i->second.received_time;
fprintf(m_log_stat_file, "%d %.2f %.2f %.2f %.2f %.2f %.2f %.2f\n", \
frameId, (double)i->second.received_time, \
comp_time*1000.0, comp_rate, comp_ratio, \
write_time*1000.0, write_rate, total_time*1000.0);
}
}
}
void CtSaving::SaveContainer::setEnableLogStat(bool enable)
{
m_log_stat_enable = enable;
}
void CtSaving::SaveContainer::getEnableLogStat(bool& enable) const
{
enable = m_log_stat_enable;
}
void CtSaving::SaveContainer::prepareLogStat(const CtSaving::Parameters& pars)
{
DEB_MEMBER_FUNCT();
DEB_TRACE() << DEB_VAR1(m_log_stat_enable);
int stream_idx = m_stream.getIndex();
if (m_log_stat_enable) {
if (m_log_stat_directory.empty()) {
m_log_stat_directory = pars.directory;
} else {
m_stream.checkDirectoryAccess(m_log_stat_directory);
}
time_t rawtime;
struct tm * timeinfo;
char buffer[80];
time(&rawtime);
timeinfo = localtime(&rawtime);
strftime(buffer,sizeof(buffer),"%Y%m%d",timeinfo);
std::string log_name(buffer);
std::ostringstream stream_str;
std::string new_path;
stream_str << stream_idx;
log_name = log_name + "_" + stream_str.str() + ".log";
new_path = m_log_stat_directory + DIR_SEPARATOR + log_name;
if (new_path != m_log_stat_filename) {
if (m_log_stat_file)
fclose(m_log_stat_file);
DEB_TRACE() << "Open Stat Log File : " << new_path;
m_log_stat_file = fopen(new_path.c_str(), "aw");
if (!m_log_stat_file) {
m_log_stat_filename = "";
DEB_ERROR() << "Cannot create log stat file";
} else {
m_log_stat_filename = new_path;
}
}
if (m_log_stat_file) {
std::ostringstream head_str;
head_str << pars;
strftime(buffer, sizeof(buffer), "%Y/%m/%d %H:%M:%S", timeinfo);
fprintf(m_log_stat_file, "\n# %s\n# %s\n", buffer, head_str.str().c_str());
fprintf(m_log_stat_file, "# frame frame_time(sec) comp_time(msec) comp_rate(MB/s) comp_ratio write_time(msec) write_rate(MB/s) total_time(msec)\n");
}
}
}
void CtSaving::SaveContainer::setStatisticSize(int aSize)
{
AutoMutex aLock = AutoMutex(m_cond.mutex());
......@@ -2314,6 +2415,7 @@ void CtSaving::SaveContainer::prepare(CtControl& ct)
}
}
m_running_writing_task = 0;
prepareLogStat(pars);
lock.unlock();
_prepare(ct); // call inheritance if needed
}
......
......@@ -67,7 +67,7 @@ class TestSaving:
del self.cam_hw
@Core.DEB_MEMBER_FUNCT
def start(self, exp_time, nb_frames, directory, prefix, fmt, overwrite, framesperfile, threads, repeats):
def start(self, exp_time, nb_frames, directory, prefix, fmt, overwrite, framesperfile, threads, repeats, log_stat):
if fmt.lower() not in self.format_list:
raise ValueError("Unsupported file format. Should be one of %s"%str(self.format_list))
......@@ -87,6 +87,8 @@ class TestSaving:
#self.ct_saving.setNextNumber(0)
self.ct_saving.setStatisticHistorySize(nb_frames)
self.ct_saving.setEnableLogStat(log_stat)
# Setting Pool thread can improve the performance on multi-core computer, e.g for compression purpose
Core.Processlib.PoolThreadMgr.get().setNumberOfThread(threads)
......@@ -131,6 +133,7 @@ def main(argv):
parser.add_argument('-t', '--threads', help='number of Processlib pool threads', type=int, required=False, default=2)
camera_list = ['simulator', 'maxipix']
parser.add_argument('-c', '--camera', help='camera to test', choices=camera_list, required=False, default='simulator')
parser.add_argument('-l', '--log-stat', help='log statistics', required=False, default=False, action='store_true')
args = parser.parse_args()
if args.verbose == 1:
......@@ -142,12 +145,13 @@ def main(argv):
elif args.verbose >= 3:
Core.DebParams.setTypeFlags(Core.DebParams.AllFlags)
Core.DebParams.setModuleFlags(Core.DebParams.AllFlags)
exp_time = 0.1
test_saving = TestSaving(args.camera)
if args.format == 'all':
format_list=test_saving.format2limaformat.keys()
format_list=test_saving.format_list
else:
format_list = args.format
format_list.sort()
......@@ -163,7 +167,8 @@ def main(argv):
args.overwrite,
args.framesperfile,
args.threads,
repeat)
repeat,
args.log_stat)
except Core.Exception, e:
raise RuntimeError
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment