Commit 84fd257d authored by Alejandro Homs Puron's avatar Alejandro Homs Puron Committed by operator for beamline
Browse files

Add multiple processing threads per recv. port (currently 4)

parent c70492ab
......@@ -1743,45 +1743,45 @@ local database:
Add LimaCCDs and SlsDetector class devices.
+----------------------------------------------------------+-----------------------------------------------+
| LimaCCDs/eiger500k/DEVICE/LimaCCDs | id10/limaccds/eiger500k |
+----------------------------------------------------------+-----------------------------------------------+
| id10/limaccds/eiger500k->LimaCameraType | SlsDetector |
+----------------------------------------------------------+-----------------------------------------------+
| id10/limaccds/eiger500k->NbProcessingThread | 23 |
+----------------------------------------------------------+-----------------------------------------------+
| id10/limaccds/eiger500k->BufferMaxMemory | 30 |
+----------------------------------------------------------+-----------------------------------------------+
| LimaCCDs/eiger500k/DEVICE/SlsDetector | id10/slsdetector/eiger500k |
+----------------------------------------------------------+-----------------------------------------------+
| id10/slsdetector/eiger500k->config_fname | /users/opid00/eiger/eiger_v3.1.1/config/ |
| | beb-021-020-direct-FO-10g.config |
+----------------------------------------------------------+-----------------------------------------------+
| id10/slsdetector/eiger500k->pixel_depth_cpu_affinity_map | | { 4: ((((CPU( 6), CPU(18), CPU( 8, 20)), |
| | (CPU( 7), CPU(19), CPU( 8, 20))), |
| | | ((CPU( 9), CPU(21), CPU(11, 23)), |
| | (CPU(10), CPU(22), CPU(11, 23)))), |
| | | CPU(*(range( 1, 6) + |
| | range(12, 18))), |
| | | CPU(0), |
| | | (('eth0,eth1,eth2,eth4,eth6,eth7, |
| | eth8,eth9', |
| | {-1: (CPU(0), CPU(0))}), |
| | | ('eth3', |
| | {-1: (CPU( 8, 20), |
| | CPU( 8, 20))}), |
| | | ('eth5', |
| | {-1: (CPU(11, 23), |
| | CPU(11, 23))}))), |
| | | 8: '@4', |
| | | 16: '@4', |
| | | 32: '@4'} |
+----------------------------------------------------------+-----------------------------------------------+
+----------------------------------------------------------+------------------------------------------------------+
| LimaCCDs/eiger500k/DEVICE/LimaCCDs | id10/limaccds/eiger500k |
+----------------------------------------------------------+------------------------------------------------------+
| id10/limaccds/eiger500k->LimaCameraType | SlsDetector |
+----------------------------------------------------------+------------------------------------------------------+
| id10/limaccds/eiger500k->NbProcessingThread | 23 |
+----------------------------------------------------------+------------------------------------------------------+
| id10/limaccds/eiger500k->BufferMaxMemory | 30 |
+----------------------------------------------------------+------------------------------------------------------+
| LimaCCDs/eiger500k/DEVICE/SlsDetector | id10/slsdetector/eiger500k |
+----------------------------------------------------------+------------------------------------------------------+
| id10/slsdetector/eiger500k->config_fname | /users/opid00/eiger/eiger_v3.1.1/config/ |
| | beb-021-020-direct-FO-10g.config |
+----------------------------------------------------------+------------------------------------------------------+
| id10/slsdetector/eiger500k->pixel_depth_cpu_affinity_map | | { 4: ((((CPU( 6), CPU(18), (CPU( 8), CPU(20), |
| | CPU( 0), CPU( 1))), |
| | | (CPU( 7), CPU(19), (CPU( 2), CPU( 3), |
| | CPU( 4), CPU( 5)))), |
| | | ((CPU( 9), CPU(21), (CPU(11), CPU(23), |
| | CPU(12), CPU(13))), |
| | | (CPU(10), CPU(22), (CPU(14), CPU(15), |
| | CPU(16), CPU(17))))), |
| | | CPU(*(range( 1, 6) + range(12, 18))), |
| | | CPU(0), |
| | | (('eth0,eth1,eth2,eth4,eth6,eth7,eth8,eth9', |
| | {-1: (CPU(0), CPU(0))}), |
| | | ('eth3', |
| | {-1: (CPU( 8, 20), CPU( 8, 20))}), |
| | | ('eth5', |
| | {-1: (CPU(11, 23), CPU(11, 23))}))), |
| | | 8: '@4', |
| | | 16: '@4', |
| | | 32: '@4'} |
+----------------------------------------------------------+------------------------------------------------------+
.. note:: in order to perform high frame rate acquisitions, the CPU affinity must be fixed for
the following tasks:
* Receivers ports (2x2=4): listeners, writers, port_threads
* Receivers ports (2x2=4): listeners, writers, port_threads (4 per port)
* Lima processing threads
* OS processes
* Net-dev group packet dispatching for Rx queues: irq & processing
......
......@@ -131,12 +131,14 @@ class Eiger : public Model
virtual void processRecvFileStart(uint32_t dsize);
virtual void processRecvPort(FrameType frame, char *dptr,
uint32_t dsize, char *bptr);
virtual bool hasPortThreadProcessing();
virtual void processPortThread(FrameType frame, char *bptr);
virtual int getNbPortProcessingThreads();
virtual void processPortThread(FrameType frame, char *bptr,
int thread_idx);
void expandPixelDepth4(FrameType frame, char *ptr);
private:
void copy2LimaBuffer(char *dptr, char *bptr);
bool hasPortThreadProcessing();
void copy2LimaBuffer(char *dptr, char *bptr, int thread_idx);
Eiger *m_eiger;
int m_port;
......@@ -145,10 +147,14 @@ class Eiger : public Model
bool m_raw;
int m_recv_idx;
int m_port_offset;
int m_ilw; // image line width
int m_slw; // source line width
int m_dlw; // dest line width
int m_scw; // source chip width
int m_dcw; // dest chip width
int m_pchips;
int m_copy_lines;
int m_sto; // source thread offset
int m_dto; // dest thread offset
int m_nb_buffers;
NumaSoftBufferAllocMgr m_buffer_alloc_mgr;
StdBufferCbMgr m_buffer_cb_mgr;
......@@ -370,6 +376,7 @@ class Eiger : public Model
static const int ChipGap;
static const int HalfModuleChips;
static const int NbRecvPorts;
static const int NbPortThreads;
struct LinScale {
double factor, offset;
......
......@@ -53,8 +53,9 @@ class Model
// TODO: add file finished callback
virtual void processRecvPort(FrameType frame, char *dptr,
uint32_t dsize, char *bptr) = 0;
virtual bool hasPortThreadProcessing() = 0;
virtual void processPortThread(FrameType frame, char *bptr) = 0;
virtual int getNbPortProcessingThreads() = 0;
virtual void processPortThread(FrameType frame, char *bptr,
int thread_idx) = 0;
};
typedef Defs::Settings Settings;
......
......@@ -71,8 +71,8 @@ private:
Port(Receiver& recv, int port);
pid_t getThreadID()
{ return m_thread.getThreadID(); }
pid_t getThreadID(int idx)
{ return m_thread_list[idx].getThreadID(); }
void prepareAcq();
......@@ -97,6 +97,9 @@ private:
Stats& getStats()
{ return m_stats; }
void setNbThreads(int nb_threads);
int getNbThreads();
private:
friend class Receiver;
......@@ -112,35 +115,39 @@ private:
"Receiver::Port::Thread",
"SlsDetector");
public:
Thread(Port& port);
virtual ~Thread();
virtual void start();
void init(Port *port, int idx);
protected:
virtual void start();
virtual void threadFunction();
private:
Port& m_port;
Port *m_port;
int m_idx;
pid_t m_tid;
volatile bool m_end;
};
typedef std::vector<Thread> ThreadList;
typedef std::vector<FrameMap::Item *> FrameMapItemList;
AutoMutex lock()
{ return m_mutex; }
void pollFrameFinished();
void pollFrameFinished(int thread_idx);
void stopPollFrameFinished();
void processFinishInfo(const FinishInfo& finfo);
Camera *m_cam;
Model *m_model;
int m_port_idx;
Model::RecvPort *m_model_port;
Mutex m_mutex;
FrameMap::Item *m_frame_map_item;
FrameMapItemList m_frame_map_item_list;
IntList m_bad_frame_list;
Stats m_stats;
Thread m_thread;
ThreadList m_thread_list;
};
typedef std::vector<AutoPtr<Port> > PortList;
......
......@@ -39,9 +39,9 @@ public:
virtual void processRecvFileStart(unsigned int dsize) = 0;
virtual void processRecvPort(unsigned long frame, char *dptr,
unsigned int dsize, char *bptr) = 0;
virtual bool hasPortThreadProcessing() = 0;
virtual void processPortThread(unsigned long frame,
char *bptr) = 0;
virtual int getNbPortProcessingThreads() = 0;
virtual void processPortThread(unsigned long frame, char *bptr,
int thread_idx) = 0;
};
Model(SlsDetector::Camera *cam, SlsDetector::Type type);
......
......@@ -399,12 +399,18 @@ void Camera::setModel(Model *model)
m_nb_recv_ports = m_model->getNbRecvPorts();
int nb_ports = getTotNbPorts();
m_frame_map.setNbItems(nb_ports);
Model::RecvPort *port = m_model->getRecvPort(0);
int nb_port_threads = max(1, port->getNbPortProcessingThreads());
m_frame_map.setNbItems(nb_ports * nb_port_threads);
RecvList::iterator it, end = m_recv_list.end();
for (it = m_recv_list.begin(); it != end; ++it)
(*it)->setNbPorts(m_nb_recv_ports);
RecvPortList port_list = getRecvPortList();
for (int i = 0; i < getTotNbPorts(); ++i)
port_list[i]->setNbThreads(nb_port_threads);
setPixelDepth(m_pixel_depth);
setSettings(m_settings);
}
......
......@@ -32,6 +32,7 @@ const int Eiger::ChipSize = 256;
const int Eiger::ChipGap = 2;
const int Eiger::HalfModuleChips = 4;
const int Eiger::NbRecvPorts = 2;
const int Eiger::NbPortThreads = 4;
const int Eiger::BitsPerXfer = 4;
const int Eiger::SuperColNbCols = 8;
......@@ -228,7 +229,7 @@ void Eiger::RecvPort::prepareAcq()
const FrameDim& frame_dim = m_eiger->m_recv_frame_dim;
const Size& size = frame_dim.getSize();
int depth = frame_dim.getDepth();
m_ilw = size.getWidth() * depth;
m_dlw = size.getWidth() * depth;
int recv_size = frame_dim.getMemSize();
m_port_offset = recv_size * m_recv_idx;
......@@ -237,11 +238,12 @@ void Eiger::RecvPort::prepareAcq()
m_dcw = m_scw;
if (m_eiger->isPixelDepth4())
m_scw /= 2;
m_slw = m_pchips * m_scw;
m_eiger->getCamera()->getRawMode(m_raw);
if (m_raw) {
// vert. port concat.
m_port_offset += ChipSize * m_ilw * m_port;
m_port_offset += ChipSize * m_dlw * m_port;
return;
} else {
// inter-chip horz. gap
......@@ -252,30 +254,35 @@ void Eiger::RecvPort::prepareAcq()
int mod_idx = m_recv_idx / 2;
for (int i = 0; i < mod_idx; ++i)
m_port_offset += m_eiger->getInterModuleGap(i) * m_ilw;
m_port_offset += m_eiger->getInterModuleGap(i) * m_dlw;
if (m_top_half_recv) {
// top-half module: vert-flipped data
m_port_offset += (ChipSize - 1) * m_ilw;
m_ilw *= -1;
m_port_offset += (ChipSize - 1) * m_dlw;
m_dlw *= -1;
} else {
// bottom-half module: inter-chip vert. gap
m_port_offset += (ChipGap / 2) * m_ilw;
m_port_offset += (ChipGap / 2) * m_dlw;
}
m_copy_lines = ChipSize;
if (hasPortThreadProcessing()) {
CPUAffinity buffer_aff = 0x6c0000;
m_buffer_alloc_mgr.setCPUAffinityMask(buffer_aff);
int nb_concat_frames = 8;
int nb_buffers = m_nb_buffers / nb_concat_frames;
FrameDim fdim(m_scw * m_pchips, ChipSize, Bpp8);
FrameDim fdim(m_slw, ChipSize, Bpp8);
m_buffer_cb_mgr.allocBuffers(nb_buffers, nb_concat_frames, fdim);
m_copy_lines /= NbPortThreads;
m_sto = m_copy_lines * m_slw;
m_dto = m_copy_lines * m_dlw;
DEB_TRACE() << DEB_VAR3(m_copy_lines, m_sto, m_dto);
m_last_recv_frame = m_last_proc_frame = -1;
m_overrun = false;
} else {
m_buffer_cb_mgr.releaseBuffers();
}
}
void Eiger::RecvPort::processRecvFileStart(uint32_t dsize)
......@@ -291,7 +298,7 @@ void Eiger::RecvPort::processRecvPort(FrameType frame, char *dptr,
DEB_PARAM() << DEB_VAR3(frame, m_recv_idx, m_port);
if (!hasPortThreadProcessing()) {
copy2LimaBuffer(dptr, bptr);
copy2LimaBuffer(dptr, bptr, 0);
return;
}
......@@ -317,12 +324,14 @@ void Eiger::RecvPort::processRecvPort(FrameType frame, char *dptr,
m_last_recv_frame = frame;
}
void Eiger::RecvPort::copy2LimaBuffer(char *dptr, char *bptr)
void Eiger::RecvPort::copy2LimaBuffer(char *dptr, char *bptr, int thread_idx)
{
DEB_MEMBER_FUNCT();
bool valid_data = (dptr != NULL);
char *src = dptr;
char *dest = bptr + m_port_offset;
for (int i = 0; i < ChipSize; ++i, dest += m_ilw) {
char *src = dptr + thread_idx * m_sto;
char *dest = bptr + m_port_offset + thread_idx * m_dto;
for (int i = 0; i < m_copy_lines; ++i, dest += m_dlw) {
char *d = dest;
for (int j = 0; j < m_pchips; ++j, src += m_scw, d += m_dcw)
if (valid_data)
......@@ -340,13 +349,25 @@ bool Eiger::RecvPort::hasPortThreadProcessing()
return thread_proc;
}
void Eiger::RecvPort::processPortThread(FrameType frame, char *bptr)
int Eiger::RecvPort::getNbPortProcessingThreads()
{
DEB_MEMBER_FUNCT();
DEB_PARAM() << DEB_VAR3(frame, m_recv_idx, m_port);
int nb_proc_threads = NbPortThreads;
DEB_RETURN() << DEB_VAR1(nb_proc_threads);
return nb_proc_threads;
}
void Eiger::RecvPort::processPortThread(FrameType frame, char *bptr,
int thread_idx)
{
DEB_MEMBER_FUNCT();
DEB_PARAM() << DEB_VAR4(frame, m_recv_idx, m_port, thread_idx);
if (!hasPortThreadProcessing())
return;
char *dptr = (char *) m_buffer_cb_mgr.getFrameBufferPtr(frame);
copy2LimaBuffer(dptr, bptr);
copy2LimaBuffer(dptr, bptr, thread_idx);
m_last_proc_frame = frame;
}
......@@ -356,7 +377,7 @@ void Eiger::RecvPort::expandPixelDepth4(FrameType frame, char *ptr)
DEB_PARAM() << DEB_VAR3(frame, m_recv_idx, m_port);
ptr += m_port_offset;
for (int i = 0; i < ChipSize; ++i, ptr += m_ilw) {
for (int i = 0; i < ChipSize; ++i, ptr += m_dlw) {
char *chip = ptr;
for (int j = 0; j < m_pchips; ++j, chip += m_dcw) {
char *src = chip + m_scw;
......
......@@ -26,13 +26,6 @@ using namespace std;
using namespace lima;
using namespace lima::SlsDetector;
Receiver::Port::Thread::Thread(Port& port)
: m_port(port)
{
DEB_CONSTRUCTOR();
}
Receiver::Port::Thread::~Thread()
{
DEB_DESTRUCTOR();
......@@ -41,7 +34,15 @@ Receiver::Port::Thread::~Thread()
return;
m_end = true;
m_port.stopPollFrameFinished();
m_port->stopPollFrameFinished();
}
void Receiver::Port::Thread::init(Port *port, int idx)
{
DEB_MEMBER_FUNCT();
m_port = port;
m_idx = idx;
start();
}
void Receiver::Port::Thread::start()
......@@ -67,20 +68,40 @@ void Receiver::Port::Thread::threadFunction()
m_end = false;
while (!m_end)
m_port.pollFrameFinished();
m_port->pollFrameFinished(m_idx);
}
Receiver::Port::Port(Receiver& recv, int port)
: m_thread(*this)
{
DEB_CONSTRUCTOR();
m_cam = recv.m_cam;
m_model = m_cam->m_model;
m_port_idx = m_cam->getPortIndex(recv.m_idx, port);
m_frame_map_item = &m_cam->m_frame_map.getItem(m_port_idx);
m_thread.start();
m_model_port = m_cam->m_model->getRecvPort(m_port_idx);
}
void Receiver::Port::setNbThreads(int nb_threads)
{
DEB_MEMBER_FUNCT();
int prev_nb_threads = m_thread_list.size();
DEB_PARAM() << DEB_VAR2(nb_threads, prev_nb_threads);
if (nb_threads == prev_nb_threads)
return;
m_frame_map_item_list.resize(nb_threads);
int item = m_port_idx * nb_threads;
for (int i = 0; i < nb_threads; ++i, ++item)
m_frame_map_item_list[i] = &m_cam->m_frame_map.getItem(item);
m_thread_list.resize(nb_threads);
for (int i = prev_nb_threads; i < nb_threads; ++i)
m_thread_list[i].init(this, i);
}
int Receiver::Port::getNbThreads()
{
return m_thread_list.size();
}
void Receiver::Port::prepareAcq()
......@@ -94,44 +115,46 @@ void Receiver::Port::prepareAcq()
void Receiver::Port::processFileStart(uint32_t dsize)
{
DEB_MEMBER_FUNCT();
Model::RecvPort *port = m_model->getRecvPort(m_port_idx);
port->processRecvFileStart(dsize);
m_model_port->processRecvFileStart(dsize);
}
void Receiver::Port::processFrame(FrameType frame, char *dptr, uint32_t dsize)
{
DEB_MEMBER_FUNCT();
m_frame_map_item->checkFinishedFrame(frame);
FrameMapItemList::iterator it, end = m_frame_map_item_list.end();
for (it = m_frame_map_item_list.begin(); it != end; ++it)
(*it)->checkFinishedFrame(frame);
bool valid = (dptr != NULL);
if (valid) {
char *bptr = m_cam->getFrameBufferPtr(frame);
Model::RecvPort *port = m_model->getRecvPort(m_port_idx);
port->processRecvPort(frame, dptr, dsize, bptr);
m_model_port->processRecvPort(frame, dptr, dsize, bptr);
}
Timestamp t0 = Timestamp::now();
m_frame_map_item->frameFinished(frame, true, valid);
for (it = m_frame_map_item_list.begin(); it != end; ++it)
(*it)->frameFinished(frame, true, valid);
Timestamp t1 = Timestamp::now();
m_stats.stats.new_finish.add(t1 - t0);
}
void Receiver::Port::pollFrameFinished()
void Receiver::Port::pollFrameFinished(int thread_idx)
{
DEB_MEMBER_FUNCT();
FrameDataList data_list = m_frame_map_item->pollFrameFinished();
Model::RecvPort *port = m_model->getRecvPort(m_port_idx);
if (port->hasPortThreadProcessing()) {
FrameMap::Item *frame_map_item = m_frame_map_item_list[thread_idx];
FrameDataList data_list = frame_map_item->pollFrameFinished();
if (m_model_port->getNbPortProcessingThreads()) {
FrameDataList::const_iterator it, end = data_list.end();
for (it = data_list.begin(); it != end; ++it) {
FrameType frame = it->first;
char *bptr = m_cam->getFrameBufferPtr(frame);
port->processPortThread(frame, bptr);
m_model_port->processPortThread(frame, bptr,
thread_idx);
}
}
FinishInfoList finfo_list;
finfo_list = m_frame_map_item->getFrameFinishInfo(data_list);
finfo_list = frame_map_item->getFrameFinishInfo(data_list);
FinishInfoList::const_iterator it, end = finfo_list.end();
for (it = finfo_list.begin(); it != end; ++it) {
const FinishInfo& finfo = *it;
......@@ -143,7 +166,9 @@ void Receiver::Port::pollFrameFinished()
void Receiver::Port::stopPollFrameFinished()
{
DEB_MEMBER_FUNCT();
m_frame_map_item->stopPollFrameFinished();
FrameMapItemList::iterator it, end = m_frame_map_item_list.end();
for (it = m_frame_map_item_list.begin(); it != end; ++it)
(*it)->stopPollFrameFinished();
}
void Receiver::Port::processFinishInfo(const FinishInfo& finfo)
......@@ -236,21 +261,35 @@ void Receiver::setCPUAffinity(const RecvCPUAffinity& recv_affinity)
Listener, Writer, PortThread, NbThreadTypes,
};
Model *model = m_cam->m_model;
Model::RecvPort *port = model->getRecvPort(0);
int nb_proc_threads = port->getNbPortProcessingThreads();
class AffinityListHelper
{
public:
AffinityListHelper(const CPUAffinityList& aff_list,
int nb_ports, ThreadType type,
int recv_idx, DebObj *deb_ptr)
int nb_ports, int nb_proc_threads,
ThreadType type, int recv_idx,
DebObj *deb_ptr)
: m_aff_list(aff_list), m_nb_ports(nb_ports),
m_nb_proc_threads(nb_proc_threads),
m_type(type), m_recv_idx(recv_idx), m_deb_ptr(deb_ptr)
{
DEB_FROM_PTR(m_deb_ptr);
unsigned int nb_aff = m_aff_list.size();
if (!defaultAffinity() && !singleAffinity() &&
(nb_aff != m_nb_ports))
(nb_aff != getNbAffinities()))
THROW_HW_ERROR(InvalidValue) <<
DEB_VAR2(nb_aff, m_nb_ports);
DEB_VAR2(nb_aff, getNbAffinities());
}
int getNbAffinities() const
{
int nb_aff = m_nb_ports;
if (m_type == PortThread)
nb_aff *= m_nb_proc_threads;
return nb_aff;
}
bool defaultAffinity() const
......@@ -284,17 +323,17 @@ void Receiver::setCPUAffinity(const RecvCPUAffinity& recv_affinity)
{
DEB_FROM_PTR(m_deb_ptr);
CPUAffinityList list;
int nb_aff = getNbAffinities();
if (defaultAffinity())
list = CPUAffinityList(m_nb_ports);
else if (singleAffinity() && (m_nb_ports > 1))
list = CPUAffinityList(m_nb_ports,
m_aff_list[0]);
list = CPUAffinityList(nb_aff);
else if (singleAffinity() && (nb_aff > 1))
list = CPUAffinityList(nb_aff, m_aff_list[0]);
else
list = m_aff_list;
string deb_head = getDebHead();
CPUAffinityList::const_iterator it = list.begin();
for (int i = 0; i < m_nb_ports; ++i, ++it)
for (int i = 0; i < nb_aff; ++i, ++it)
DEB_TRACE() << deb_head << i << " "
<< "CPU mask to " << *it;
......@@ -317,6 +356,7 @@ void Receiver::setCPUAffinity(const RecvCPUAffinity& recv_affinity)
private:
const CPUAffinityList& m_aff_list;
int m_nb_ports;
int m_nb_proc_threads;
ThreadType m_type;
int m_recv_idx;
DebObj *m_deb_ptr;
......@@ -325,7 +365,7 @@ void Receiver::setCPUAffinity(const RecvCPUAffinity& recv_affinity)
int nb_ports = m_port_list.size();
#define CreateHelper(n, a, t) \
AffinityListHelper n(a, nb_ports, t, m_idx, DEB_PTR());
AffinityListHelper n(a, nb_ports, nb_proc_threads, t, m_idx, DEB_PTR());
CreateHelper(lh, recv_affinity.listeners, Listener);
CreateHelper(wh, recv_affinity.writers, Writer);
......@@ -346,9 +386,11 @@ void Receiver::setCPUAffinity(const RecvCPUAffinity& recv_affinity)
CPUAffinityList port_thread_aff_list = pth.getThreadAffinityList();
CPUAffinityList::const_iterator tit = port_thread_aff_list.begin();
PortList::iterator pit, pend = m_port_list.end();
for (pit = m_port_list.begin(); pit != pend; ++pit, ++tit) {
pid_t tid = (*pit)->getThreadID();
(*tit).applyToTask(tid, false);
for (pit = m_port_list.begin(); pit != pend; ++pit) {
for (int i = 0; i < (*pit)->getNbThreads(); ++i, ++tit) {