Commit 2e9f5035 authored by Samuel Debionne's avatar Samuel Debionne
Browse files

Update doc & test

parents 2603c30f edef17f1
......@@ -216,8 +216,8 @@ the entire disk:
(parted) quit
Information: You may need to update /etc/fstab.
Note: the partition is aligned to 2048 sectors (1 MByte). The end sector
is obtained by:
.. note:: the partition is aligned to 2048 sectors (1 MByte). The end sector
is obtained by:
::
......@@ -289,22 +289,85 @@ the highest priority:
lid10eiger1:~ # cat /etc/security/limits.d/net-performance.conf
@netperf - rtprio 99
and also to execute *sudo* in order to change other tasks' CPU affinity
(*taskset*) and configure configure the network devices (*ethtool* and
Compile the *netdev_set_queue_rps_cpus* util, used by the *SlsDetector* plugin
to change the network packet dispatching tasks' CPU affinity, and install it
in */usr/local/bin*:
::
lid10eiger1:~ # cat /tmp/netdev_set_queue_rps_cpus.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
int main(int argc, char *argv[])
{
char *dev, *queue, *p, fname[256], buffer[128];
int fd, len, ret;
long aff;
if (argc != 4)
exit(1);
if (!strlen(argv[1]) || !strlen(argv[2]) || !strlen(argv[3]))
exit(2);
dev = argv[1];
queue = argv[2];
errno = 0;
aff = strtol(argv[3], &p, 0);
if (errno || *p)
exit(3);
len = sizeof(fname);
ret = snprintf(fname, len, "/sys/class/net/%s/queues/%s/rps_cpus",
dev, queue);
if ((ret < 0) || (ret == len))
exit(4);
len = sizeof(buffer);
ret = snprintf(buffer, len, "%016lx", aff);
if ((ret < 0) || (ret == len))
exit(5);
fd = open(fname, O_WRONLY);
if (fd < 0)
exit(6);
for (p = buffer; *p; p += ret)
if ((ret = write(fd, p, strlen(p))) < 0)
exit(7);
if (close(fd) < 0)
exit(8);
return 0;
}
lid10eiger1:~ # gcc -Wall -o /tmp/netdev_set_queue_rps_cpus /tmp/netdev_set_queue_rps_cpus.c
lid10eiger1:~ # cp /tmp/netdev_set_queue_rps_cpus /usr/local/bin
Allow *netperf* users to execute *sudo* in order to change other tasks' CPU affinity
(*taskset* and *netdev_set_queue_rps_cpus*) and to configure the network devices (*ethtool* and
*ifconfig*):
::
lid10eiger1:~ # cat /etc/sudoers.d/netperf
%netperf ALL=(root) NOPASSWD: /usr/bin/taskset, /sbin/ethtool, \
/sbin/ifconfig
%netperf ALL=(root) NOPASSWD: /usr/bin/taskset, /sbin/ethtool, \
/sbin/ifconfig, \
/usr/local/bin/netdev_set_queue_rps_cpus
Tune the OS network buffer sizes:
::
lid10eiger1:~ # cat /etc/sysctl.d/net-performance.conf
# Receive buffers
# Tune network buffers for UDP RX performance
# Original values: sysctl -a | grep net
#...
......@@ -323,16 +386,12 @@ Tune the OS network buffer sizes:
#net.ipv4.udp_rmem_min = 4096
#net.ipv4.udp_wmem_min = 4096
# Max & default OS receive buffer size (Bytes) for all types
# Max OS socket receive buffer size (in bytes) for all types
net.core.rmem_max = 134217728
net.core.rmem_default = 134217728
# Buffer before Linux Kernel processes them
# Size of per-device buffer (in packets) before Linux kernel dispatching
net.core.netdev_max_backlog = 262144
# The minimum UDP receive buffer size (Bytes)
net.ipv4.udp_rmem_min = 134217728
Network configuration
---------------------
......@@ -380,8 +439,8 @@ Then check the PCI tree:
| \-00.1
...
From the tree we identify the parent root device of each Intel X520
Ethernet adapter:
From the tree we identify the parent root device of each *Intel X520
Ethernet adapter*:
+--------------+--------------+
| Node | Parent |
......@@ -1132,12 +1191,13 @@ plugin:
Compile *Lima*, including *slsDetectorPackage* using *CMake*:
::
(bliss) lid10eiger1:~ % cd ${LIMA_DIR}
(bliss) lid10eiger1:~/esrf/sls_detectors/Lima % mkdir -p ${LIMA_DIR}/install/python
(bliss) lid10eiger1:~/esrf/sls_detectors/Lima % ./install.sh \
--install-prefix=${LIMA_DIR}/install \
--install-python-prefix=${LIMA_DIR}/install/python \
slsdetector sps gldisplay edfgz python pytango-server
slsdetector sps-image gldisplay edfgz python pytango-server
...
Add *Lima* to the *PATH*, *LD_LIBRARY_PATH* and *PYTHONPATH* environment variables in
......@@ -1281,29 +1341,49 @@ local database:
(bliss) lid10eiger1:~ % bliss_dserver -fg start LimaCCDs
Starting: LimaCCDs/eiger500k
Add LimaCCDs and SlsDetector class devices:
+-----------------------------------+-----------------------------------+
| LimaCCDs/eiger500k/DEVICE/LimaCCD | id10/limaccds/eiger500k |
| s | |
+-----------------------------------+-----------------------------------+
| id10/limaccds/eiger500k->LimaCame | SlsDetector |
| raType | |
+-----------------------------------+-----------------------------------+
| id10/limaccds/eiger500k->NbProces | 10 |
| singThread | |
+-----------------------------------+-----------------------------------+
| LimaCCDs/eiger500k/DEVICE/SlsDete | id10/slsdetector/eiger500k |
| ctor | |
+-----------------------------------+-----------------------------------+
| id10/slsdetector/eiger500k->confi | /users/opid00/eiger/eiger_v2.3.2/ |
| g_fname | config/beb-021-020-direct-FO-10g. |
| | config |
+-----------------------------------+-----------------------------------+
Add LimaCCDs and SlsDetector class devices.
+----------------------------------------------------------+-------------------------------------------+
| LimaCCDs/eiger500k/DEVICE/LimaCCDs | id10/limaccds/eiger500k |
+----------------------------------------------------------+-------------------------------------------+
| id10/limaccds/eiger500k->LimaCameraType | SlsDetector |
+----------------------------------------------------------+-------------------------------------------+
| id10/limaccds/eiger500k->NbProcessingThread | 11 |
+----------------------------------------------------------+-------------------------------------------+
| LimaCCDs/eiger500k/DEVICE/SlsDetector | id10/slsdetector/eiger500k |
+----------------------------------------------------------+-------------------------------------------+
| id10/slsdetector/eiger500k->config_fname | /users/opid00/eiger/eiger_v2.3.2/config/ |
| | beb-021-020-direct-FO-10g.config |
+----------------------------------------------------------+-------------------------------------------+
| id10/slsdetector/eiger500k->netdev_groups | | eth0,eth1,eth2,eth4,eth6,eth7,eth8,eth9 |
| | | eth3,eth5 |
+----------------------------------------------------------+-------------------------------------------+
| id10/slsdetector/eiger500k->pixel_depth_cpu_affinity_map | | 4,0xf00,0xfc,0x2,0x1,0x1,0x2 |
| | | 8,0xf00,0xfc,0x2,0x1,0x1,0x2 |
| | | 16,0xfff,0xfff,0xfff,0xfff,0xfff,0xfff |
| | | 32,0xfff,0xfff,0xfff,0xfff,0xfff,0xfff |
+----------------------------------------------------------+-------------------------------------------+
.. note:: in order to perform high frame rate acquisitions, the CPU affinity must be fixed for
the following tasks:
* Receiver listeners
* Receiver writers
* Lima processing threads
* OS processes
* Net-dev group #1 packet dispatching
* Net-dev group #2 packet dispatching
* ...
The previous example is based on a dual 6-core CPUs backend (12 cores). After the data acquisition finishes
the Lima processing threads will run also on the CPUs assigned to listeners and writers (0xffe), that is
11 cores in total, which is used for setting the NbProcessingThreads. Please note that there are two network
groups and four pixel_depth->cpu_affinity settings (4-, 8-, 16- and 32-bit), each one represented by a line
in a multi-line string array.
Finally, configure *opid10* as the default *DSERVER_USER*, which is used
by the *dserver_daemon*
::
# as blissadm
......@@ -1323,13 +1403,14 @@ and restart the *blcontrol* subsystem:
BL control ...
...
**Note**: when running the *LimaCCDs* server from the *dserver_daemon*,
the real-time priority capabilities are lost. **The server must be
started using the *bliss_dserver* script**. Moreover, **the command
*bliss_dserver start* must be used (start in background, avoid *-fg*
option)**, so the *LimaCCDs* process is decoupled from the terminal,
reducing the risks of CPU blocking.
.. note:: the latest version of the *daemon_adm* package allows the
propagation of the real-time priority capabilities configured as
resource limits, so **it is safe** to start the server through the
*dserver* remote utility. **If the command *bliss_dserver start* is
used, start the server in background and avoid *-fg* option**, so the
*LimaCCDs* process is decoupled from the terminal, reducing the
risks of CPU blocking.
SPEC
----
......@@ -1437,12 +1518,13 @@ interfaces.
#########################
if (index(ccd_dev, "eiger500k") > 0) {
limaccdsetup eiger500k ccd_u id10/limaccds/eiger500k
taco_io(ccd_dev, "timeout", 30)
taco_io(ccd_dev, "timeout", 30)
tango_io("id10/slsdetector/eiger500k", "timeout", 30)
}
}
}'
lima_ccd_resetup_all
***Note:*** The 30 seconds timeout is necessary for large memory
allocations (long sequences)
.. note:: the 30 seconds timeout is necessary for large memory
allocations (long sequences)
......@@ -108,7 +108,7 @@ class SlsDetectorAcq:
self.m_acq_state = Core.AcqState()
self.m_corr = None
if self.m_cam.getType() == SlsDetector.Camera.EigerDet:
if self.m_cam.getType() == SlsDetector.EigerDet:
self.m_eiger = SlsDetector.Eiger(self.m_cam)
self.m_corr = self.m_eiger.createCorrectionTask()
else:
......@@ -196,7 +196,7 @@ class SlsDetectorAcq:
time.sleep(self.m_sleep_time)
self.m_cam.waitState(SlsDetector.Camera.Idle);
self.m_cam.waitState(SlsDetector.Idle);
deb.Trace("Camera finished");
pool_thread_mgr = processlib.PoolThreadMgr.get()
......@@ -229,6 +229,16 @@ class SlsDetectorAcq:
def setBin(self, bin):
self.m_ct_image.setBin(bin)
@Core.DEB_MEMBER_FUNCT
def checkValidRoi(self, roi):
frame_dim = self.m_ct_image.getImageDim()
valid = True
try:
frame_dim.checkValidRoi(roi)
except:
valid = False
return valid
@Core.DEB_MEMBER_FUNCT
def setRoi(self, roi):
self.m_ct_image.setRoi(roi)
......@@ -278,14 +288,15 @@ def test_slsdetector_control(config_fname, enable_debug, use_events,
deb.Always("Done!")
roi = Core.Roi(Core.Point(256, 512), Core.Size(256, 512));
acq.setRoi(roi);
roi_tl, roi_size = roi.getTopLeft(), roi.getSize()
deb.Always("Run roi=<%s,%s>-<%sx%s>" %
(roi_tl.x, roi_tl.y,
roi_size.getWidth(), roi_size.getHeight()))
acq.run()
deb.Always("Done!")
if acq.checkValidRoi(roi):
acq.setRoi(roi);
roi_tl, roi_size = roi.getTopLeft(), roi.getSize()
deb.Always("Run roi=<%s,%s>-<%sx%s>" %
(roi_tl.x, roi_tl.y,
roi_size.getWidth(), roi_size.getHeight()))
acq.run()
deb.Always("Done!")
def main(argv):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment