Commit eb86354d authored by payno's avatar payno
Browse files

[pyhstcaller] fix pyhstcaller for oar

- add a variable OAR_UTILS to define the path to 'make_oar_pyhst2' like /data/id19/inhouse/OAR_UTILITIES/pyhst/make_oar_pyhst2
- add unit test for consistancy tomwer 0.2 / tomwer 0.4
- add unit test to check tomwer 0.4 and higher behavior

TODO: add a slurm option
parent 93069f02
......@@ -36,6 +36,7 @@ from tomwer.core.process.reconstruction.ftseries.params import ReconsParams
from tomwer.core.scan.scanbase import TomoBase
from tomwer.core.settings import get_lbsram_path, get_dest_path
from tomwer.core.log import TomwerLogger
from tomwer.core import settings
from tomwer.unitsystem import metricsystem
import socket
import subprocess
......@@ -1239,20 +1240,20 @@ class PyHSTCaller(object):
def loadFileInfoFrmPar(filePath):
info = 'Reading %s to get reconstruction parameters'
logger.info(info)
ofile = open(filePath, 'rb')
ofile = open(filePath, 'r')
lines = ofile.readlines()
outputFile = None
filePrefix = None
for line in lines:
l = str(line)
if 'OUTPUT_FILE' in l:
splitLine = l.split()
outputFile = splitLine[2]
splitLine = l.replace(' ', '').split('=')
outputFile = splitLine[1].rstrip('\n')
logger.info('Found OUTPUT_FILE: %s' % outputFile)
if 'FILE_PREFIX' in l:
splitLine = l.split()
filePrefix = splitLine[2]
splitLine = l.replace(' ', '').split('=')
filePrefix = splitLine[1].rstrip('\n')
logger.info('Found FILE_PREFIX: %s' % filePrefix)
ofile.close()
return outputFile, filePrefix
......@@ -1265,7 +1266,7 @@ class PyHSTCaller(object):
if os.path.isfile(par_file):
recFile, prefix = loadFileInfoFrmPar(par_file)
else:
err = 'No %s file found. Can\'t deduce reconstruction parameters.'
err = 'No %s file found. Can\'t deduce reconstruction parameters.' % par_file
raise ValueError(err)
if recFile is None or prefix is None:
......@@ -1285,7 +1286,9 @@ class PyHSTCaller(object):
prefix = prefix_full
# Get parameter for writing the script
utilsMakeOAR = recons_params['PYHSTEXE']['MAKE_OAR_FILE']
#utilsMakeOAR = recons_params['PYHSTEXE']['MAKE_OAR_FILE']
# For now hard coded
utilsMakeOAR = settings.MAKE_OAR_PYST2_PATH
logger.info(('Processing the file %s %s' % (prefix_full, prefix)))
......@@ -1301,25 +1304,25 @@ class PyHSTCaller(object):
fidshell.write(bytes('case "${os}" in\n', 'UTF-8'))
fidshell.write(bytes(' centos*)\n', 'UTF-8'))
if vol_output == 0:
fidshell.write(bytes(' mkdir ' + recFile + ' \n', 'UTF-8'))
fidshell.write(bytes(' chmod 775 ' + recFile + ' \n', 'UTF-8'))
fidshell.write(bytes(' mkdir ' + os.path.dirname(recFile) + ' \n', 'UTF-8'))
fidshell.write(bytes(' chmod 775 ' + os.path.dirname(recFile) + ' \n', 'UTF-8'))
fidshell.write(bytes(
' ' + str(utilsMakeOAR) + ' ' + prefix_full + ' 1\n',
' ' + str(utilsMakeOAR) + ' ' + os.path.basename(prefix_full) + ' 1\n',
'UTF-8'))
fidshell.write(bytes(
' oarsub -q gpu -S ./' + prefix_full + '.oar\n', 'UTF-8'))
' oarsub -q gpu -S ./' + os.path.basename(prefix_full) + '.oar\n', 'UTF-8'))
fidshell.write(bytes(' ;;\n', 'UTF-8'))
fidshell.write(bytes(' debian*)\n', 'UTF-8'))
if vol_output == 0:
fidshell.write(bytes(' mkdir ' + recFile + ' \n', 'UTF-8'))
fidshell.write(bytes(' mkdir ' + os.path.dirname(recFile) + ' \n', 'UTF-8'))
fidshell.write(bytes(
' ' + str(utilsMakeOAR) + ' ' + prefix_full + ' 1\n', 'UTF-8'))
' ' + str(utilsMakeOAR) + ' ' + os.path.basename(prefix_full) + ' 1\n', 'UTF-8'))
fidshell.write(bytes(
' oarsub -q gpu -S ./' + prefix_full + '.oar\n', 'UTF-8'))
' oarsub -q gpu -S ./' + os.path.basename(prefix_full) + '.oar\n', 'UTF-8'))
fidshell.write(bytes(' ;;\n', 'UTF-8'))
fidshell.write(bytes(' *)\n', 'UTF-8'))
......
......@@ -34,7 +34,7 @@ import tempfile
import unittest
import socket
from glob import glob
import filecmp
from tomwer.core.process.reconstruction.pyhstcaller import PyHSTCaller
from tomwer.core.utils.pyhstutils import _findPyHSTVersions, _getPyHSTDir
from tomwer.test.utils import UtilsTest, rebaseParFile
......@@ -53,7 +53,7 @@ pyhstVersion = _findPyHSTVersions(_getPyHSTDir())
@unittest.skipIf(len(pyhstVersion) is 0, "PyHST2 missing")
class TestRecHST(unittest.TestCase):
class TestRecHSTCreation(unittest.TestCase):
"""
Test that rec HST file are correctly processing
"""
......@@ -92,6 +92,75 @@ class TestRecHST(unittest.TestCase):
self.assertTrue(os.path.isfile(rec_file))
class TestRecHSTWrite(unittest.TestCase):
"""Test the creation of a .rec from a .par file"""
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
dataTestDir = UtilsTest.getDatasets('rec_HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_')
self.data_dir_tom2 = os.path.join(self.tmp_dir, 'rec_HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_', 'tomwer_02')
self.data_dir_tom4 = os.path.join(self.tmp_dir, 'rec_HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_', 'tomwer_04')
self._original_rec_file = os.path.join(dataTestDir, 'tomwer_02', 'HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_pag.rec')
shutil.copytree(os.path.join(dataTestDir, 'tomwer_02'), self.data_dir_tom2)
shutil.copytree(os.path.join(dataTestDir, 'tomwer_04'), self.data_dir_tom4)
self.recons_params = ReconsParams()
self._pyhstcaller = PyHSTCaller()
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def testOriginalRecWrite(self):
"""Check writing of a .rec file from an original .par file"""
generated_rec_file = os.path.join(self.data_dir_tom2, 'HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_pag.rec')
par_file = os.path.join(self.data_dir_tom2, 'HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_pag.par')
self._pyhstcaller.make_rec_file(rec_file=generated_rec_file,
par_file=par_file,
recons_params=self.recons_params)
self.assertTrue(filecmp.cmp(generated_rec_file,
self._original_rec_file, shallow=False))
def testNewRecWrite(self):
"""Check writing of a .rec file from a created .par file"""
generated_rec_file = os.path.join(self.data_dir_tom4, 'HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_.rec')
par_file = os.path.join(self.data_dir_tom4, 'HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_.par')
self._pyhstcaller.make_rec_file(rec_file=generated_rec_file,
par_file=par_file,
recons_params=self.recons_params)
def get_command(cmd, file_):
f = open(file_, "r")
lines = f.readlines()
for line in lines:
l = line.lstrip(' ').rstrip(' ')
l = l.rstrip('\n').rstrip(' ')
if l.startswith(cmd):
return l
def get_oar_command(file_):
return get_command('oarsub', file_)
def get_mkdir_command(file_):
return get_command('mkdir', file_)
def get_chmod_command(file_):
return get_command('chmod', file_)
self.assertEqual(get_oar_command(generated_rec_file),
'oarsub -q gpu -S ./HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_.oar')
self.assertEqual(str(get_mkdir_command(generated_rec_file)),
'mkdir /data/visitor/mi1226/id19/payno/no_backup/HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_')
self.assertEqual(get_chmod_command(generated_rec_file),
'chmod 775 /data/visitor/mi1226/id19/payno/no_backup/HA-700_45.4_136keV_BM5_burrow_GSA-1-44_021_avg-2_')
shutil.copyfile(src=generated_rec_file,
dst='/home/payno/tmp/test_ut/generated.rec')
shutil.copyfile(src=par_file,
dst='/home/payno/tmp/test_ut/original.par')
class TestParFile(unittest.TestCase):
"""Test the behavior of the `makeParFile` function"""
def tearDown(self):
......@@ -609,7 +678,7 @@ class TestLbsramBehavior(unittest.TestCase):
def suite():
test_suite = unittest.TestSuite()
for ui in (TestParFile, TestRecHST, TestUtils, TestSlices,
TestLbsramBehavior):
for ui in (TestParFile, TestRecHSTCreation, TestUtils, TestSlices,
TestLbsramBehavior, TestRecHSTWrite):
test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
......@@ -39,6 +39,9 @@ MOCK_LBSRAM = False
MAX_MEM_USED = 80
"Maximal percentage of used memory from which we should skip some processing"
MAKE_OAR_PYST2_PATH = '/data/id19/inhouse/OAR_UTILITIES/pyhst/make_oar_pyhst2'
"""path to the make_oar_pyhst2 path"""
def mock_lsbram(b):
assert type(b) is bool
......
......@@ -52,4 +52,4 @@ def _getPyHSTDir():
elif os.path.isdir('/usr/bin/'):
return '/usr/bin/'
else:
return None
\ No newline at end of file
return None
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment