Commit 65430774 authored by Wout De Nolf's avatar Wout De Nolf

[pep8]

parent 36d5bad0
......@@ -172,28 +172,28 @@ class Job(object):
"""Effective execution time, excluding queue time
"""
dt = self._time_diff('stopTime', 'startTime')
return max(dt,datetime.timedelta(seconds=0))
return max(dt, datetime.timedelta(seconds=0))
@property
def time_scheduled(self):
"""Time this job was scheduled, excluding queue and runtime
"""
dt = self._time_diff('startTime', 'scheduledStart')
return max(dt,datetime.timedelta(seconds=0))
return max(dt, datetime.timedelta(seconds=0))
@property
def time_enqueued(self):
"""Time this job was enqueue (not scheduled for execution)
"""
dt = self._time_diff('scheduledStart', 'submissionTime')
return max(dt,datetime.timedelta(seconds=0))
return max(dt, datetime.timedelta(seconds=0))
@property
def time_to_start(self):
"""Time until the job starts
"""
dt = self._time_diff('startTime', timeutils.now())
return max(dt,datetime.timedelta(seconds=0))
return max(dt, datetime.timedelta(seconds=0))
@property
def stderr_file(self):
......@@ -219,7 +219,8 @@ class Job(object):
else:
return os.path.join(self.working_directory, filename)
def _std_read(self, filename):
@staticmethod
def _std_read(filename):
try:
with open(filename, mode='r') as f:
return f.read()
......@@ -309,7 +310,7 @@ class Job(object):
for key in ['name', 'project', 'state', 'owner']:
lst.append('{} = {}'.format(key, stats[key]))
lst.append('runtime = {}'.format(self.runtime))
return 'Job({})\n '.format(self.jobid)+'\n '.join(lst)
return 'Job({})\n '.format(self.jobid) + '\n '.join(lst)
except RuntimeError:
return 'Job(non existing {})'.format(self.jobid)
......@@ -351,7 +352,8 @@ class Job(object):
kwargs['resource'] = self.resource
return JobFactory(**kwargs)
def _parse_stats(self, stats):
@staticmethod
def _parse_stats(stats):
# Dates are in local timezone
for timekey in ['startTime', 'stopTime', 'scheduledStart', 'submissionTime']:
value = stats.get(timekey, 0)
......@@ -374,7 +376,7 @@ class Job(object):
:return timedelta:
"""
with self.fixed_stats():
return self._get_time(end)-self._get_time(start)
return self._get_time(end) - self._get_time(start)
def _get_time(self, tm):
"""
......@@ -383,7 +385,7 @@ class Job(object):
:type tm: str or datetime or None
:return datetime:
"""
if isinstance(tm,datetime.datetime):
if isinstance(tm, datetime.datetime):
return tm
else:
tm = self[tm]
......@@ -438,7 +440,7 @@ class JobFactory(object):
return self._name
@name.setter
def name(self,value):
def name(self, value):
self._name = value
self._default_name()
......@@ -447,7 +449,7 @@ class JobFactory(object):
return self._log_base
@log_base.setter
def log_base(self,value):
def log_base(self, value):
if value:
self._log_base = value
else:
......@@ -548,8 +550,8 @@ def search(name=None, project=None, owner=None, state=None,
if end:
end = timeutils.totimestamp(end)
sqlquery.append("stop_time<='{}'".format(end))
for k, (op,v) in properties.items():
sqlquery.append("{}{}'{}'".format(k, v))
for k, (op, v) in properties.items():
sqlquery.append("{}{}'{}'".format(k, op, v))
if not sqlquery:
return []
......
......@@ -24,7 +24,6 @@
__authors__ = ["W. De Nolf"]
__license__ = "MIT"
import datetime
import re
from . import oarshell
......@@ -40,12 +39,14 @@ def str2walltime(s):
h, m, s = s.split(':')
return datetime.timedelta(hours=int(h), minutes=int(m), seconds=int(s))
property_pattern = re.compile('(\w+)([>=<]+)\'?([\w\d]+)\'?')
# Includes the default ones (space around operator):
#property_pattern = re.compile('(\w+) *([>=<]+) *\'?([\w\d]+)\'?')
# property_pattern = re.compile('(\w+) *([>=<]+) *\'?([\w\d]+)\'?')
class Resource(object):
def __init__(self, host=None, nodes=None, cpu=None, core=None, gpu=False,
walltime=None, mem_core_mb=None, **properties):
"""
......@@ -89,7 +90,7 @@ class Resource(object):
@property
def memory(self):
return self.cores*self.mem_core_mb
return self.cores * self.mem_core_mb
@property
def cores(self):
......@@ -124,7 +125,7 @@ class Resource(object):
properties.append("gpu='YES'")
if self.mem_core_mb:
properties.append("mem_core_mb>={:d}".format(self.mem_core_mb))
for k,(op,v) in self.properties.items():
for k, (op, v) in self.properties.items():
properties.append("{}{}'{}'".format(k, op, v))
cliarg = ' and '.join(properties)
return cliarg
......@@ -145,10 +146,10 @@ class Resource(object):
return oarshell.cli_args2str(*self.cli_arguments)
@classmethod
def from_cli(cls, cmd, properties = None):
def from_cli(cls, cmd, properties=None):
kwargs = {}
cli_arguments = list(oarshell.cli_str2args(cmd))
arg = oarshell._cli_getarg(cli_arguments, '-l')
arg = oarshell.cli_getarg(cli_arguments, '-l')
if arg:
for key in ['host', 'nodes', 'cpu', 'core']:
match = re.search('{}=(\d+)'.format(key), arg)
......@@ -157,18 +158,18 @@ class Resource(object):
match = re.search('walltime=(\d+:\d+:\d+)', arg)
if match:
kwargs['walltime'] = str2walltime(match.groups()[0])
arg = oarshell._cli_getarg(cli_arguments, '-p')
arg = oarshell.cli_getarg(cli_arguments, '-p')
if not arg and properties:
arg = properties
if arg:
for match in property_pattern.finditer(arg):
k,op,v = match.groups()
if k=='gpu':
kwargs['gpu'] = v=='YES'
elif k=='mem_core_mb':
k, op, v = match.groups()
if k == 'gpu':
kwargs['gpu'] = v == 'YES'
elif k == 'mem_core_mb':
kwargs['mem_core_mb'] = int(v)
else:
kwargs[k] = op,v
kwargs[k] = op, v
return cls(**kwargs)
def __eq__(self, other):
......
......@@ -71,7 +71,7 @@ def oarinstalled():
def executejson(cmd, *args):
"""
:param int jobid:
:param str cmd:
:return tuple: out(dict), err(str or None), exitcode(int)
"""
out, err, code = execute(cmd, '-J', *args)
......@@ -91,7 +91,6 @@ def executejson(cmd, *args):
def oarstat(*args):
"""
:param int jobid:
:return tuple: out(dict), err(str or None), exitcode(int)
"""
out, err, code = execute('oarstat', '-J', *args)
......@@ -120,7 +119,6 @@ def oarjobstat(jobid, *args):
def oarsub(*args):
"""
:param int jobid:
:return tuple: out(str or None), err(str or None), exitcode(int)
"""
out, err, code = execute('oarsub', '-J', *args)
......@@ -194,7 +192,7 @@ def cli_str2args(cmd):
return tuple(shlex.split(cmd))
def _cli_getarg(cli_arguments, flag):
def cli_getarg(cli_arguments, flag):
if flag in cli_arguments:
i = cli_arguments.index(flag)
ret = cli_arguments[i+1]
......
......@@ -26,6 +26,7 @@ from . import test_oarshell
from . import test_oarresource
from . import test_oarjob
def test_suite():
"""Test suite including all test suites"""
testSuite = unittest.TestSuite()
......@@ -33,7 +34,8 @@ def test_suite():
testSuite.addTest(test_oarresource.test_suite())
testSuite.addTest(test_oarjob.test_suite())
return testSuite
if __name__ == '__main__':
import sys
......
......@@ -32,14 +32,14 @@ from .. import oarjob
from ..oarresource import Resource
from ..oarshell import oarinstalled
class test_oarjob(unittest.TestCase):
class test_oarjob(unittest.TestCase):
def setUp(self):
self.working_directory = tempfile.mkdtemp(dir=os.getcwd())
def tearDown(self):
shutil.rmtree(self.working_directory)
def skipoar(self):
if not oarinstalled():
raise unittest.SkipTest("oar cli tools not available")
......@@ -54,31 +54,31 @@ class test_oarjob(unittest.TestCase):
self.assertFalse(job.is_waiting)
self.assertFalse(job.is_intermediate)
self.assertFalse(job.needsresume)
self.assertEqual(job['state'],job.status)
self.assertEqual(job['state'], job.status)
def test_jobfactory(self):
self.skipoar()
self.assertTrue(oarjob.JobFactory())
self.assertTrue(oarjob.JobFactory(command='ls -all'))
res = Resource(nodes=1, cpu=2, core=2, gpu=True, mem_core_mb=8000)
info = oarjob.JobFactory(resource=res, command='ls -all', log_base='test')
cli = "-O test.stdout -E test.stderr -l nodes=1/cpu=2/core=2 -p \"gpu='YES' and mem_core_mb>=8000\" \"ls -all\""
self.assertEqual(cli, info.cli_string)
def definition(self, seconds, name):
command = 'python -c "from time import sleep\nfor i in range({}):\n print(i)\n sleep(1)"'
resource = Resource(core=1,walltime={'seconds':seconds*3})
resource = Resource(core=1, walltime={'seconds': seconds * 3})
jobdef = oarjob.JobFactory(name=name, project='oarpy',
resource=resource, command=command.format(seconds),
working_directory=self.working_directory)
output = list(range(seconds))
return jobdef,output
return jobdef, output
def _check_output(self, job, expected):
output = [int(i) for i in job.stdout.split('\n') if i]
self.assertEqual(output, expected)
def test_immediate(self):
self.skipoar()
jobdef, expected = self.definition(5, 'immediate')
......@@ -87,7 +87,7 @@ class test_oarjob(unittest.TestCase):
job.wait(silent=True)
self.assertEqual(job.exit_code, 0)
self._check_output(job, expected)
def test_notimmediate(self):
self.skipoar()
jobdef, expected = self.definition(5, 'notimmediate')
......@@ -97,29 +97,30 @@ class test_oarjob(unittest.TestCase):
job.wait(silent=True)
self.assertEqual(job.exit_code, 0)
self._check_output(job, expected)
def test_interrupt(self):
self.skipoar()
jobdef,expected = self.definition(60, 'interrupt')
jobdef, expected = self.definition(60, 'interrupt')
job = jobdef.submit()
job.wait(states=('Running', 'Terminated', 'Error'), silent=True)
sleep(5)
job.interrupt()
job.wait(silent=True)
self.assertEqual(job.exit_code,None)
self.assertEqual(job.exit_code, None)
def test_suspend(self):
self.skipoar()
jobdef, expected = self.definition(60, 'interrupt')
job = jobdef.submit()
job.wait(states=('Running', 'Terminated', 'Error'), silent=True)
self.assertRaises(RuntimeError, job.suspend)
#job.wait(states='Hold', silent=True)
#job.resume()
#job.wait(silent=True)
#self.assertEqual(job.exit_code,0)
#self._check_output(job,expected)
# job.wait(states='Hold', silent=True)
# job.resume()
# job.wait(silent=True)
# self.assertEqual(job.exit_code,0)
# self._check_output(job,expected)
def test_suite():
"""Test suite including all test suites"""
testSuite = unittest.TestSuite()
......@@ -130,7 +131,8 @@ def test_suite():
testSuite.addTest(test_oarjob("test_interrupt"))
testSuite.addTest(test_oarjob("test_suspend"))
return testSuite
if __name__ == '__main__':
import sys
......
......@@ -25,33 +25,34 @@
import unittest
from ..oarresource import Resource
class test_oarresource(unittest.TestCase):
class test_oarresource(unittest.TestCase):
def test_cores(self):
self.assertFalse(Resource())
self.assertTrue(Resource(core=1))
self.assertEqual(Resource(nodes=1).cores,1)
self.assertEqual(Resource(cpu=1).cores,1)
self.assertEqual(Resource(core=1).cores,1)
self.assertEqual(Resource(nodes=1).cores, 1)
self.assertEqual(Resource(cpu=1).cores, 1)
self.assertEqual(Resource(core=1).cores, 1)
def test_to_cli(self):
res = Resource(nodes=1,cpu=2,core=2,gpu=True,mem_core_mb=8000)
res = Resource(nodes=1, cpu=2, core=2, gpu=True, mem_core_mb=8000)
cli = '-l nodes=1/cpu=2/core=2 -p "gpu=\'YES\' and mem_core_mb>=8000"'
self.assertEqual(cli,res.cli_string)
self.assertEqual(res.cores,4)
self.assertEqual(cli, res.cli_string)
self.assertEqual(res.cores, 4)
def test_from_cli(self):
res = Resource()
self.assertEqual(res,Resource.from_cli(res.cli_string))
self.assertEqual(res, Resource.from_cli(res.cli_string))
res = Resource(core=10)
self.assertEqual(res,Resource.from_cli(res.cli_string))
res = Resource(nodes=1,cpu=2,core=2,gpu=True)
self.assertEqual(res,Resource.from_cli(res.cli_string))
res = Resource(nodes=1,cpu=2,core=2,mem_core_mb=8000)
self.assertEqual(res,Resource.from_cli(res.cli_string))
res = Resource(nodes=1,cpu=2,core=2,gpu=True,mem_core_mb=8000)
self.assertEqual(res,Resource.from_cli(res.cli_string))
self.assertEqual(res, Resource.from_cli(res.cli_string))
res = Resource(nodes=1, cpu=2, core=2, gpu=True)
self.assertEqual(res, Resource.from_cli(res.cli_string))
res = Resource(nodes=1, cpu=2, core=2, mem_core_mb=8000)
self.assertEqual(res, Resource.from_cli(res.cli_string))
res = Resource(nodes=1, cpu=2, core=2, gpu=True, mem_core_mb=8000)
self.assertEqual(res, Resource.from_cli(res.cli_string))
def test_suite():
"""Test suite including all test suites"""
testSuite = unittest.TestSuite()
......@@ -59,7 +60,8 @@ def test_suite():
testSuite.addTest(test_oarresource("test_to_cli"))
testSuite.addTest(test_oarresource("test_from_cli"))
return testSuite
if __name__ == '__main__':
import sys
......
......@@ -25,6 +25,7 @@
import unittest
from .. import oarshell
class test_utils(unittest.TestCase):
def skipoar(self):
......@@ -36,27 +37,28 @@ class test_utils(unittest.TestCase):
def test_oarstat(self):
self.skipoar()
out,err,code = oarshell.jobstats(1)
self.assertTrue(isinstance(out,dict))
out, err, code = oarshell.jobstats(1)
self.assertTrue(isinstance(out, dict))
def test_cli(self):
args = '-p',"gpu=YES"
args = '-p', "gpu=YES"
cmd1 = oarshell.cli_args2str(*args)
cmd2 = "-p gpu=YES"
self.assertEqual(cmd1,cmd2)
self.assertEqual(args,oarshell.cli_str2args(cmd2))
args = '-p',"gpu='YES'"
self.assertEqual(cmd1, cmd2)
self.assertEqual(args, oarshell.cli_str2args(cmd2))
args = '-p', "gpu='YES'"
cmd1 = oarshell.cli_args2str(*args)
cmd2 = "-p gpu='YES'"
self.assertEqual(cmd1,cmd2)
self.assertEqual(('-p',"gpu=YES"),oarshell.cli_str2args(cmd1))
args = '-p',"gpu='YES' and mem_core_mb>=8000"
self.assertEqual(cmd1, cmd2)
self.assertEqual(('-p', "gpu=YES"), oarshell.cli_str2args(cmd1))
args = '-p', "gpu='YES' and mem_core_mb>=8000"
cmd1 = oarshell.cli_args2str(*args)
cmd2 = '-p "gpu=\'YES\' and mem_core_mb>=8000"'
self.assertEqual(cmd1,cmd2)
self.assertEqual(args,oarshell.cli_str2args(cmd2))
self.assertEqual(cmd1, cmd2)
self.assertEqual(args, oarshell.cli_str2args(cmd2))
def test_suite():
"""Test suite including all test suites"""
......@@ -65,7 +67,8 @@ def test_suite():
testSuite.addTest(test_utils("test_oarstat"))
testSuite.addTest(test_utils("test_cli"))
return testSuite
if __name__ == '__main__':
import sys
......
......@@ -24,7 +24,6 @@
__authors__ = ["W. De Nolf"]
__license__ = "MIT"
import datetime
import dateutil.tz
......@@ -35,6 +34,7 @@ def astimezone(dt, tz):
except ValueError:
return dt.replace(tzinfo=tz)
tzlocal = dateutil.tz.tzlocal()
tzutc = dateutil.tz.tzutc()
epoch = astimezone(datetime.datetime(1970, 1, 1), tzutc)
......
......@@ -27,16 +27,19 @@ import os
import io
import sys
import logging
logger = logging.getLogger(__file__)
try:
import sphinx
import sphinx.util.console
sphinx.util.console.color_terminal = lambda: False
from sphinx.setup_command import BuildDoc
except ImportError:
sphinx = None
try:
from setuptools import Command
logger.info("Use setuptools")
except ImportError:
try:
......@@ -73,6 +76,7 @@ def get_readme():
long_description = fp.read()
return long_description
if sphinx is not None:
class BuildDocCommand(BuildDoc):
"""Command to build documentation using sphinx.
......@@ -111,8 +115,8 @@ else:
'Sphinx is required to build or test the documentation.\n'
'Please install Sphinx (http://www.sphinx-doc.org).')
BuildDocCommand = SphinxExpectedCommand
BuildDocCommand = SphinxExpectedCommand
PACKAGES = find_packages()
......@@ -211,6 +215,5 @@ DATA_FILES = [
# Data files that will be installed outside site-packages folder
]
if __name__ == "__main__":
setup_package()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment