Commit e7d681cc authored by Wout De Nolf's avatar Wout De Nolf

Merge branch 'feat-quicksubmit' into 'master'

Feat quicksubmit

See merge request !3
parents 7d1f81d4 ee5f4a0a
......@@ -4,58 +4,52 @@ oarpy: OAR job management in python
Getting started
---------------
Hello world (minimal):
Submit
~~~~~~
Run an OAR job that prints "Hello world":
.. code-block:: python
from oarpy import oarjob
jobdef = oarjob.JobFactory(command='echo "Hello word"')
job = jobdef.submit()
from oarpy.oarjob import submit
job = submit(command='echo "Hello word"', name='helloword',
project='oarpy', core=1, gpu=False, walltime={'hours':2})
print(job)
job.wait()
if job.exit_code:
print('Failed:\n{}'.format(job.stderr))
elif job.exit_code is None:
print('Interrupted:\n{}'.format(job.stdout))
else:
print('Succes:\n{}'.format(job.stdout))
job.remove_logs()
Hello world (specify resources and postpone execution):
Only "command" is required, all other arguments are optional. See documentation for more fine-grained control with the JobFactory and Resource classes.
Search
~~~~~~
Find a job in case you know the job ID:
.. code-block:: python
from oarpy import oarjob
from oarpy.oarresource import Resource
resource = Resource(core=2,walltime={'hours':2},gpu=True)
jobdef = oarjob.JobFactory(name='helloworld',project='oarpy',
command='echo "Hello word"',resource=resource)
job = jobdef.submit(hold=True)
job.wait(states='Hold')
# job is waiting for you to resume it
job.resume()
job.wait()
if job.exit_code:
print('Failed:\n{}'.format(job.stderr))
elif job.exit_code is None:
print('Interrupted:\n{}'.format(job.stdout))
else:
print('Succes:\n{}'.format(job.stdout))
job.remove_logs()
Find all jobs started in the last 5 minutes:
from oarpy.oarjob import Job
job = Job(1130922)
print(job)
Find a job in case you do not know the job ID (not all arguments are necessary):
.. code-block:: python
from oarpy import oarjob
from oarpy.oarjob import search
from oarpy import timeutils
start = timeutils.add(timeutils.now(),minutes=-5)
jobs = oarjob.search(start=start)
print(set([(job['owner'],job.status) for job in jobs]))
print('Job statistics: {}'.format(jobs[0].stats.keys()))
import os
owner = os.getlogin()
start = timeutils.add(timeutils.now(),minutes=-10)
jobs = search(owner=owner, start=start, name='quickstart',
project='oarpy', state='Terminated')
print(jobs)
\ No newline at end of file
......@@ -5,4 +5,5 @@ Tutorials
:maxdepth: 3
tutorials/quickstart.rst
tutorials/getting_started.rst
tutorials/oarjob.rst
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Getting started\n",
"\n",
"## Hello world\n",
"OAR job that prints \"Hello world\" to the standard output.\n",
"\n",
"### Minimal"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Job(1130826)\n",
" name = OAR\n",
" project = default\n",
" state = Waiting\n",
" owner = denolf\n",
" runtime = 0:00:00\n",
"........\n",
"Succes:\n",
"Hello word\n",
"\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"\n",
"jobdef = oarjob.JobFactory(command='echo \"Hello word\"')\n",
"job = jobdef.submit()\n",
"print(job)\n",
"job.wait()\n",
"\n",
"if job.exit_code:\n",
" print('Failed:\\n{}'.format(job.stderr))\n",
"elif job.exit_code is None:\n",
" print('Interrupted:\\n{}'.format(job.stdout))\n",
"else:\n",
" print('Succes:\\n{}'.format(job.stdout))\n",
"job.remove_logs()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### With resources and postponed execution"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Job(1130827)\n",
" name = helloworld\n",
" project = oarpy\n",
" state = Hold\n",
" owner = denolf\n",
" runtime = 0:00:00\n",
"...............\n",
"Succes:\n",
"Hello word\n",
"\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"from oarpy.oarresource import Resource\n",
"\n",
"resource = Resource(core=1, walltime={'hours':1,'minutes':30}, gpu=False)\n",
"jobdef = oarjob.JobFactory(name='helloworld', project='oarpy',\n",
" command='echo \"Hello word\"', resource=resource)\n",
"\n",
"job = jobdef.submit(hold=True)\n",
"print(job)\n",
"job.wait(states='Hold')\n",
"# job is waiting for you to resume it\n",
"job.resume()\n",
"job.wait()\n",
"\n",
"if job.exit_code:\n",
" print('Failed:\\n{}'.format(job.stderr))\n",
"elif job.exit_code is None:\n",
" print('Interrupted:\\n{}'.format(job.stdout))\n",
"else:\n",
" print('Succes:\\n{}'.format(job.stdout))\n",
"job.remove_logs()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Job management\n",
"\n",
"Find all jobs started in the last 5 minutes:"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Job(1130826), Job(1130827), Job(976220)]\n",
"set([(u'ljacques', u'Error'), (u'denolf', u'Terminated')])\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"from oarpy import timeutils\n",
"\n",
"start = timeutils.add(timeutils.now(), minutes=-1)\n",
"jobs = oarjob.search(start=start)\n",
"print(sorted(jobs))\n",
"print(set([(job['owner'], job.status) for job in jobs]))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Retrieve job definition (can be used to resubmit a job):"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Initial request:\n",
" \n",
"\n",
"Wanted resources:\n",
" -l \"{type = 'default'}/host=1/core=16,walltime=16:0:0\" \n",
"\n",
"Properties:\n",
" (((((gpu='YES') AND desktop_computing = 'NO') AND cluster = 'NICE') AND opsys = 'debian8') AND interactive = 'MIXED') AND drain='NO'\n",
"\n",
"Job resource:\n",
" -l nodes=1/core=16,walltime=16:00:00 -p \"gpu='YES' and drain='NO'\"\n",
"\n",
"Job definition:\n",
" -n Gecko_2_0p7um_2474_3031__001_.par --project default -d /mntdirect/_data_visitor/md1189/id17/GeckosHR/Gecko_2/Gecko_2_0p7um_2474_3031__001_/Slices -O OAR.Gecko_2_0p7um_2474_3031__001_.par.%jobid%.stdout -E OAR.Gecko_2_0p7um_2474_3031__001_.par.%jobid%.stderr -l nodes=1/core=16,walltime=16:00:00 -p \"gpu='YES' and drain='NO'\" /mntdirect/_data_visitor/md1189/id17/GeckosHR/Gecko_2/Gecko_2_0p7um_2474_3031__001_/Slices//./tmpmd1189.sh\n",
"\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"\n",
"job = oarjob.Job(1103714)\n",
"if job.exists:\n",
" jobdef = job.definition\n",
" stats = job.stats\n",
" print('Initial request:\\n {}\\n'.format(stats['initial_request']))\n",
" print('Wanted resources:\\n {}\\n'.format(stats['wanted_resources']))\n",
" print('Properties:\\n {}\\n'.format(stats['properties']))\n",
" print('Job resource:\\n {}\\n'.format(jobdef.resource))\n",
" print('Job definition:\\n {}\\n'.format(jobdef))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Remove the log files of all succesfully finished jobs of a particular user and project:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"from oarpy import oarjob\n",
"jobs = oarjob.search(owner='testuser', project='oarpy', state='Terminated')\n",
"for job in jobs:\n",
" job.remove_logs()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
......@@ -180,7 +180,7 @@
"\n",
"end = timeutils.now()\n",
"start = timeutils.add(end,minutes=-10)\n",
"jobs = oarjob.search(start=start,end=end)\n",
"jobs = oarjob.search(start=start, end=end)\n",
"\n",
"print(str(len(jobs))+' jobs found')\n",
"\n",
......@@ -245,7 +245,7 @@
}
],
"source": [
"jobdef = oarjob.JobFactory(name='test',project='oarpy',command='ls')\n",
"jobdef = oarjob.JobFactory(name='test', project='oarpy', command='ls')\n",
"print(jobdef)"
]
},
......@@ -270,8 +270,8 @@
}
],
"source": [
"jobdef = oarjob.JobFactory(name='test',project='oarpy',command='ls',\n",
" working_directory='/tmp/oarpy',log_directory='/tmp/oarpy/log')\n",
"jobdef = oarjob.JobFactory(name='test', project='oarpy', command='ls',\n",
" working_directory='/tmp/oarpy', log_directory='/tmp/oarpy/log')\n",
"print(jobdef)"
]
},
......@@ -307,8 +307,8 @@
],
"source": [
"resource = Resource(nodes=1,core=8,walltime={'minutes':1})\n",
"jobdef = oarjob.JobFactory(name='test',project='oarpy',\n",
" command='ls',resource=resource)\n",
"jobdef = oarjob.JobFactory(name='test', project='oarpy',\n",
" command='ls', resource=resource)\n",
"print(jobdef)"
]
},
......@@ -329,8 +329,8 @@
"def definition(seconds):\n",
" command = 'python -c \"from time import sleep\\nfor i in range({}):\\n print(i)\\n sleep(1)\"'\n",
" resource = Resource(core=1,walltime={'seconds':seconds*3})\n",
" return oarjob.JobFactory(name='test{}'.format(seconds),project='oarpy',\n",
" resource=resource,command=command.format(seconds))"
" return oarjob.JobFactory(name='test{}'.format(seconds), project='oarpy',\n",
" resource=resource, command=command.format(seconds))"
]
},
{
......
......@@ -6,22 +6,27 @@
"source": [
"# Quick start\n",
"\n",
"## Hello world\n",
"OAR job that prints \"Hello world\" to the standard output.\n",
"## Submit\n",
"\n",
"### Minimal"
"Run an OAR job that prints \"Hello world\":"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"......................\n",
"Job(1131384)\n",
" name = quickstart\n",
" project = oarpy\n",
" state = Waiting\n",
" owner = denolf\n",
" runtime = 0:00:00\n",
"..............\n",
"Succes:\n",
"Hello word\n",
"\n"
......@@ -29,10 +34,11 @@
}
],
"source": [
"from oarpy import oarjob\n",
"from oarpy.oarjob import submit\n",
"\n",
"jobdef = oarjob.JobFactory(command='echo \"Hello word\"')\n",
"job = jobdef.submit()\n",
"job = submit(command='echo \"Hello word\"', name='quickstart',\n",
" project='oarpy', core=1, gpu=False, walltime={'hours':2})\n",
"print(job)\n",
"job.wait()\n",
"\n",
"if job.exit_code:\n",
......@@ -41,52 +47,7 @@
" print('Interrupted:\\n{}'.format(job.stdout))\n",
"else:\n",
" print('Succes:\\n{}'.format(job.stdout))\n",
"job.remove_logs()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### With resources and postponed execution"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"......................................\n",
"Succes:\n",
"Hello word\n",
"\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"from oarpy.oarresource import Resource\n",
"\n",
"resource = Resource(core=1,walltime={'hours':1,'minutes':30},gpu=False)\n",
"jobdef = oarjob.JobFactory(name='helloworld',project='oarpy',\n",
" command='echo \"Hello word\"',resource=resource)\n",
"\n",
"job = jobdef.submit(hold=True)\n",
"job.wait(states='Hold')\n",
"# job is waiting for you to resume it\n",
"job.resume()\n",
"job.wait()\n",
"\n",
"if job.exit_code:\n",
" print('Failed:\\n{}'.format(job.stderr))\n",
"elif job.exit_code is None:\n",
" print('Interrupted:\\n{}'.format(job.stdout))\n",
"else:\n",
" print('Succes:\\n{}'.format(job.stdout))\n",
"job.remove_logs()"
]
},
......@@ -94,100 +55,65 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Job management\n",
"## Search\n",
"\n",
"Find all jobs started in the last 5 minutes:"
"Find the job started above in case you know the job ID:"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[Job(1122759), Job(1122760), Job(1124796), Job(1125514), Job(1125515), Job(1125516), Job(976220)]\n",
"set([(u'denolf', u'Finishing'), (u'denolf', u'Error'), (u'in1081', u'Running'), (u'denolf', u'Terminated'), (u'ljacques', u'Error'), (u'in1096', u'Waiting')])\n"
"Job(1131384)\n",
" name = quickstart\n",
" project = oarpy\n",
" state = Terminated\n",
" owner = denolf\n",
" runtime = 0:00:07\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"from oarpy import timeutils\n",
"\n",
"start = timeutils.add(timeutils.now(),minutes=-1)\n",
"jobs = oarjob.search(start=start)\n",
"print(sorted(jobs))\n",
"print(set([(job['owner'],job.status) for job in jobs]))"
"from oarpy.oarjob import Job\n",
"job = Job(1131384)\n",
"print(job)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Retrieve job definition (can be used to resubmit a job):"
"Find the job started above in case you do not know the job ID (not all arguments are necessary):"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Initial request:\n",
" \n",
"\n",
"Wanted resources:\n",
" -l \"{type = 'default'}/host=1/core=16,walltime=16:0:0\" \n",
"\n",
"Properties:\n",
" (((((gpu='YES') AND desktop_computing = 'NO') AND cluster = 'NICE') AND opsys = 'debian8') AND interactive = 'MIXED') AND drain='NO'\n",
"\n",
"Job resource:\n",
" -l nodes=1/core=16,walltime=16:00:00 -p \"gpu='YES' and drain='NO'\"\n",
"\n",
"Job definition:\n",
" -n Gecko_2_0p7um_2474_3031__001_.par --project default -d /mntdirect/_data_visitor/md1189/id17/GeckosHR/Gecko_2/Gecko_2_0p7um_2474_3031__001_/Slices -O OAR.Gecko_2_0p7um_2474_3031__001_.par.%jobid%.stdout -E OAR.Gecko_2_0p7um_2474_3031__001_.par.%jobid%.stderr -l nodes=1/core=16,walltime=16:00:00 -p \"gpu='YES' and drain='NO'\" /mntdirect/_data_visitor/md1189/id17/GeckosHR/Gecko_2/Gecko_2_0p7um_2474_3031__001_/Slices//./tmpmd1189.sh\n",
"\n"
"[Job(1131384)]\n"
]
}
],
"source": [
"from oarpy import oarjob\n",
"from oarpy.oarjob import search\n",
"from oarpy import timeutils\n",
"import os\n",
"\n",
"job = oarjob.Job(1103714)\n",
"if job.exists:\n",
" jobdef = job.definition\n",
" stats = job.stats\n",
" print('Initial request:\\n {}\\n'.format(stats['initial_request']))\n",
" print('Wanted resources:\\n {}\\n'.format(stats['wanted_resources']))\n",
" print('Properties:\\n {}\\n'.format(stats['properties']))\n",
" print('Job resource:\\n {}\\n'.format(jobdef.resource))\n",
" print('Job definition:\\n {}\\n'.format(jobdef))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Remove the log files of all succesfully finished jobs of a particular user and project:"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"from oarpy import oarjob\n",
"jobs = oarjob.search(owner='testuser',project='oarpy',state='Terminated')\n",
"for job in jobs:\n",
" job.remove_logs()"
"owner = os.getlogin()\n",
"start = timeutils.add(timeutils.now(), minutes=-3)\n",
"jobs = search(owner=owner, start=start, name='quickstart',\n",
" project='oarpy', state='Terminated')\n",
"print(jobs)"
]
}
],
......
......@@ -424,10 +424,15 @@ class JobFactory(object):
"""
def __init__(self, command=None, resource=None, name=None, project=None,
working_directory=None, log_directory=None, log_base=None):
working_directory=None, log_directory=None, log_base=None, **resource_parameters):
self._name = None
self._log_base = None
self.command = command
if resource_parameters:
if resource:
logger.warning('Skip resource parameters {}'.format(resource_parameters))
else:
resource = oarresource.Resource(**resource_parameters)
self.resource = resource
self.name = name
self.project = project
......@@ -522,6 +527,11 @@ class JobFactory(object):
return Job(jobid)
def submit(hold=False, **parameters):
jobdef = JobFactory(**parameters)
return jobdef.submit(hold=hold)
def search(name=None, project=None, owner=None, state=None,
start=None, end=None, **properties):
"""
......
......@@ -69,7 +69,7 @@ classifiers = ["Development Status :: 4 - Beta",
def get_readme():
"""Returns content of README.md file"""
"""Returns content of README.rst file"""
dirname = os.path.dirname(os.path.abspath(__file__))
filename = os.path.join(dirname, "README.rst")
with io.open(filename, "r", encoding="utf-8") as fp:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment