/usr/share/pyshared/pytools/batchjob.py is in python-pytools 2011.5-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | def _cp(src, dest):
from pytools import assert_not_a_file
assert_not_a_file(dest)
inf = open(src, "rb")
try:
outf = open(dest, "wb")
try:
outf.write(inf.read())
finally:
outf.close()
finally:
inf.close()
def get_timestamp():
from datetime import datetime
return datetime.now().strftime("%Y-%m-%d-%H%M%S")
class BatchJob(object):
def __init__(self, moniker, main_file, aux_files=[], timestamp=None):
import os
import os.path
if timestamp is None:
timestamp = get_timestamp()
self.moniker = (
moniker
.replace("/", "-")
.replace("-$DATE", "")
.replace("$DATE-", "")
.replace("$DATE", "")
)
self.subdir = moniker.replace("$DATE", timestamp)
self.path = os.path.join(
os.getcwd(),
self.subdir)
os.makedirs(self.path)
runscript = open("%s/run.sh" % self.path, "w")
import sys
runscript.write("%s %s setup.cpy"
% (sys.executable, main_file))
runscript.close()
from os.path import basename
if not main_file.startswith("-m "):
_cp(main_file, os.path.join(self.path, basename(main_file)))
for aux_file in aux_files:
_cp(aux_file, os.path.join(self.path, basename(aux_file)))
def write_setup(self, lines):
import os.path
setup = open(os.path.join(self.path, "setup.cpy"), "w")
setup.write("\n".join(lines))
setup.close()
class INHERIT(object):
pass
class GridEngineJob(BatchJob):
def submit(self, env={"LD_LIBRARY_PATH": INHERIT, "PYTHONPATH": INHERIT},
memory_megs=None, extra_args=[]):
from subprocess import Popen
args = [
"-N", self.moniker,
"-cwd",
]
from os import getenv
for var, value in env.iteritems():
if value is INHERIT:
value = getenv(var)
args += ["-v", "%s=%s" % (var, value)]
if memory_megs is not None:
args.extend(["-l", "mem=%d" % memory_megs])
args.extend(extra_args)
subproc = Popen(["qsub"] + args + ["run.sh"], cwd=self.path)
if subproc.wait() != 0:
raise RuntimeError("Process submission of %s failed" % self.moniker)
class PBSJob(BatchJob):
def submit(self, env={"LD_LIBRARY_PATH": INHERIT, "PYTHONPATH": INHERIT},
memory_megs=None, extra_args=[]):
from subprocess import Popen
args = [
"-N", self.moniker,
"-d", self.path,
]
if memory_megs is not None:
args.extend(["-l", "pmem=%dmb" % memory_megs])
from os import getenv
for var, value in env.iteritems():
if value is INHERIT:
value = getenv(var)
args += ["-v", "%s=%s" % (var, value)]
args.extend(extra_args)
subproc = Popen(["qsub"] + args + ["run.sh"], cwd=self.path)
if subproc.wait() != 0:
raise RuntimeError("Process submission of %s failed" % self.moniker)
def guess_job_class():
from subprocess import Popen, PIPE, STDOUT
qstat_helplines = Popen(["qstat", "--help"],
stdout=PIPE, stderr=STDOUT).communicate()[0].split("\n")
if qstat_helplines[0].startswith("GE"):
return GridEngineJob
else:
return PBSJob
class ConstructorPlaceholder:
def __init__(self, classname, *args, **kwargs):
self.classname = classname
self.args = args
self.kwargs = kwargs
def arg(self, i):
return self.args[i]
def kwarg(self, name):
return self.kwargs[name]
def __str__(self):
return "%s(%s)" % (self.classname,
",".join(
[str(arg) for arg in self.args]
+ ["%s=%s" % (kw, repr(val)) for kw, val in self.kwargs.iteritems()]
)
)
__repr__ = __str__
|