crunch_scripts/run-command

   1 #!/usr/bin/env python
   2
   3 import arvados
   4 import re
   5 import os
   6 import subprocess
   7 import sys
   8 import shutil
   9 import subst
  10 import time
  11
  12 os.umask(0077)
  13
  14 t = arvados.current_task().tmpdir
  15
  16 api = arvados.api('v1')
  17
  18 os.chdir(arvados.current_task().tmpdir)
  19 os.mkdir("tmpdir")
  20 os.mkdir("output")
  21
  22 os.chdir("output")
  23
  24 if len(arvados.current_task()['parameters']) > 0:
  25     p = arvados.current_task()['parameters']
  26 else:
  27     p = arvados.current_job()['script_parameters']
  28
  29 links = []
  30
  31 def sub_link(v):
  32     r = os.path.basename(v)
  33     os.symlink(os.path.join(os.environ['TASK_KEEPMOUNT'], v) , r)
  34     links.append(r)
  35     return r
  36
  37 def sub_tmpdir(v):
  38     return os.path.join(arvados.current_task().tmpdir, 'tmpdir')
  39
  40 def sub_cores(v):
  41      return os.environ['CRUNCH_NODE_SLOTS']
  42
  43 subst.default_subs["link "] = sub_link
  44 subst.default_subs["tmpdir"] = sub_tmpdir
  45 subst.default_subs["node.cores"] = sub_cores
  46
  47 rcode = 1
  48
  49 try:
  50     cmd = []
  51     for c in p["command"]:
  52         cmd.append(subst.do_substitution(p, c))
  53
  54     stdoutname = None
  55     stdoutfile = None
  56     if "stdout" in p:
  57         stdoutname = subst.do_substitution(p, p["stdout"])
  58         stdoutfile = open(stdoutname, "wb")
  59
  60     print("run-command: {}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
  61
  62     rcode = subprocess.call(cmd, stdout=stdoutfile)
  63
  64     print("run-command: completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
  65
  66 except Exception as e:
  67     print("run-command: caught exception: {}".format(e))
  68
  69 finally:
  70     for l in links:
  71         os.unlink(l)
  72
  73     print("run-command: the follow output files will be saved to keep:")
  74
  75     subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"])
  76
  77     print("run-command: start writing output to keep")
  78
  79     done = False
  80     while not done:
  81         try:
  82             out = arvados.CollectionWriter()
  83             out.write_directory_tree(".", max_manifest_depth=0)
  84             outuuid = out.finish()
  85             api.job_tasks().update(uuid=arvados.current_task()['uuid'],
  86                                                  body={
  87                                                      'output':outuuid,
  88                                                      'success': (rcode == 0),
  89                                                      'progress':1.0
  90                                                  }).execute()
  91             done = True
  92         except Exception as e:
  93             print("run-command: caught exception: {}".format(e))
  94             time.sleep(5)
  95
  96 sys.exit(rcode)