import time
import threading
-EMPTY_BLOCK_LOCATOR = 'd41d8cd98f00b204e9800998ecf8427e+0'
-
from api import *
-from stream import *
from collection import *
from keep import *
+from stream import *
+import errors
import util
-class errors:
- class SyntaxError(Exception):
- pass
- class AssertionError(Exception):
- pass
- class NotFoundError(Exception):
- pass
- class CommandFailedError(Exception):
- pass
- class KeepWriteError(Exception):
- pass
- class NotImplementedError(Exception):
- pass
+# Set up Arvados logging based on the user's configuration.
+# All Arvados code should log under the arvados hierarchy.
+log_handler = logging.StreamHandler()
+log_handler.setFormatter(logging.Formatter(
+ '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s',
+ '%Y-%m-%d %H:%M:%S'))
+logger = logging.getLogger('arvados')
+logger.addHandler(log_handler)
+logger.setLevel(logging.DEBUG if config.get('ARVADOS_DEBUG')
+ else logging.WARNING)
def task_set_output(self,s):
api('v1').job_tasks().update(uuid=self['uuid'],
def getjobparam(*args):
return current_job()['script_parameters'].get(*args)
+def get_job_param_mount(*args):
+ return os.path.join(os.environ['TASK_KEEPMOUNT'], current_job()['script_parameters'].get(*args))
+
+def get_task_param_mount(*args):
+ return os.path.join(os.environ['TASK_KEEPMOUNT'], current_task()['parameters'].get(*args))
+
class JobTask(object):
def __init__(self, parameters=dict(), runtime_constraints=dict()):
print "init jobtask %s %s" % (parameters, runtime_constraints)
class job_setup:
@staticmethod
- def one_task_per_input_file(if_sequence=0, and_end_task=True):
+ def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False):
if if_sequence != current_task()['sequence']:
return
job_input = current_job()['script_parameters']['input']
cr = CollectionReader(job_input)
for s in cr.all_streams():
for f in s.all_files():
- task_input = f.as_manifest()
+ if input_as_path:
+ task_input = os.path.join(job_input, s.name(), f.name())
+ else:
+ task_input = f.as_manifest()
new_task_attrs = {
'job_uuid': current_job()['uuid'],
'created_by_job_task_uuid': current_task()['uuid'],