X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d81ea65da05119d5c6480d373b5d42bbee8ae1ad..442d78e1fb39f5d18f728f083e04f320ced44ebb:/sdk/python/arvados/__init__.py diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py index 060ed95d95..e90f381298 100644 --- a/sdk/python/arvados/__init__.py +++ b/sdk/python/arvados/__init__.py @@ -1,92 +1,140 @@ -import gflags -import httplib -import httplib2 -import logging +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +"""Arvados Python SDK + +This module provides the entire Python SDK for Arvados. The most useful modules +include: + +* arvados.api - After you `import arvados`, you can call `arvados.api.api` as + `arvados.api` to construct a client object. + +* arvados.collection - The `arvados.collection.Collection` class provides a + high-level interface to read and write collections. It coordinates sending + data to and from Keep, and synchronizing updates with the collection object. + +* arvados.util - Utility functions to use mostly in conjunction with the API + client object and the results it returns. + +Other submodules provide lower-level functionality. +""" + +import logging as stdliblog import os -import pprint import sys import types -import subprocess -import json -import UserDict -import re -import hashlib -import string -import bz2 -import zlib -import fcntl -import time -import threading - -from api import * -from collection import * -from keep import * -from stream import * -import errors -import util + +from collections import UserDict + +from .api import api, api_from_config, http_cache +from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter +from arvados.keep import * +from arvados.stream import * +from .arvfile import StreamFileReader +from .logging import log_format, log_date_format, log_handler +from .retry import RetryLoop +import arvados.errors as errors +import arvados.util as util + +# Override logging module pulled in via `from ... import *` +# so users can `import arvados.logging`. +logging = sys.modules['arvados.logging'] # Set up Arvados logging based on the user's configuration. # All Arvados code should log under the arvados hierarchy. -log_handler = logging.StreamHandler() -log_handler.setFormatter(logging.Formatter( - '%(asctime)s %(name)s[%(process)d] %(levelname)s: %(message)s', - '%Y-%m-%d %H:%M:%S')) -logger = logging.getLogger('arvados') +logger = stdliblog.getLogger('arvados') logger.addHandler(log_handler) -logger.setLevel(logging.DEBUG if config.get('ARVADOS_DEBUG') - else logging.WARNING) +logger.setLevel(stdliblog.DEBUG if config.get('ARVADOS_DEBUG') + else stdliblog.WARNING) -def task_set_output(self,s): - api('v1').job_tasks().update(uuid=self['uuid'], - body={ - 'output':s, - 'success':True, - 'progress':1.0 - }).execute() +@util._deprecated('3.0', 'arvados-cwl-runner or the containers API') +def task_set_output(self, s, num_retries=5): + for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0): + try: + return api('v1').job_tasks().update( + uuid=self['uuid'], + body={ + 'output':s, + 'success':True, + 'progress':1.0 + }).execute() + except errors.ApiError as error: + if retry.check_http_response_success(error.resp.status) is None and tries_left > 0: + logger.debug("task_set_output: job_tasks().update() raised {}, retrying with {} tries left".format(repr(error),tries_left)) + else: + raise _current_task = None -def current_task(): +@util._deprecated('3.0', 'arvados-cwl-runner or the containers API') +def current_task(num_retries=5): global _current_task if _current_task: return _current_task - t = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute() - t = UserDict.UserDict(t) - t.set_output = types.MethodType(task_set_output, t) - t.tmpdir = os.environ['TASK_WORK'] - _current_task = t - return t + + for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2): + try: + task = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute() + task = UserDict(task) + task.set_output = types.MethodType(task_set_output, task) + task.tmpdir = os.environ['TASK_WORK'] + _current_task = task + return task + except errors.ApiError as error: + if retry.check_http_response_success(error.resp.status) is None and tries_left > 0: + logger.debug("current_task: job_tasks().get() raised {}, retrying with {} tries left".format(repr(error),tries_left)) + else: + raise _current_job = None -def current_job(): +@util._deprecated('3.0', 'arvados-cwl-runner or the containers API') +def current_job(num_retries=5): global _current_job if _current_job: return _current_job - t = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute() - t = UserDict.UserDict(t) - t.tmpdir = os.environ['JOB_WORK'] - _current_job = t - return t + for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2): + try: + job = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute() + job = UserDict(job) + job.tmpdir = os.environ['JOB_WORK'] + _current_job = job + return job + except errors.ApiError as error: + if retry.check_http_response_success(error.resp.status) is None and tries_left > 0: + logger.debug("current_job: jobs().get() raised {}, retrying with {} tries left".format(repr(error),tries_left)) + else: + raise + +@util._deprecated('3.0', 'arvados-cwl-runner or the containers API') def getjobparam(*args): return current_job()['script_parameters'].get(*args) +@util._deprecated('3.0', 'arvados-cwl-runner or the containers API') def get_job_param_mount(*args): return os.path.join(os.environ['TASK_KEEPMOUNT'], current_job()['script_parameters'].get(*args)) +@util._deprecated('3.0', 'arvados-cwl-runner or the containers API') def get_task_param_mount(*args): return os.path.join(os.environ['TASK_KEEPMOUNT'], current_task()['parameters'].get(*args)) class JobTask(object): + @util._deprecated('3.0', 'arvados-cwl-runner or the containers API') def __init__(self, parameters=dict(), runtime_constraints=dict()): - print "init jobtask %s %s" % (parameters, runtime_constraints) + print("init jobtask %s %s" % (parameters, runtime_constraints)) -class job_setup: +class job_setup(object): @staticmethod - def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False): + @util._deprecated('3.0', 'arvados-cwl-runner or the containers API') + def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, api_client=None): if if_sequence != current_task()['sequence']: return + + if not api_client: + api_client = api('v1') + job_input = current_job()['script_parameters']['input'] - cr = CollectionReader(job_input) + cr = CollectionReader(job_input, api_client=api_client) + cr.normalize() for s in cr.all_streams(): for f in s.all_files(): if input_as_path: @@ -101,14 +149,15 @@ class job_setup: 'input':task_input } } - api('v1').job_tasks().create(body=new_task_attrs).execute() + api_client.job_tasks().create(body=new_task_attrs).execute() if and_end_task: - api('v1').job_tasks().update(uuid=current_task()['uuid'], + api_client.job_tasks().update(uuid=current_task()['uuid'], body={'success':True} ).execute() exit(0) @staticmethod + @util._deprecated('3.0', 'arvados-cwl-runner or the containers API') def one_task_per_input_stream(if_sequence=0, and_end_task=True): if if_sequence != current_task()['sequence']: return @@ -130,5 +179,3 @@ class job_setup: body={'success':True} ).execute() exit(0) - -