Merge branch 'master' into 13306-arvados-cwl-runner-py3-support

Arvados-DCO-1.1-Signed-off-by:  Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --git a/.gitignore b/.gitignore
index 517166c3fbd2043bff880db2b11b97238e4eb5f2..db3020ae82d13231817872355b04dd09849f87a0 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,5 @@ services/api/config/arvados-clients.yml
 .DS_Store
 .vscode
 .Rproj.user
-_version.py
\ No newline at end of file
+_version.py
+*.bak

diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index 9393c1accec46a4e6e3bb6cb35df3cd4bef54bf6..2e4c4573823e4d67efc84ad22b70c4ec915aff95 100755 (executable)
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -16,6 +16,7 @@ Syntax:
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 CWLTOOL=path           (optional) Path to cwltool git repository.
 SALAD=path             (optional) Path to schema_salad git repository.
+PYCMD=pythonexec       (optional) Specify the python executable to use in the docker image. Defaults to "python".
 
 EOF
 
@@ -35,6 +36,11 @@ fi
 
 cd "$WORKSPACE"
 
+py=python
+if [[ -n "$PYCMD" ]] ; then
+    py="$PYCMD" ;
+fi
+
 (cd sdk/python && python setup.py sdist)
 sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
 
@@ -69,6 +75,6 @@ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
     cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
 fi
 
-docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
+docker build --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
 echo arv-keepdocker arvados/jobs $cwl_runner_version
 arv-keepdocker arvados/jobs $cwl_runner_version

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 749075d81576242a29db0d1b075414dc5f1d0270..7438ee6489b6ced5afc656a923e9720fc26dda58 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -596,6 +596,7 @@ pythonstuff=(
     sdk/python
     sdk/python:py3
     sdk/cwl
+    sdk/cwl:py3
     services/dockercleaner:py3
     services/fuse
     services/nodemanager

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7e149528308fc9c6e38e0021af858da5450b58f8..52fd4d21a115f29ee7fb388c11ebeb0564b40ae2 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -6,6 +6,9 @@
 # Implement cwl-runner interface for submitting and running work on Arvados, using
 # either the Crunch jobs API or Crunch containers API.
 
+from future.utils import viewitems
+from builtins import str
+
 import argparse
 import logging
 import os
@@ -66,9 +69,9 @@ def versionstring():
 def arg_parser():  # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(description='Arvados executor for Common Workflow Language')
 
-    parser.add_argument("--basedir", type=str,
+    parser.add_argument("--basedir",
                         help="Base directory used to resolve relative references in the input, default to directory of input object file or current directory (if inputs piped/provided on command line).")
-    parser.add_argument("--outdir", type=str, default=os.path.abspath('.'),
+    parser.add_argument("--outdir", default=os.path.abspath('.'),
                         help="Output directory, default current directory")
 
     parser.add_argument("--eval-timeout",
@@ -99,9 +102,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         default=True, dest="enable_reuse",
                         help="Disable job or container reuse")
 
-    parser.add_argument("--project-uuid", type=str, metavar="UUID", help="Project that will own the workflow jobs, if not provided, will go to home project.")
-    parser.add_argument("--output-name", type=str, help="Name to use for collection that stores the final output.", default=None)
-    parser.add_argument("--output-tags", type=str, help="Tags for the final output collection separated by commas, e.g., '--output-tags tag0,tag1,tag2'.", default=None)
+    parser.add_argument("--project-uuid", metavar="UUID", help="Project that will own the workflow jobs, if not provided, will go to home project.")
+    parser.add_argument("--output-name", help="Name to use for collection that stores the final output.", default=None)
+    parser.add_argument("--output-tags", help="Tags for the final output collection separated by commas, e.g., '--output-tags tag0,tag1,tag2'.", default=None)
     parser.add_argument("--ignore-docker-for-reuse", action="store_true",
                         help="Ignore Docker image version when deciding whether to reuse past jobs.",
                         default=False)
@@ -114,7 +117,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--create-template", action="store_true", help="(Deprecated) synonym for --create-workflow.",
                          dest="create_workflow")
     exgroup.add_argument("--create-workflow", action="store_true", help="Create an Arvados workflow (if using the 'containers' API) or pipeline template (if using the 'jobs' API). See --api.")
-    exgroup.add_argument("--update-workflow", type=str, metavar="UUID", help="Update an existing Arvados workflow or pipeline template with the given UUID.")
+    exgroup.add_argument("--update-workflow", metavar="UUID", help="Update an existing Arvados workflow or pipeline template with the given UUID.")
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--wait", action="store_true", help="After submitting workflow runner job, wait for completion.",
@@ -128,7 +131,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--no-log-timestamps", action="store_false", help="No timestamp on logging lines",
                         default=True, dest="log_timestamps")
 
-    parser.add_argument("--api", type=str,
+    parser.add_argument("--api",
                         default=None, dest="work_api",
                         choices=("jobs", "containers"),
                         help="Select work submission API.  Default is 'jobs' if that API is available, otherwise 'containers'.")
@@ -141,7 +144,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         help="RAM (in MiB) required for the workflow runner job (default 1024)",
                         default=None)
 
-    parser.add_argument("--submit-runner-image", type=str,
+    parser.add_argument("--submit-runner-image",
                         help="Docker image for workflow runner job, default arvados/jobs:%s" % __version__,
                         default=None)
 
@@ -150,11 +153,11 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         default=False)
 
     exgroup = parser.add_mutually_exclusive_group()
-    exgroup.add_argument("--submit-request-uuid", type=str,
+    exgroup.add_argument("--submit-request-uuid",
                          default=None,
                          help="Update and commit to supplied container request instead of creating a new one (containers API only).",
                          metavar="UUID")
-    exgroup.add_argument("--submit-runner-cluster", type=str,
+    exgroup.add_argument("--submit-runner-cluster",
                          help="Submit workflow runner to a remote cluster (containers API only)",
                          default=None,
                          metavar="CLUSTER_ID")
@@ -163,7 +166,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         default=None,
                         help="Collection cache size (in MiB, default 256).")
 
-    parser.add_argument("--name", type=str,
+    parser.add_argument("--name",
                         help="Name to use for workflow execution instance.",
                         default=None)
 
@@ -175,7 +178,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--enable-dev", action="store_true",
                         help="Enable loading and running development versions "
                              "of CWL spec.", default=False)
-    parser.add_argument('--storage-classes', default="default", type=str,
+    parser.add_argument('--storage-classes', default="default",
                         help="Specify comma separated list of storage classes to be used when saving workflow output to Keep.")
 
     parser.add_argument("--intermediate-output-ttl", type=int, metavar="N",
@@ -208,7 +211,7 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
                         default=False, dest="trash_intermediate",
                         help="Do not trash intermediate outputs (default).")
 
-    parser.add_argument("workflow", type=str, default=None, help="The workflow to execute")
+    parser.add_argument("workflow", default=None, help="The workflow to execute")
     parser.add_argument("job_order", nargs=argparse.REMAINDER, help="The input object to the workflow.")
 
     return parser
@@ -232,7 +235,7 @@ def add_arv_hints():
     ])
 
 def exit_signal_handler(sigcode, frame):
-    logger.error("Caught signal {}, exiting.".format(sigcode))
+    logger.error(str(u"Caught signal {}, exiting.").format(sigcode))
     sys.exit(-sigcode)
 
 def main(args, stdout, stderr, api_client=None, keep_client=None,
@@ -243,7 +246,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
     arvargs = parser.parse_args(args)
 
     if len(arvargs.storage_classes.strip().split(',')) > 1:
-        logger.error("Multiple storage classes are not supported currently.")
+        logger.error(str(u"Multiple storage classes are not supported currently."))
         return 1
 
     arvargs.use_container = True
@@ -261,7 +264,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
         else:
             want_api = None
         if want_api and arvargs.work_api and want_api != arvargs.work_api:
-            logger.error('--update-workflow arg {!r} uses {!r} API, but --api={!r} specified'.format(
+            logger.error(str(u'--update-workflow arg {!r} uses {!r} API, but --api={!r} specified').format(
                 arvargs.update_workflow, want_api, arvargs.work_api))
             return 1
         arvargs.work_api = want_api
@@ -271,7 +274,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
 
     add_arv_hints()
 
-    for key, val in cwltool.argparser.get_default_args().items():
+    for key, val in viewitems(cwltool.argparser.get_default_args()):
         if not hasattr(arvargs, key):
             setattr(arvargs, key, val)
 

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 6a91d6ff3c5076a28e06a685ed8c73bf45a84218..a45321d37834f1ddcdb214b848e2a5dfd00f3962 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -2,10 +2,14 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+
 import logging
 import json
 import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import time
 import datetime
 import ciso8601
@@ -136,7 +140,7 @@ class ArvadosContainer(JobBase):
                 generatemapper = NoFollowPathMapper(self.generatefiles["listing"], "", "",
                                                     separateDirs=False)
 
-                sorteditems = sorted(generatemapper.items(), None, key=lambda n: n[1].target)
+                sorteditems = sorted(list(generatemapper.items()), key=lambda n: n[1].target)
 
                 logger.debug("generatemapper is %s", sorteditems)
 
@@ -158,7 +162,7 @@ class ArvadosContainer(JobBase):
                                 }
                             else:
                                 with vwd.open(p.target, "w") as n:
-                                    n.write(p.resolved.encode("utf-8"))
+                                    n.write(p.resolved)
 
                 def keepemptydirs(p):
                     if isinstance(p, arvados.collection.RichCollectionBase):

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 87d6d1049a08d349d8d45b41b47a5fccb99f237c..bdf798d300f31da91d9810d33be85f0141701dfd 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -2,6 +2,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from past.builtins import basestring
+from builtins import object
+from future.utils import viewitems
+
 import logging
 import re
 import copy
@@ -71,7 +75,7 @@ class ArvadosJob(JobBase):
                                                separateDirs=False)
 
                 with Perf(metrics, "createfiles %s" % self.name):
-                    for f, p in generatemapper.items():
+                    for f, p in list(generatemapper.items()):
                         if p.type == "CreateFile":
                             with vwd.open(p.target, "w") as n:
                                 n.write(p.resolved.encode("utf-8"))
@@ -85,7 +89,7 @@ class ArvadosJob(JobBase):
                                      trash_at=info["trash_at"],
                                      properties=info["properties"])
 
-                for f, p in generatemapper.items():
+                for f, p in list(generatemapper.items()):
                     if p.type == "File":
                         script_parameters["task.vwd"][p.target] = p.resolved
                     if p.type == "CreateFile":
@@ -243,7 +247,7 @@ class ArvadosJob(JobBase):
                                                                    api_client=self.arvrunner.api,
                                                                    keep_client=self.arvrunner.keep_client,
                                                                    num_retries=self.arvrunner.num_retries)
-                        log = logc.open(logc.keys()[0])
+                        log = logc.open(list(logc.keys())[0])
                         dirs = {
                             "tmpdir": "/tmpdir",
                             "outdir": "/outdir",
@@ -343,7 +347,7 @@ class RunnerJob(Runner):
             find_or_create=self.enable_reuse
         ).execute(num_retries=self.arvrunner.num_retries)
 
-        for k,v in job_spec["script_parameters"].items():
+        for k,v in viewitems(job_spec["script_parameters"]):
             if v is False or v is None or isinstance(v, dict):
                 job_spec["script_parameters"][k] = {"value": v}
 

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index c4e9f44abb0b20ecb66a7bdc13c5240beaaeeccb..31e6be12b533cb9d96c65f74189935fad8c3fcbe 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -16,7 +16,7 @@ def validate_cluster_target(arvrunner, runtimeContext):
         runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and
         runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]):
         raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster,
-                                                                                                  ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys())))
+                                                                                                  ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
 def set_cluster_target(tool, arvrunner, builder, runtimeContext):
     cluster_target_req = None
     for field in ("hints", "requirements"):

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index ea167d4044d76fa91953eb401962107afd6b878e..ff95041779f0e37d2a10113b7cf7d9146f9076f6 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from past.builtins import basestring
+
 import os
 import json
 import copy
@@ -135,7 +137,7 @@ class ArvadosWorkflowStep(WorkflowStep):
         runtimeContext = runtimeContext.copy()
         runtimeContext.toplevel = True  # Preserve behavior for #13365
 
-        builder = make_builder({shortname(k): v for k,v in joborder.items()}, self.hints, self.requirements, runtimeContext)
+        builder = make_builder({shortname(k): v for k,v in list(joborder.items())}, self.hints, self.requirements, runtimeContext)
         runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
         return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
 

diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 61f9cbbe0dc80a7ce7c4894ccb2697c0b0310652..d9a986663b83ec1eb79024181e89cbb87cd570ee 100644 (file)
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -10,6 +10,8 @@
 # tool.  When the workflow completes, record the output object in an output
 # collection for this runner job.
 
+from past.builtins import basestring
+
 import arvados
 import arvados_cwl
 import arvados.collection
@@ -61,7 +63,7 @@ def run():
             if "location" in v:
                 v["location"] = keeppath(v["location"])
 
-        for k,v in job_order_object.items():
+        for k,v in list(job_order_object.items()):
             if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
                 job_order_object[k] = {
                     "class": "File",

diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 6d46e79cb84b33bee3af508c86498201aa37acd1..378192b620ce5d390068782fa94f0bd48f8b1ea8 100644 (file)
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future.utils import viewvalues
+
 import re
 from cwltool.errors import WorkflowException
 from collections import deque
@@ -63,7 +65,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
     containersapi = ("crunch-run.txt" in logcollection)
     mergelogs = {}
 
-    for log in logcollection.keys():
+    for log in list(logcollection):
         if not containersapi or log in ("crunch-run.txt", "stdout.txt", "stderr.txt"):
             logname = log[:-4]
             logt = deque([], maxlen)
@@ -77,7 +79,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
                         logt.append(l)
 
     if containersapi:
-        keys = mergelogs.keys()
+        keys = list(mergelogs)
         loglines = []
         while True:
             earliest = None
@@ -91,7 +93,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
             loglines.append("%s %s %s" % (ts, earliest, msg))
         loglines = loglines[-maxlen:]
     else:
-        loglines = mergelogs.values()[0]
+        loglines = viewvalues(mergelogs)[0]
 
     logtxt = "\n  ".join(l.strip() for l in loglines)
     logfunc("%s\n\n  %s", header, logtxt)

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 27774b2f7cf6bd1fbb9bd8474f5dde4e7e4d6d51..eab12efaac6e34f08cc7ffc2869ebe344a7ae44b 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -2,6 +2,11 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import division
+from builtins import next
+from builtins import object
+from future.utils import viewvalues
+
 import argparse
 import logging
 import os
@@ -157,7 +162,7 @@ class ArvCwlExecutor(object):
                 raise Exception("Unsupported API '%s', expected one of %s" % (arvargs.work_api, expected_api))
 
         if self.work_api == "jobs":
-            logger.warn("""
+            logger.warning("""
 *******************************
 Using the deprecated 'jobs' API.
 
@@ -180,6 +185,11 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         # if running inside a container
         if arvados_cwl.util.get_current_container(self.api, self.num_retries, logger):
             root_logger = logging.getLogger('')
+
+            # Remove existing RuntimeStatusLoggingHandlers if they exist
+            handlers = [h for h in root_logger.handlers if not isinstance(h, RuntimeStatusLoggingHandler)]
+            root_logger.handlers = handlers
+
             handler = RuntimeStatusLoggingHandler(self.runtime_status_update)
             root_logger.addHandler(handler)
 
@@ -332,7 +342,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                 if self.stop_polling.is_set():
                     break
                 with self.workflow_eval_lock:
-                    keys = list(self.processes.keys())
+                    keys = list(self.processes)
                 if not keys:
                     remain_wait = self.poll_interval
                     continue
@@ -351,7 +361,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                     try:
                         proc_states = table.list(filters=[["uuid", "in", page]]).execute(num_retries=self.num_retries)
                     except Exception as e:
-                        logger.warn("Error checking states on API server: %s", e)
+                        logger.warning("Error checking states on API server: %s", e)
                         remain_wait = self.poll_interval
                         continue
 
@@ -383,7 +393,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
             try:
                 self.api.collections().delete(uuid=i).execute(num_retries=self.num_retries)
             except:
-                logger.warn("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
+                logger.warning("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
             if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
                 break
 
@@ -401,7 +411,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                             "Option 'dockerOutputDirectory' must be an absolute path.")
             if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
                 raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
-            for v in obj.itervalues():
+            for v in viewvalues(obj):
                 self.check_features(v)
         elif isinstance(obj, list):
             for i,v in enumerate(obj):
@@ -445,7 +455,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                 logger.error("Creating CollectionReader for '%s' '%s': %s", k, v, e)
                 raise
             except IOError as e:
-                logger.warn("While preparing output collection: %s", e)
+                logger.warning("While preparing output collection: %s", e)
 
         def rewrite(fileobj):
             fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
@@ -457,7 +467,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
         adjustFileObjs(outputObj, rewrite)
 
         with final.open("cwl.output.json", "w") as f:
-            json.dump(outputObj, f, sort_keys=True, indent=4, separators=(',',': '))
+            res = json.dumps(outputObj, sort_keys=True, indent=4, separators=(',',': '), ensure_ascii=False).encode('utf-8').decode()
+            f.write(res)           
 
         final.save_new(name=name, owner_uuid=self.project_uuid, storage_classes=storage_classes, ensure_unique_name=True)
 
@@ -608,7 +619,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                         visited.add(m.group(1))
                         estimated_size[0] += int(m.group(2))
             visit_class(job_order, ("File", "Directory"), estimate_collection_cache)
-            runtimeContext.collection_cache_size = max(((estimated_size[0]*192) / (1024*1024))+1, 256)
+            runtimeContext.collection_cache_size = max(((estimated_size[0]*192) // (1024*1024))+1, 256)
             self.collection_cache.set_cap(runtimeContext.collection_cache_size*1024*1024)
 
         logger.info("Using collection cache size %s MiB", runtimeContext.collection_cache_size)
@@ -660,7 +671,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                            runtimeContext)
 
         if runtimeContext.submit and not runtimeContext.wait:
-            runnerjob = jobiter.next()
+            runnerjob = next(jobiter)
             runnerjob.run(runtimeContext)
             return (runnerjob.uuid, "success")
 

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 0816ee8fc05b74198ae9abad69887905bf8113ee..fc0bbb80b9ea05715770c2d3142e7208ca19dcb2 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -2,10 +2,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
+from future.utils import viewvalues
+
 import fnmatch
 import os
 import errno
-import urlparse
+import urllib.parse
 import re
 import logging
 import threading
@@ -48,7 +53,7 @@ class CollectionCache(object):
 
     def cap_cache(self, required):
         # ordered dict iterates from oldest to newest
-        for pdh, v in self.collections.items():
+        for pdh, v in list(self.collections.items()):
             available = self.cap - self.total
             if available >= required or len(self.collections) < self.min_entries:
                 return
@@ -90,7 +95,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
         p = sp[0]
         if p.startswith("keep:") and arvados.util.keep_locator_pattern.match(p[5:]):
             pdh = p[5:]
-            return (self.collection_cache.get(pdh), urlparse.unquote(sp[1]) if len(sp) == 2 else None)
+            return (self.collection_cache.get(pdh), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None)
         else:
             return (None, path)
 
@@ -188,7 +193,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
                 raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash()))
             if not isinstance(dir, arvados.collection.RichCollectionBase):
                 raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash()))
-            return [abspath(l, fn) for l in dir.keys()]
+            return [abspath(l, fn) for l in list(dir.keys())]
         else:
             return super(CollectionFsAccess, self).listdir(fn)
 
@@ -243,11 +248,11 @@ class CollectionFetcher(DefaultFetcher):
         if not url:
             return base_url
 
-        urlsp = urlparse.urlsplit(url)
+        urlsp = urllib.parse.urlsplit(url)
         if urlsp.scheme or not base_url:
             return url
 
-        basesp = urlparse.urlsplit(base_url)
+        basesp = urllib.parse.urlsplit(base_url)
         if basesp.scheme in ("keep", "arvwf"):
             if not basesp.path:
                 raise IOError(errno.EINVAL, "Invalid Keep locator", base_url)
@@ -268,7 +273,7 @@ class CollectionFetcher(DefaultFetcher):
                 baseparts.pop()
 
             path = "/".join([pdh] + baseparts + urlparts)
-            return urlparse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment))
+            return urllib.parse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment))
 
         return super(CollectionFetcher, self).urljoin(base_url, url)
 
@@ -283,21 +288,21 @@ pipeline_template_uuid_pattern = re.compile(r'[a-z0-9]{5}-p5p6p-[a-z0-9]{15}')
 
 def collectionResolver(api_client, document_loader, uri, num_retries=4):
     if uri.startswith("keep:") or uri.startswith("arvwf:"):
-        return uri
+        return uri.encode("utf-8").decode()
 
     if workflow_uuid_pattern.match(uri):
-        return "arvwf:%s#main" % (uri)
+        return u"arvwf:%s#main" % (uri)
 
     if pipeline_template_uuid_pattern.match(uri):
         pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
-        return "keep:" + pt["components"].values()[0]["script_parameters"]["cwl:tool"]
+        return u"keep:" + viewvalues(pt["components"])[0]["script_parameters"]["cwl:tool"]
 
     p = uri.split("/")
     if arvados.util.keep_locator_pattern.match(p[0]):
-        return "keep:%s" % (uri)
+        return u"keep:%s" % (uri)
 
     if arvados.util.collection_uuid_pattern.match(p[0]):
-        return "keep:%s%s" % (api_client.collections().
+        return u"keep:%s%s" % (api_client.collections().
                               get(uuid=p[0]).execute()["portable_data_hash"],
                               uri[len(p[0]):])
 

diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py
index 4516de021b8522f25990cf2988b7b4345f13849d..ccc2e793b067f4e0853da275947a08d3630723a3 100644 (file)
--- a/sdk/cwl/arvados_cwl/http.py
+++ b/sdk/cwl/arvados_cwl/http.py
@@ -2,6 +2,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+
 import requests
 import email.utils
 import time
@@ -9,7 +13,7 @@ import datetime
 import re
 import arvados
 import arvados.collection
-import urlparse
+import urllib.parse
 import logging
 import calendar
 
@@ -91,13 +95,13 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
         if fresh_cache(url, properties, now):
             # Do nothing
             cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
-            return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0])
+            return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0])
 
         if not changed(url, properties, now):
             # ETag didn't change, same content, just update headers
             api.collections().update(uuid=item["uuid"], body={"collection":{"properties": properties}}).execute()
             cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
-            return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0])
+            return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0])
 
     properties = {}
     req = requests.get(url, stream=True, allow_redirects=True)
@@ -123,7 +127,7 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
         else:
             name = grp.group(4)
     else:
-        name = urlparse.urlparse(url).path.split("/")[-1]
+        name = urllib.parse.urlparse(url).path.split("/")[-1]
 
     count = 0
     start = time.time()
@@ -134,14 +138,14 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
             f.write(chunk)
             loopnow = time.time()
             if (loopnow - checkpoint) > 20:
-                bps = (float(count)/float(loopnow - start))
+                bps = count / (loopnow - start)
                 if cl is not None:
                     logger.info("%2.1f%% complete, %3.2f MiB/s, %1.0f seconds left",
-                                float(count * 100) / float(cl),
-                                bps/(1024*1024),
-                                (cl-count)/bps)
+                                ((count * 100) / cl),
+                                (bps // (1024*1024)),
+                                ((cl-count) // bps))
                 else:
-                    logger.info("%d downloaded, %3.2f MiB/s", count, bps/(1024*1024))
+                    logger.info("%d downloaded, %3.2f MiB/s", count, (bps / (1024*1024)))
                 checkpoint = loopnow
 
     c.save_new(name="Downloaded from %s" % url, owner_uuid=project_uuid, ensure_unique_name=True)

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 0b2a22788e6f98537b0f5a3437a2d540a57d47ee..e0445febdc9a0731314607417739747c8f0e632c 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -2,11 +2,17 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from past.builtins import basestring
+from future.utils import viewitems
+
 import re
 import logging
 import uuid
 import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 import arvados_cwl.util
 import arvados.commands.run
@@ -60,7 +66,7 @@ class ArvPathMapper(PathMapper):
             src = src[:src.index("#")]
 
         if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
-            self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
+            self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True)
 
         debug = logger.isEnabledFor(logging.DEBUG)
 
@@ -77,7 +83,7 @@ class ArvPathMapper(PathMapper):
                     if isinstance(st, arvados.commands.run.UploadFile):
                         uploadfiles.add((src, ab, st))
                     elif isinstance(st, arvados.commands.run.ArvFile):
-                        self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True)
+                        self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True)
                     else:
                         raise WorkflowException("Input file path '%s' is invalid" % st)
             elif src.startswith("_:"):
@@ -114,7 +120,7 @@ class ArvPathMapper(PathMapper):
             remap.append((obj["location"], path + "/" + obj["basename"]))
         elif obj["location"].startswith("_:") and "contents" in obj:
             with c.open(path + "/" + obj["basename"], "w") as f:
-                f.write(obj["contents"].encode("utf-8"))
+                f.write(obj["contents"])
             remap.append((obj["location"], path + "/" + obj["basename"]))
         else:
             raise SourceLine(obj, "location", WorkflowException).makeError("Don't know what to do with '%s'" % obj["location"])
@@ -176,7 +182,7 @@ class ArvPathMapper(PathMapper):
                                          packed=False)
 
         for src, ab, st in uploadfiles:
-            self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
+            self._pathmap[src] = MapperEnt(urllib.parse.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
                                            "Directory" if os.path.isdir(ab) else "File", True)
 
         for srcobj in referenced_files:
@@ -228,7 +234,7 @@ class ArvPathMapper(PathMapper):
                                                               ab, "File", True)
                 if srcobj.get("secondaryFiles"):
                     ab = self.collection_pattern % c.portable_data_hash()
-                    self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
+                    self._pathmap["_:" + str(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
 
             if remap:
                 for loc, sub in remap:
@@ -301,7 +307,7 @@ class VwdPathMapper(StagingPathMapper):
         # with any secondary files.
         self.visitlisting(referenced_files, self.stagedir, basedir)
 
-        for path, (ab, tgt, type, staged) in self._pathmap.items():
+        for path, (ab, tgt, type, staged) in viewitems(self._pathmap):
             if type in ("File", "Directory") and ab.startswith("keep:"):
                 self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type, staged)
 

diff --git a/sdk/cwl/arvados_cwl/perf.py b/sdk/cwl/arvados_cwl/perf.py
index 39f475fe8d30e6fda700e2f3c965577745b0638f..cc3ea969df99cb000119cc60ffd5c29c28656d01 100644 (file)
--- a/sdk/cwl/arvados_cwl/perf.py
+++ b/sdk/cwl/arvados_cwl/perf.py
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import object
+
 import time
 import uuid
 

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4b3275fa36b4ee497ec83c3de2b3e8a0c938fb62..b65e2c58e180a00f6bb8bca6688498880c3d5e00 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -2,15 +2,19 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+from future.utils import  viewvalues, viewitems
+
 import os
-import urlparse
+import urllib.parse
 from functools import partial
 import logging
 import json
 import subprocess32 as subprocess
 from collections import namedtuple
 
-from StringIO import StringIO
+from io import StringIO
 
 from schema_salad.sourceline import SourceLine, cmap
 
@@ -61,7 +65,7 @@ def find_defaults(d, op):
         if "default" in d:
             op(d)
         else:
-            for i in d.itervalues():
+            for i in viewvalues(d):
                 find_defaults(i, op)
 
 def setSecondary(t, fileobj, discovered):
@@ -98,7 +102,7 @@ def upload_dependencies(arvrunner, name, document_loader,
     loaded = set()
     def loadref(b, u):
         joined = document_loader.fetcher.urljoin(b, u)
-        defrg, _ = urlparse.urldefrag(joined)
+        defrg, _ = urllib.parse.urldefrag(joined)
         if defrg not in loaded:
             loaded.add(defrg)
             # Use fetch_text to get raw file (before preprocessing).
@@ -171,7 +175,7 @@ def upload_dependencies(arvrunner, name, document_loader,
 
     visit_class(workflowobj, ("CommandLineTool", "Workflow"), discover_default_secondary_files)
 
-    for d in list(discovered.keys()):
+    for d in list(discovered):
         # Only interested in discovered secondaryFiles which are local
         # files that need to be uploaded.
         if d.startswith("file:"):
@@ -232,7 +236,7 @@ def packed_workflow(arvrunner, tool, merged_map):
     packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
                   tool.tool["id"], tool.metadata, rewrite_out=rewrites)
 
-    rewrite_to_orig = {v: k for k,v in rewrites.items()}
+    rewrite_to_orig = {v: k for k,v in viewitems(rewrites)}
 
     def visit(v, cur_id):
         if isinstance(v, dict):

diff --git a/sdk/cwl/arvados_cwl/task_queue.py b/sdk/cwl/arvados_cwl/task_queue.py
index 1c233fac0ad98f4b0421a4e0856b00fd19d1422f..d75fec6c63e719949d6f19b7d2813d9f828262a6 100644 (file)
--- a/sdk/cwl/arvados_cwl/task_queue.py
+++ b/sdk/cwl/arvados_cwl/task_queue.py
@@ -2,7 +2,12 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import Queue
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import object
+
+import queue
 import threading
 import logging
 
@@ -11,13 +16,13 @@ logger = logging.getLogger('arvados.cwl-runner')
 class TaskQueue(object):
     def __init__(self, lock, thread_count):
         self.thread_count = thread_count
-        self.task_queue = Queue.Queue(maxsize=self.thread_count)
+        self.task_queue = queue.Queue(maxsize=self.thread_count)
         self.task_queue_threads = []
         self.lock = lock
         self.in_flight = 0
         self.error = None
 
-        for r in xrange(0, self.thread_count):
+        for r in range(0, self.thread_count):
             t = threading.Thread(target=self.task_queue_func)
             self.task_queue_threads.append(t)
             t.start()
@@ -51,7 +56,7 @@ class TaskQueue(object):
                     return
                 self.task_queue.put(task, block=True, timeout=3)
                 return
-            except Queue.Full:
+            except queue.Full:
                 pass
             finally:
                 unlock.acquire()
@@ -62,7 +67,7 @@ class TaskQueue(object):
             # Drain queue
             while not self.task_queue.empty():
                 self.task_queue.get(True, .1)
-        except Queue.Empty:
+        except queue.Empty:
             pass
 
     def join(self):

diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
index 98a2a89a1d281e056ee3067752e6094349123115..776fc6bc25dae06e232e2546cab501246d6cd6b3 100644 (file)
--- a/sdk/cwl/arvados_cwl/util.py
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -26,6 +26,9 @@ def get_current_container(api, num_retries=0, logger=None):
         current_container = api.containers().current().execute(num_retries=num_retries)
     except ApiError as e:
         # Status code 404 just means we're not running in a container.
-        if e.resp.status != 404 and logger:
-            logger.info("Getting current container: %s", e)
+        if e.resp.status != 404:
+            if logger:
+                logger.info("Getting current container: %s", e)
+            raise e
+            
     return current_container

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d13dd5ec538e678268d7b79836d745ba89d46047..fa35fe7da6380baf1f82cb8a890e21e9e31a7afd 100644 (file)
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import str
+from builtins import next
+
 import subprocess
 import time
 import os

diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
index 4dc8448476123934dae7193fe680141671a2b7ec..d6a4c24a785e0ca6258550dee4b41c2cc01b9ad7 100644 (file)
--- a/sdk/cwl/gittaggers.py
+++ b/sdk/cwl/gittaggers.py
@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import str
+from builtins import next
+
 from setuptools.command.egg_info import egg_info
 import subprocess
 import time

diff --git a/sdk/cwl/tests/federation/framework/check_exist.py b/sdk/cwl/tests/federation/framework/check_exist.py
index b3338939edd983850ec7d426fa32619176cc4d84..1458772a3f65f22dfe494df7ddbe55d7ca308f2e 100644 (file)
--- a/sdk/cwl/tests/federation/framework/check_exist.py
+++ b/sdk/cwl/tests/federation/framework/check_exist.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import print_function
 import arvados
 import json
 

diff --git a/sdk/cwl/tests/federation/framework/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py
index 6fe90813e7a720c9ba5c11d9650af34e49ee9cdd..40bb843b2980877a0dbe10f18b41463c255609e5 100644 (file)
--- a/sdk/cwl/tests/federation/framework/prepare.py
+++ b/sdk/cwl/tests/federation/framework/prepare.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import print_function
 import arvados
 import json
 

diff --git a/sdk/cwl/tests/hw.py b/sdk/cwl/tests/hw.py
index 99c26523ed471dc6d14b165aee9cceba5d9e57b5..e45bd72642df7e735bdc0d5529286536bf536de8 100644 (file)
--- a/sdk/cwl/tests/hw.py
+++ b/sdk/cwl/tests/hw.py
@@ -2,4 +2,5 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-print "Hello world"
+from __future__ import print_function
+print("Hello world")

diff --git a/sdk/cwl/tests/matcher.py b/sdk/cwl/tests/matcher.py
index 50c0c60b1d0a337527ddd2342ae1ed46e22b1708..04e67b7dbd48aaefec075e77c2a8836d81cb2631 100644 (file)
--- a/sdk/cwl/tests/matcher.py
+++ b/sdk/cwl/tests/matcher.py
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import object
+
 import difflib
 import json
 import re

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index a34d550f62ad2b24a1ff156d7b740377c811cab5..de21fc0b92d1c7437978be0a5018c3ea51fd76c3 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import str
+from builtins import object
+
 import arvados_cwl
 import arvados_cwl.context
 import arvados_cwl.util

diff --git a/sdk/cwl/tests/test_http.py b/sdk/cwl/tests/test_http.py
index 0c66c39c0b4607e492725a280d8dfca3aa52ec19..88bd49fcd329faee02fc29f777adbdd41d3f21a2 100644 (file)
--- a/sdk/cwl/tests/test_http.py
+++ b/sdk/cwl/tests/test_http.py
@@ -2,8 +2,11 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+
 import copy
-import cStringIO
+import io
 import functools
 import hashlib
 import json

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 2aaac0ae50699f5c012f36ba2f28eee1ccd281c4..022d75be11a09031746adf7a39975793ec293e83 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -2,6 +2,11 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from builtins import next
+
 import functools
 import json
 import logging
@@ -9,7 +14,7 @@ import mock
 import os
 import unittest
 import copy
-import StringIO
+import io
 
 import arvados
 import arvados_cwl
@@ -126,7 +131,7 @@ class TestJob(unittest.TestCase):
                     # sharing link on the job
                     runner.api.links().create.side_effect = ApiError(
                         mock.MagicMock(return_value={'status': 403}),
-                        'Permission denied')
+                        bytes(b'Permission denied'))
                     j.run(runtimeContext)
                 else:
                     assert not runner.api.links().create.called
@@ -213,11 +218,12 @@ class TestJob(unittest.TestCase):
         runner.num_retries = 0
         runner.ignore_docker_for_reuse = False
 
-        reader().open.return_value = StringIO.StringIO(
-            """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
+        reader().keys.return_value = "log.txt"
+        reader().open.return_value = io.StringIO(
+            str(u"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
-        """)
+        """))
         api.collections().list().execute.side_effect = ({"items": []},
                                                         {"items": [{"manifest_text": "XYZ"}]},
                                                         {"items": []},
@@ -286,11 +292,12 @@ class TestJob(unittest.TestCase):
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
 
-        reader().open.return_value = StringIO.StringIO(
-            """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
+        reader().keys.return_value = "log.txt"
+        reader().open.return_value = io.StringIO(
+            str(u"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
-        """)
+        """))
 
         api.collections().list().execute.side_effect = (
             {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
@@ -398,8 +405,8 @@ class TestWorkflow(unittest.TestCase):
         arvtool.formatgraph = None
         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
 
-        it.next().run(runtimeContext)
-        it.next().run(runtimeContext)
+        next(it).run(runtimeContext)
+        next(it).run(runtimeContext)
 
         with open("tests/wf/scatter2_subwf.cwl") as f:
             subwf = StripYAMLComments(f.read())
@@ -435,7 +442,7 @@ class TestWorkflow(unittest.TestCase):
 
         mockc.open().__enter__().write.assert_has_calls([mock.call(subwf)])
         mockc.open().__enter__().write.assert_has_calls([mock.call(
-'''{
+bytes(b'''{
   "fileblub": {
     "basename": "token.txt",
     "class": "File",
@@ -443,7 +450,7 @@ class TestWorkflow(unittest.TestCase):
     "size": 0
   },
   "sleeptime": 5
-}''')])
+}'''))])
 
     # The test passes no builder.resources
     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@@ -477,8 +484,9 @@ class TestWorkflow(unittest.TestCase):
         arvtool = arvados_cwl.ArvadosWorkflow(runner, tool, loadingContext)
         arvtool.formatgraph = None
         it = arvtool.job({}, mock.MagicMock(), runtimeContext)
-        it.next().run(runtimeContext)
-        it.next().run(runtimeContext)
+        
+        next(it).run(runtimeContext)
+        next(it).run(runtimeContext)
 
         with open("tests/wf/echo-subwf.cwl") as f:
             subwf = StripYAMLComments(f.read())

diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index baeb4145ee6dbc5ba4db326f88acd54ce04352f4..562d1765daa24254cf4ad1a33b03af1129f57eff 100644 (file)
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -2,12 +2,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+
 import functools
 import json
 import logging
 import mock
 import os
-import StringIO
+import io
 import unittest
 
 import arvados
@@ -35,7 +38,7 @@ class TestMakeOutput(unittest.TestCase):
         final_uuid = final.manifest_locator()
         num_retries = runner.num_retries
 
-        cwlout = StringIO.StringIO()
+        cwlout = io.StringIO()
         openmock = mock.MagicMock()
         final.open.return_value = openmock
         openmock.__enter__.return_value = cwlout

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 90dab01471ef61ab380955e6301a73306648edef..782282d2a0d4312a8a25a18b970d009bc6a9f66f 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -2,8 +2,14 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
+from builtins import str
+from future.utils import viewvalues
+
 import copy
-import cStringIO
+import io
 import functools
 import hashlib
 import json
@@ -12,6 +18,8 @@ import mock
 import sys
 import unittest
 
+from io import BytesIO, StringIO
+
 import arvados
 import arvados.collection
 import arvados_cwl
@@ -33,7 +41,7 @@ def stubs(func):
     @mock.patch("arvados.keep.KeepClient")
     @mock.patch("arvados.events.subscribe")
     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
-        class Stubs:
+        class Stubs(object):
             pass
         stubs = Stubs()
         stubs.events = events
@@ -60,6 +68,11 @@ def stubs(func):
         stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
         stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
 
+        if sys.version_info[0] < 3:
+            stubs.capture_stdout = BytesIO()
+        else:
+            stubs.capture_stdout = StringIO()
+
         stubs.api = mock.MagicMock()
         stubs.api._rootDesc = get_rootDesc()
 
@@ -78,18 +91,18 @@ def stubs(func):
                 return self.exe
 
         def collection_createstub(created_collections, body, ensure_unique_name=None):
-            mt = body["manifest_text"]
+            mt = body["manifest_text"].encode('utf-8')
             uuid = "zzzzz-4zz18-zzzzzzzzzzzzzx%d" % len(created_collections)
             pdh = "%s+%i" % (hashlib.md5(mt).hexdigest(), len(mt))
             created_collections[uuid] = {
                 "uuid": uuid,
                 "portable_data_hash": pdh,
-                "manifest_text": mt
+                "manifest_text": mt.decode('utf-8')
             }
             return CollectionExecute(created_collections[uuid])
 
         def collection_getstub(created_collections, uuid):
-            for v in created_collections.itervalues():
+            for v in viewvalues(created_collections):
                 if uuid in (v["uuid"], v["portable_data_hash"]):
                     return CollectionExecute(v)
 
@@ -318,12 +331,10 @@ class TestSubmit(unittest.TestCase):
                 return '999999999999999999999999999999d4+99'
         arvdock.side_effect = get_image
 
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.collections().create.assert_has_calls([
             mock.call(body=JsonDiffMatcher({
@@ -354,19 +365,17 @@ class TestSubmit(unittest.TestCase):
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @mock.patch("time.sleep")
     @stubs
     def test_submit_no_reuse(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:enable_reuse"] = {"value": False}
@@ -374,8 +383,9 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_error_when_multiple_storage_classes_specified(self, stubs):
@@ -389,49 +399,44 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_on_error(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:on_error"] = "stop"
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @mock.patch("time.sleep")
     @stubs
     def test_submit_runner_ram(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         expect_pipeline["components"]["cwl-runner"]["runtime_constraints"]["min_ram_mb_per_node"] = 2048
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @mock.patch("time.sleep")
     @stubs
     def test_submit_invalid_runner_ram(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 1)
 
     @mock.patch("time.sleep")
@@ -439,30 +444,27 @@ class TestSubmit(unittest.TestCase):
     def test_submit_output_name(self, stubs, tm):
         output_name = "test_output_name"
 
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["arv:output_name"] = output_name
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @mock.patch("time.sleep")
     @stubs
     def test_submit_pipeline_name(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
@@ -470,7 +472,7 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
 
     @mock.patch("time.sleep")
@@ -478,11 +480,10 @@ class TestSubmit(unittest.TestCase):
     def test_submit_output_tags(self, stubs, tm):
         output_tags = "tag0,tag1,tag2"
 
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
@@ -490,7 +491,7 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
 
     @mock.patch("time.sleep")
@@ -511,16 +512,11 @@ class TestSubmit(unittest.TestCase):
             body=JsonDiffMatcher(expect_pipeline))
 
     @stubs
-    def test_submit_container(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+    def test_submit_container(self, stubs):        
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         stubs.api.collections().create.assert_has_calls([
             mock.call(body=JsonDiffMatcher({
@@ -539,20 +535,16 @@ class TestSubmit(unittest.TestCase):
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_no_reuse(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = [
@@ -566,18 +558,16 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
-                         stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n') 
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_wf_no_reuse.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
         self.assertEqual(exited, 0)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
@@ -602,22 +592,17 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
 
 
     @stubs
     def test_submit_container_on_error(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
-
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+ 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
                                        '--no-log-timestamps', '--disable-validate',
@@ -628,22 +613,18 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_output_name(self, stubs):
         output_name = "test_output_name"
-
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+  
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -656,20 +637,16 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
-    def test_submit_storage_classes(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+    def test_submit_storage_classes(self, stubs):     
+        exited = arvados_cwl.main(
+            ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -681,8 +658,9 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @mock.patch("arvados_cwl.task_queue.TaskQueue")
     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@@ -694,16 +672,13 @@ class TestSubmit(unittest.TestCase):
             return []
         job.side_effect = set_final_output
 
-        try:
-            exited = arvados_cwl.main(
-                ["--debug", "--local", "--storage-classes=foo",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--debug", "--local", "--storage-classes=foo",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+        self.assertEqual(exited, 0)
 
     @mock.patch("arvados_cwl.task_queue.TaskQueue")
     @mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@@ -715,28 +690,20 @@ class TestSubmit(unittest.TestCase):
             return []
         job.side_effect = set_final_output
 
-        try:
-            exited = arvados_cwl.main(
-                ["--debug", "--local",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--debug", "--local",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            sys.stdin, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         make_output.assert_called_with(u'Output of submit_wf.cwl', ['default'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_output_ttl(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -749,20 +716,17 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_trash_intermediate(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -775,22 +739,18 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_output_tags(self, stubs):
         output_tags = "tag0,tag1,tag2"
 
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -802,56 +762,48 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_runner_ram(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["runtime_constraints"]["ram"] = (2048+256)*1024*1024
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @mock.patch("arvados.collection.CollectionReader")
     @mock.patch("time.sleep")
     @stubs
     def test_submit_file_keepref(self, stubs, tm, collectionReader):
-        capture_stdout = cStringIO.StringIO()
         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_keepref_wf.cwl"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
 
-
     @mock.patch("arvados.collection.CollectionReader")
     @mock.patch("time.sleep")
     @stubs
     def test_submit_keepref(self, stubs, tm, reader):
-        capture_stdout = cStringIO.StringIO()
-
         with open("tests/wf/expect_arvworkflow.cwl") as f:
             reader().open().__enter__().read.return_value = f.read()
 
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_container = {
             'priority': 500,
@@ -896,24 +848,21 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @mock.patch("arvados.collection.CollectionReader")
     @mock.patch("time.sleep")
     @stubs
     def test_submit_jobs_keepref(self, stubs, tm, reader):
-        capture_stdout = cStringIO.StringIO()
-
         with open("tests/wf/expect_arvworkflow.cwl") as f:
             reader().open().__enter__().read.return_value = f.read()
 
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug",
              "keep:99999999999999999999999999999994+99/expect_arvworkflow.cwl#main", "-x", "XxX"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         expect_pipeline["components"]["cwl-runner"]["script_parameters"]["x"] = "XxX"
@@ -923,20 +872,18 @@ class TestSubmit(unittest.TestCase):
         expect_pipeline["name"] = "expect_arvworkflow.cwl#main"
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
+        self.assertEqual(exited, 0)
 
     @mock.patch("time.sleep")
     @stubs
     def test_submit_arvworkflow(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
-
         with open("tests/wf/expect_arvworkflow.cwl") as f:
             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
 
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "962eh-7fd4e-gkbzl62qqtfig37", "-x", "XxX"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_container = {
             'priority': 500,
@@ -1020,60 +967,47 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_name(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["name"] = "hello container 123"
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_missing_input(self, stubs):
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
         self.assertEqual(exited, 0)
 
-        capture_stdout = cStringIO.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
         self.assertEqual(exited, 1)
 
-
     @stubs
     def test_submit_container_project(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["owner_uuid"] = project_uuid
@@ -1087,21 +1021,16 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_eval_timeout(self, stubs):
-        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -1113,21 +1042,16 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_collection_cache(self, stubs):
-        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -1140,22 +1064,16 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_thread_count(self, stubs):
-        project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
@@ -1167,82 +1085,64 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_job_runner_image(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         stubs.expect_pipeline_instance["components"]["cwl-runner"]["runtime_constraints"]["docker_image"] = "999999999999999999999999999999d5+99"
 
         expect_pipeline = copy.deepcopy(stubs.expect_pipeline_instance)
         stubs.api.pipeline_instances().create.assert_called_with(
             body=JsonDiffMatcher(expect_pipeline))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_runner_image(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         stubs.expect_container_spec["container_image"] = "999999999999999999999999999999d5+99"
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_priority(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         stubs.expect_container_spec["priority"] = 669
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_wf_runner_resources(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug",
-                 "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug",
+                "tests/wf/submit_wf_runner_resources.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
         expect_container["runtime_constraints"] = {
@@ -1270,8 +1170,9 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     def tearDown(self):
         arvados_cwl.arvdocker.arv_docker_clear_cache()
@@ -1315,16 +1216,10 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_secrets(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug",
-                 "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
-
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug",
+                "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = {
             "command": [
@@ -1479,8 +1374,9 @@ class TestSubmit(unittest.TestCase):
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_request_uuid(self, stubs):
@@ -1492,50 +1388,41 @@ class TestSubmit(unittest.TestCase):
             "state": "Queued"
         }
 
-        capture_stdout = cStringIO.StringIO()
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         stubs.api.container_requests().update.assert_called_with(
             uuid="zzzzz-xvhdp-yyyyyyyyyyyyyyy", body=JsonDiffMatcher(stubs.expect_container_spec), cluster_id="zzzzz")
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_container_cluster_id(self, stubs):
-        capture_stdout = cStringIO.StringIO()
         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
-        try:
-            exited = arvados_cwl.main(
-                ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
-                 "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-                capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
-            self.assertEqual(exited, 0)
-        except:
-            logging.exception("")
+
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zbbbb",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
 
         expect_container = copy.deepcopy(stubs.expect_container_spec)
 
         stubs.api.container_requests().create.assert_called_with(
             body=JsonDiffMatcher(expect_container), cluster_id="zbbbb")
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_container_request_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_submit_validate_cluster_id(self, stubs):
-        capture_stdout = cStringIO.StringIO()
         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
         self.assertEqual(exited, 1)
 
 
@@ -1565,15 +1452,12 @@ class TestCreateTemplate(unittest.TestCase):
     def test_create(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
              "--api=jobs",
              "--project-uuid", project_uuid,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.pipeline_instances().create.refute_called()
         stubs.api.jobs().create.refute_called()
@@ -1590,24 +1474,21 @@ class TestCreateTemplate(unittest.TestCase):
         stubs.api.pipeline_templates().create.assert_called_with(
             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_template_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_create_name(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
              "--project-uuid", project_uuid,
              "--api=jobs",
              "--name", "testing 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.pipeline_instances().create.refute_called()
         stubs.api.jobs().create.refute_called()
@@ -1624,16 +1505,14 @@ class TestCreateTemplate(unittest.TestCase):
         stubs.api.pipeline_templates().create.assert_called_with(
             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
 
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_pipeline_template_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_update_name(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_template_uuid,
              "--debug",
@@ -1641,8 +1520,7 @@ class TestCreateTemplate(unittest.TestCase):
              "--api=jobs",
              "--name", "testing 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.pipeline_instances().create.refute_called()
         stubs.api.jobs().create.refute_called()
@@ -1660,9 +1538,10 @@ class TestCreateTemplate(unittest.TestCase):
         stubs.api.pipeline_templates().update.assert_called_with(
             body=JsonDiffMatcher(expect_template), uuid=self.existing_template_uuid)
 
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          self.existing_template_uuid + '\n')
-
+        self.assertEqual(exited, 0)
+        
 
 class TestCreateWorkflow(unittest.TestCase):
     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
@@ -1673,15 +1552,12 @@ class TestCreateWorkflow(unittest.TestCase):
     def test_create(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
              "--api=containers",
              "--project-uuid", project_uuid,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.pipeline_templates().create.refute_called()
         stubs.api.container_requests().create.refute_called()
@@ -1697,24 +1573,21 @@ class TestCreateWorkflow(unittest.TestCase):
         stubs.api.workflows().create.assert_called_with(
             body=JsonDiffMatcher(body))
 
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_workflow_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_create_name(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
              "--api=containers",
              "--project-uuid", project_uuid,
              "--name", "testing 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.pipeline_templates().create.refute_called()
         stubs.api.container_requests().create.refute_called()
@@ -1730,14 +1603,16 @@ class TestCreateWorkflow(unittest.TestCase):
         stubs.api.workflows().create.assert_called_with(
             body=JsonDiffMatcher(body))
 
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_workflow_uuid + '\n')
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_incompatible_api(self, stubs):
-        capture_stderr = cStringIO.StringIO()
-        logging.getLogger('arvados.cwl-runner').addHandler(
-            logging.StreamHandler(capture_stderr))
+        capture_stderr = io.StringIO()
+        acr_logger = logging.getLogger('arvados.cwl-runner')
+        stderr_logger = logging.StreamHandler(capture_stderr)
+        acr_logger.addHandler(stderr_logger)
 
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
@@ -1749,17 +1624,15 @@ class TestCreateWorkflow(unittest.TestCase):
         self.assertRegexpMatches(
             capture_stderr.getvalue(),
             "--update-workflow arg '{}' uses 'containers' API, but --api='jobs' specified".format(self.existing_workflow_uuid))
+        acr_logger.removeHandler(stderr_logger)
 
     @stubs
     def test_update(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
              "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         body = {
             "workflow": {
@@ -1771,20 +1644,17 @@ class TestCreateWorkflow(unittest.TestCase):
         stubs.api.workflows().update.assert_called_with(
             uuid=self.existing_workflow_uuid,
             body=JsonDiffMatcher(body))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          self.existing_workflow_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_update_name(self, stubs):
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
              "--debug", "--name", "testing 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         body = {
             "workflow": {
@@ -1796,23 +1666,20 @@ class TestCreateWorkflow(unittest.TestCase):
         stubs.api.workflows().update.assert_called_with(
             uuid=self.existing_workflow_uuid,
             body=JsonDiffMatcher(body))
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          self.existing_workflow_uuid + '\n')
-
+        self.assertEqual(exited, 0)
 
     @stubs
     def test_create_collection_per_tool(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
-
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
              "--api=containers",
              "--project-uuid", project_uuid,
              "tests/collection_per_tool/collection_per_tool.cwl"],
-            capture_stdout, sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
         expect_workflow = StripYAMLComments(open(toolfile).read())
@@ -1828,8 +1695,9 @@ class TestCreateWorkflow(unittest.TestCase):
         stubs.api.workflows().create.assert_called_with(
             body=JsonDiffMatcher(body))
 
-        self.assertEqual(capture_stdout.getvalue(),
+        self.assertEqual(stubs.capture_stdout.getvalue(),
                          stubs.expect_workflow_uuid + '\n')
+        self.assertEqual(exited, 0)
 
 class TestTemplateInputs(unittest.TestCase):
     expect_template = {
@@ -1884,19 +1752,19 @@ class TestTemplateInputs(unittest.TestCase):
         exited = arvados_cwl.main(
             ["--create-template",
              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
-            cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         stubs.api.pipeline_templates().create.assert_called_with(
             body=JsonDiffMatcher(self.expect_template), ensure_unique_name=True)
 
+        self.assertEqual(exited, 0)
+
     @stubs
     def test_inputs(self, stubs):
         exited = arvados_cwl.main(
             ["--create-template",
              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
-            cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
-        self.assertEqual(exited, 0)
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api)
 
         expect_template = copy.deepcopy(self.expect_template)
         params = expect_template[
@@ -1908,3 +1776,4 @@ class TestTemplateInputs(unittest.TestCase):
 
         stubs.api.pipeline_templates().create.assert_called_with(
             body=JsonDiffMatcher(expect_template), ensure_unique_name=True)
+        self.assertEqual(exited, 0)

diff --git a/sdk/cwl/tests/test_urljoin.py b/sdk/cwl/tests/test_urljoin.py
index 948f31d4f9815f3deadf595c1f5cbd3e1a464c29..86a053ea484835980c791d4a728824919d01f78f 100644 (file)
--- a/sdk/cwl/tests/test_urljoin.py
+++ b/sdk/cwl/tests/test_urljoin.py
@@ -23,37 +23,37 @@ class TestUrljoin(unittest.TestCase):
 
         cf = CollectionFetcher({}, None)
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/hw.py",
+        self.assertEqual("keep:99999999999999999999999999999991+99/hw.py",
                           cf.urljoin("keep:99999999999999999999999999999991+99", "hw.py"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/hw.py",
+        self.assertEqual("keep:99999999999999999999999999999991+99/hw.py",
                           cf.urljoin("keep:99999999999999999999999999999991+99/", "hw.py"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/hw.py#main",
+        self.assertEqual("keep:99999999999999999999999999999991+99/hw.py#main",
                           cf.urljoin("keep:99999999999999999999999999999991+99", "hw.py#main"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/hw.py#main",
+        self.assertEqual("keep:99999999999999999999999999999991+99/hw.py#main",
                           cf.urljoin("keep:99999999999999999999999999999991+99/hw.py", "#main"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/dir/hw.py#main",
+        self.assertEqual("keep:99999999999999999999999999999991+99/dir/hw.py#main",
                           cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "#main"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/dir/wh.py",
+        self.assertEqual("keep:99999999999999999999999999999991+99/dir/wh.py",
                           cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "wh.py"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/wh.py",
+        self.assertEqual("keep:99999999999999999999999999999991+99/wh.py",
                           cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "/wh.py"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/wh.py#main",
+        self.assertEqual("keep:99999999999999999999999999999991+99/wh.py#main",
                           cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "/wh.py#main"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/wh.py",
+        self.assertEqual("keep:99999999999999999999999999999991+99/wh.py",
                           cf.urljoin("keep:99999999999999999999999999999991+99/hw.py#main", "wh.py"))
 
-        self.assertEquals("keep:99999999999999999999999999999992+99",
+        self.assertEqual("keep:99999999999999999999999999999992+99",
                           cf.urljoin("keep:99999999999999999999999999999991+99", "keep:99999999999999999999999999999992+99"))
 
-        self.assertEquals("keep:99999999999999999999999999999991+99/dir/wh.py",
+        self.assertEqual("keep:99999999999999999999999999999991+99/dir/wh.py",
                           cf.urljoin("keep:99999999999999999999999999999991+99/dir/", "wh.py"))
 
     def test_resolver(self):

diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py
index 2532bd596c0f3dfac3554d3b5fe31ffb9d7e50d4..3ca02c7df65f023c32dab341224534f588d32e67 100644 (file)
--- a/sdk/cwl/tests/test_util.py
+++ b/sdk/cwl/tests/test_util.py
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import bytes
+
 import unittest
 import mock
 import datetime
@@ -39,7 +41,16 @@ class TestUtil(unittest.TestCase):
 
     def test_get_current_container_error(self):
         api = mock.MagicMock()
-        api.containers().current().execute.side_effect = ApiError(httplib2.Response({"status": 300}), "")
+        api.containers().current().execute.side_effect = ApiError(httplib2.Response({"status": 300}), bytes(b""))
+        logger = mock.MagicMock()
+
+        with self.assertRaises(ApiError):
+            get_current_container(api, num_retries=0, logger=logger)
+
+    def test_get_current_container_404_error(self):
+        api = mock.MagicMock()
+        api.containers().current().execute.side_effect = ApiError(httplib2.Response({"status": 404}), bytes(b""))
         logger = mock.MagicMock()
 
-        self.assertRaises(ApiError, get_current_container(api, num_retries=0, logger=logger))
+        current_container = get_current_container(api, num_retries=0, logger=logger)
+        self.assertEqual(current_container, None)
\ No newline at end of file

diff --git a/sdk/cwl/tests/wf/check_mem.py b/sdk/cwl/tests/wf/check_mem.py
index 355872232bc7f430a8b61f7e8f8dffbe09cc5530..b4322a809320b7be5823296f6cb72a39d4273f24 100644 (file)
--- a/sdk/cwl/tests/wf/check_mem.py
+++ b/sdk/cwl/tests/wf/check_mem.py
@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from __future__ import print_function
+from __future__ import division
+
 import arvados
 import sys
 import os
@@ -9,7 +12,7 @@ import os
 if "JOB_UUID" in os.environ:
     requested = arvados.api().jobs().get(uuid=os.environ["JOB_UUID"]).execute()["runtime_constraints"]["min_ram_mb_per_node"]
 else:
-    requested = arvados.api().containers().current().execute()["runtime_constraints"]["ram"]/(1024*1024)
+    requested = arvados.api().containers().current().execute()["runtime_constraints"]["ram"] // (1024*1024)
 
 print("Requested %d expected %d" % (requested, int(sys.argv[1])))
 

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index d33956ccc3f74caa7d6b64958b4c9863f09bbd70..9c4a5aa3345c9a900a87e04884f057250d497f12 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -18,9 +18,10 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN apt-get update -q && apt-get install -qy git python-pip python-virtualenv python-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
+ARG pythoncmd=python
 
-RUN pip install -U setuptools six requests
+RUN apt-get update -q && apt-get install -qy git ${pythoncmd}-pip python-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev libgnutls28-dev nodejs python-pyasn1-modules
+RUN if [ "$pythoncmd" = "python3" ]; then pip3 install -U setuptools six requests; else pip install -U setuptools six requests; fi
 
 ARG sdk
 ARG runner
@@ -32,10 +33,10 @@ ADD cwl/salad_dist/$salad /tmp/
 ADD cwl/cwltool_dist/$cwltool /tmp/
 ADD cwl/dist/$runner /tmp/
 
-RUN cd /tmp/arvados-python-client-* && python setup.py install
-RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && python setup.py install ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && python setup.py install ; fi
-RUN cd /tmp/arvados-cwl-runner-* && python setup.py install
+RUN cd /tmp/arvados-python-client-* && $pythoncmd setup.py install
+RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pythoncmd setup.py install ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pythoncmd setup.py install ; fi
+RUN cd /tmp/arvados-cwl-runner-* && $pythoncmd setup.py install
 
 # Install dependencies and set up system.
 RUN /usr/sbin/adduser --disabled-password \

diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 55fc6b626d5de9798b989d2b252d158ca89baab6..2d306d516a29652d7a0942e47acc7ef3572414ff 100644 (file)
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -133,6 +133,7 @@ def docker_images():
     next(list_output)  # Ignore the header line
     for line in list_output:
         words = line.split()
+        words = [word.decode() for word in words]
         size_index = len(words) - 2
         repo, tag, imageid = words[:3]
         ctime = ' '.join(words[3:size_index])