13306: Changes to arvados-cwl-runner code after running futurize --stage2

Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

diff --git a/.gitignore b/.gitignore
index 517166c3fbd2043bff880db2b11b97238e4eb5f2..db3020ae82d13231817872355b04dd09849f87a0 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,5 @@ services/api/config/arvados-clients.yml
 .DS_Store
 .vscode
 .Rproj.user
-_version.py
\ No newline at end of file
+_version.py
+*.bak

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7e149528308fc9c6e38e0021af858da5450b58f8..eb2a0df592364bfd220d8436ce9c3c38de6083aa 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -271,7 +271,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
 
     add_arv_hints()
 
-    for key, val in cwltool.argparser.get_default_args().items():
+    for key, val in list(cwltool.argparser.get_default_args().items()):
         if not hasattr(arvargs, key):
             setattr(arvargs, key, val)
 

diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 6a91d6ff3c5076a28e06a685ed8c73bf45a84218..d2f04b5dd0f7b1df07f06f6d20eeeb5341945722 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -1,3 +1,6 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -5,7 +8,7 @@
 import logging
 import json
 import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import time
 import datetime
 import ciso8601
@@ -136,7 +139,7 @@ class ArvadosContainer(JobBase):
                 generatemapper = NoFollowPathMapper(self.generatefiles["listing"], "", "",
                                                     separateDirs=False)
 
-                sorteditems = sorted(generatemapper.items(), None, key=lambda n: n[1].target)
+                sorteditems = sorted(list(generatemapper.items()), None, key=lambda n: n[1].target)
 
                 logger.debug("generatemapper is %s", sorteditems)
 

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 87d6d1049a08d349d8d45b41b47a5fccb99f237c..bd8ab1137e9b4eda0e006162812bf8ea12d7bc7d 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -1,3 +1,5 @@
+from past.builtins import basestring
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -71,7 +73,7 @@ class ArvadosJob(JobBase):
                                                separateDirs=False)
 
                 with Perf(metrics, "createfiles %s" % self.name):
-                    for f, p in generatemapper.items():
+                    for f, p in list(generatemapper.items()):
                         if p.type == "CreateFile":
                             with vwd.open(p.target, "w") as n:
                                 n.write(p.resolved.encode("utf-8"))
@@ -85,7 +87,7 @@ class ArvadosJob(JobBase):
                                      trash_at=info["trash_at"],
                                      properties=info["properties"])
 
-                for f, p in generatemapper.items():
+                for f, p in list(generatemapper.items()):
                     if p.type == "File":
                         script_parameters["task.vwd"][p.target] = p.resolved
                     if p.type == "CreateFile":
@@ -243,7 +245,7 @@ class ArvadosJob(JobBase):
                                                                    api_client=self.arvrunner.api,
                                                                    keep_client=self.arvrunner.keep_client,
                                                                    num_retries=self.arvrunner.num_retries)
-                        log = logc.open(logc.keys()[0])
+                        log = logc.open(list(logc.keys())[0])
                         dirs = {
                             "tmpdir": "/tmpdir",
                             "outdir": "/outdir",
@@ -343,7 +345,7 @@ class RunnerJob(Runner):
             find_or_create=self.enable_reuse
         ).execute(num_retries=self.arvrunner.num_retries)
 
-        for k,v in job_spec["script_parameters"].items():
+        for k,v in list(job_spec["script_parameters"].items()):
             if v is False or v is None or isinstance(v, dict):
                 job_spec["script_parameters"][k] = {"value": v}
 

diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index c4e9f44abb0b20ecb66a7bdc13c5240beaaeeccb..31e6be12b533cb9d96c65f74189935fad8c3fcbe 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -16,7 +16,7 @@ def validate_cluster_target(arvrunner, runtimeContext):
         runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and
         runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]):
         raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster,
-                                                                                                  ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys())))
+                                                                                                  ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
 def set_cluster_target(tool, arvrunner, builder, runtimeContext):
     cluster_target_req = None
     for field in ("hints", "requirements"):

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index ea167d4044d76fa91953eb401962107afd6b878e..b878c4d9b64b9eb710fc852ed07e046a5b6bacf1 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -1,3 +1,4 @@
+from past.builtins import basestring
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -135,7 +136,7 @@ class ArvadosWorkflowStep(WorkflowStep):
         runtimeContext = runtimeContext.copy()
         runtimeContext.toplevel = True  # Preserve behavior for #13365
 
-        builder = make_builder({shortname(k): v for k,v in joborder.items()}, self.hints, self.requirements, runtimeContext)
+        builder = make_builder({shortname(k): v for k,v in list(joborder.items())}, self.hints, self.requirements, runtimeContext)
         runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
         return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
 

diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 61f9cbbe0dc80a7ce7c4894ccb2697c0b0310652..fc2d0e8767faadf727a5332e91d6ff1b0ebb569e 100644 (file)
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -1,3 +1,4 @@
+from past.builtins import basestring
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -61,7 +62,7 @@ def run():
             if "location" in v:
                 v["location"] = keeppath(v["location"])
 
-        for k,v in job_order_object.items():
+        for k,v in list(job_order_object.items()):
             if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
                 job_order_object[k] = {
                     "class": "File",

diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 6d46e79cb84b33bee3af508c86498201aa37acd1..9c9195a6ab6f68d5d335153a1b7473bcac2a93cb 100644 (file)
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -63,7 +63,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
     containersapi = ("crunch-run.txt" in logcollection)
     mergelogs = {}
 
-    for log in logcollection.keys():
+    for log in list(logcollection.keys()):
         if not containersapi or log in ("crunch-run.txt", "stdout.txt", "stderr.txt"):
             logname = log[:-4]
             logt = deque([], maxlen)
@@ -77,7 +77,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
                         logt.append(l)
 
     if containersapi:
-        keys = mergelogs.keys()
+        keys = list(mergelogs.keys())
         loglines = []
         while True:
             earliest = None
@@ -91,7 +91,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
             loglines.append("%s %s %s" % (ts, earliest, msg))
         loglines = loglines[-maxlen:]
     else:
-        loglines = mergelogs.values()[0]
+        loglines = list(mergelogs.values())[0]
 
     logtxt = "\n  ".join(l.strip() for l in loglines)
     logfunc("%s\n\n  %s", header, logtxt)

diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 31ac4c2478dccc5dcf60775c8f1e28db5ec4632d..dbe4992cbad338ef6c875c6167fab9f102185538 100644 (file)
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -1,3 +1,7 @@
+from __future__ import division
+from builtins import next
+from builtins import object
+from past.utils import old_div
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -401,7 +405,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                             "Option 'dockerOutputDirectory' must be an absolute path.")
             if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
                 raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
-            for v in obj.itervalues():
+            for v in obj.values():
                 self.check_features(v)
         elif isinstance(obj, list):
             for i,v in enumerate(obj):
@@ -424,7 +428,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                                               keep_client=self.keep_client,
                                               num_retries=self.num_retries)
 
-        for k,v in generatemapper.items():
+        for k,v in list(generatemapper.items()):
             if k.startswith("_:"):
                 if v.type == "Directory":
                     continue
@@ -608,7 +612,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
                         visited.add(m.group(1))
                         estimated_size[0] += int(m.group(2))
             visit_class(job_order, ("File", "Directory"), estimate_collection_cache)
-            runtimeContext.collection_cache_size = max(((estimated_size[0]*192) / (1024*1024))+1, 256)
+            runtimeContext.collection_cache_size = max((old_div((estimated_size[0]*192), (1024*1024)))+1, 256)
             self.collection_cache.set_cap(runtimeContext.collection_cache_size*1024*1024)
 
         logger.info("Using collection cache size %s MiB", runtimeContext.collection_cache_size)

diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 0816ee8fc05b74198ae9abad69887905bf8113ee..708b81818cb80e147cc1c0edb6a680e690daaef6 100644 (file)
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -1,3 +1,6 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -5,7 +8,7 @@
 import fnmatch
 import os
 import errno
-import urlparse
+import urllib.parse
 import re
 import logging
 import threading
@@ -48,7 +51,7 @@ class CollectionCache(object):
 
     def cap_cache(self, required):
         # ordered dict iterates from oldest to newest
-        for pdh, v in self.collections.items():
+        for pdh, v in list(self.collections.items()):
             available = self.cap - self.total
             if available >= required or len(self.collections) < self.min_entries:
                 return
@@ -90,7 +93,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
         p = sp[0]
         if p.startswith("keep:") and arvados.util.keep_locator_pattern.match(p[5:]):
             pdh = p[5:]
-            return (self.collection_cache.get(pdh), urlparse.unquote(sp[1]) if len(sp) == 2 else None)
+            return (self.collection_cache.get(pdh), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None)
         else:
             return (None, path)
 
@@ -188,7 +191,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
                 raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash()))
             if not isinstance(dir, arvados.collection.RichCollectionBase):
                 raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash()))
-            return [abspath(l, fn) for l in dir.keys()]
+            return [abspath(l, fn) for l in list(dir.keys())]
         else:
             return super(CollectionFsAccess, self).listdir(fn)
 
@@ -243,11 +246,11 @@ class CollectionFetcher(DefaultFetcher):
         if not url:
             return base_url
 
-        urlsp = urlparse.urlsplit(url)
+        urlsp = urllib.parse.urlsplit(url)
         if urlsp.scheme or not base_url:
             return url
 
-        basesp = urlparse.urlsplit(base_url)
+        basesp = urllib.parse.urlsplit(base_url)
         if basesp.scheme in ("keep", "arvwf"):
             if not basesp.path:
                 raise IOError(errno.EINVAL, "Invalid Keep locator", base_url)
@@ -268,7 +271,7 @@ class CollectionFetcher(DefaultFetcher):
                 baseparts.pop()
 
             path = "/".join([pdh] + baseparts + urlparts)
-            return urlparse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment))
+            return urllib.parse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment))
 
         return super(CollectionFetcher, self).urljoin(base_url, url)
 
@@ -290,7 +293,7 @@ def collectionResolver(api_client, document_loader, uri, num_retries=4):
 
     if pipeline_template_uuid_pattern.match(uri):
         pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
-        return "keep:" + pt["components"].values()[0]["script_parameters"]["cwl:tool"]
+        return "keep:" + list(pt["components"].values())[0]["script_parameters"]["cwl:tool"]
 
     p = uri.split("/")
     if arvados.util.keep_locator_pattern.match(p[0]):

diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py
index 4516de021b8522f25990cf2988b7b4345f13849d..9b3649a61a871afa73e8ba3482fed8dc700b4128 100644 (file)
--- a/sdk/cwl/arvados_cwl/http.py
+++ b/sdk/cwl/arvados_cwl/http.py
@@ -1,3 +1,7 @@
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from past.utils import old_div
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -9,7 +13,7 @@ import datetime
 import re
 import arvados
 import arvados.collection
-import urlparse
+import urllib.parse
 import logging
 import calendar
 
@@ -91,13 +95,13 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
         if fresh_cache(url, properties, now):
             # Do nothing
             cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
-            return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0])
+            return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0])
 
         if not changed(url, properties, now):
             # ETag didn't change, same content, just update headers
             api.collections().update(uuid=item["uuid"], body={"collection":{"properties": properties}}).execute()
             cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
-            return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0])
+            return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0])
 
     properties = {}
     req = requests.get(url, stream=True, allow_redirects=True)
@@ -123,7 +127,7 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
         else:
             name = grp.group(4)
     else:
-        name = urlparse.urlparse(url).path.split("/")[-1]
+        name = urllib.parse.urlparse(url).path.split("/")[-1]
 
     count = 0
     start = time.time()
@@ -134,14 +138,14 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
             f.write(chunk)
             loopnow = time.time()
             if (loopnow - checkpoint) > 20:
-                bps = (float(count)/float(loopnow - start))
+                bps = (old_div(float(count),float(loopnow - start)))
                 if cl is not None:
                     logger.info("%2.1f%% complete, %3.2f MiB/s, %1.0f seconds left",
-                                float(count * 100) / float(cl),
-                                bps/(1024*1024),
-                                (cl-count)/bps)
+                                old_div(float(count * 100), float(cl)),
+                                old_div(bps,(1024*1024)),
+                                old_div((cl-count),bps))
                 else:
-                    logger.info("%d downloaded, %3.2f MiB/s", count, bps/(1024*1024))
+                    logger.info("%d downloaded, %3.2f MiB/s", count, old_div(bps,(1024*1024)))
                 checkpoint = loopnow
 
     c.save_new(name="Downloaded from %s" % url, owner_uuid=project_uuid, ensure_unique_name=True)

diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 0b2a22788e6f98537b0f5a3437a2d540a57d47ee..7353d7239c2578622e7c409bd829d5e9e5fb1e80 100644 (file)
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -1,3 +1,7 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from past.builtins import basestring
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -6,7 +10,7 @@ import re
 import logging
 import uuid
 import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 import arvados_cwl.util
 import arvados.commands.run
@@ -60,7 +64,7 @@ class ArvPathMapper(PathMapper):
             src = src[:src.index("#")]
 
         if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
-            self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
+            self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True)
 
         debug = logger.isEnabledFor(logging.DEBUG)
 
@@ -77,7 +81,7 @@ class ArvPathMapper(PathMapper):
                     if isinstance(st, arvados.commands.run.UploadFile):
                         uploadfiles.add((src, ab, st))
                     elif isinstance(st, arvados.commands.run.ArvFile):
-                        self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True)
+                        self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True)
                     else:
                         raise WorkflowException("Input file path '%s' is invalid" % st)
             elif src.startswith("_:"):
@@ -176,7 +180,7 @@ class ArvPathMapper(PathMapper):
                                          packed=False)
 
         for src, ab, st in uploadfiles:
-            self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
+            self._pathmap[src] = MapperEnt(urllib.parse.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
                                            "Directory" if os.path.isdir(ab) else "File", True)
 
         for srcobj in referenced_files:
@@ -228,7 +232,7 @@ class ArvPathMapper(PathMapper):
                                                               ab, "File", True)
                 if srcobj.get("secondaryFiles"):
                     ab = self.collection_pattern % c.portable_data_hash()
-                    self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
+                    self._pathmap["_:" + str(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
 
             if remap:
                 for loc, sub in remap:
@@ -301,7 +305,7 @@ class VwdPathMapper(StagingPathMapper):
         # with any secondary files.
         self.visitlisting(referenced_files, self.stagedir, basedir)
 
-        for path, (ab, tgt, type, staged) in self._pathmap.items():
+        for path, (ab, tgt, type, staged) in list(self._pathmap.items()):
             if type in ("File", "Directory") and ab.startswith("keep:"):
                 self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type, staged)
 

diff --git a/sdk/cwl/arvados_cwl/perf.py b/sdk/cwl/arvados_cwl/perf.py
index 39f475fe8d30e6fda700e2f3c965577745b0638f..e736d9ef00d53800562d641471d9bcd38a572643 100644 (file)
--- a/sdk/cwl/arvados_cwl/perf.py
+++ b/sdk/cwl/arvados_cwl/perf.py
@@ -1,3 +1,4 @@
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4b3275fa36b4ee497ec83c3de2b3e8a0c938fb62..9f6390e3073e63d6482d404ec52d7330c6ddd40c 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -1,16 +1,18 @@
+from future import standard_library
+standard_library.install_aliases()
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
 import os
-import urlparse
+import urllib.parse
 from functools import partial
 import logging
 import json
 import subprocess32 as subprocess
 from collections import namedtuple
 
-from StringIO import StringIO
+from io import StringIO
 
 from schema_salad.sourceline import SourceLine, cmap
 
@@ -61,7 +63,7 @@ def find_defaults(d, op):
         if "default" in d:
             op(d)
         else:
-            for i in d.itervalues():
+            for i in d.values():
                 find_defaults(i, op)
 
 def setSecondary(t, fileobj, discovered):
@@ -98,7 +100,7 @@ def upload_dependencies(arvrunner, name, document_loader,
     loaded = set()
     def loadref(b, u):
         joined = document_loader.fetcher.urljoin(b, u)
-        defrg, _ = urlparse.urldefrag(joined)
+        defrg, _ = urllib.parse.urldefrag(joined)
         if defrg not in loaded:
             loaded.add(defrg)
             # Use fetch_text to get raw file (before preprocessing).
@@ -232,7 +234,7 @@ def packed_workflow(arvrunner, tool, merged_map):
     packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
                   tool.tool["id"], tool.metadata, rewrite_out=rewrites)
 
-    rewrite_to_orig = {v: k for k,v in rewrites.items()}
+    rewrite_to_orig = {v: k for k,v in list(rewrites.items())}
 
     def visit(v, cur_id):
         if isinstance(v, dict):
@@ -314,7 +316,7 @@ def upload_workflow_deps(arvrunner, tool):
                                      discovered_secondaryfiles=discovered_secondaryfiles)
             document_loader.idx[deptool["id"]] = deptool
             toolmap = {}
-            for k,v in pm.items():
+            for k,v in list(pm.items()):
                 toolmap[k] = v.resolved
             merged_map[deptool["id"]] = FileUpdates(toolmap, discovered_secondaryfiles)
 

diff --git a/sdk/cwl/arvados_cwl/task_queue.py b/sdk/cwl/arvados_cwl/task_queue.py
index 1c233fac0ad98f4b0421a4e0856b00fd19d1422f..f7d7c071ce11cb1355debd4442a1586175aa7dc5 100644 (file)
--- a/sdk/cwl/arvados_cwl/task_queue.py
+++ b/sdk/cwl/arvados_cwl/task_queue.py
@@ -1,8 +1,12 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import Queue
+import queue
 import threading
 import logging
 
@@ -11,13 +15,13 @@ logger = logging.getLogger('arvados.cwl-runner')
 class TaskQueue(object):
     def __init__(self, lock, thread_count):
         self.thread_count = thread_count
-        self.task_queue = Queue.Queue(maxsize=self.thread_count)
+        self.task_queue = queue.Queue(maxsize=self.thread_count)
         self.task_queue_threads = []
         self.lock = lock
         self.in_flight = 0
         self.error = None
 
-        for r in xrange(0, self.thread_count):
+        for r in range(0, self.thread_count):
             t = threading.Thread(target=self.task_queue_func)
             self.task_queue_threads.append(t)
             t.start()
@@ -51,7 +55,7 @@ class TaskQueue(object):
                     return
                 self.task_queue.put(task, block=True, timeout=3)
                 return
-            except Queue.Full:
+            except queue.Full:
                 pass
             finally:
                 unlock.acquire()
@@ -62,7 +66,7 @@ class TaskQueue(object):
             # Drain queue
             while not self.task_queue.empty():
                 self.task_queue.get(True, .1)
-        except Queue.Empty:
+        except queue.Empty:
             pass
 
     def join(self):

diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d13dd5ec538e678268d7b79836d745ba89d46047..bd23cdd0af46f9a22ce10496da01de5c540d88c3 100644 (file)
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import next
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0

diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
index 4dc8448476123934dae7193fe680141671a2b7ec..f32ab8000a0a1c7ddb1d48fa214527dd7152ecdb 100644 (file)
--- a/sdk/cwl/gittaggers.py
+++ b/sdk/cwl/gittaggers.py
@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import next
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0

diff --git a/sdk/cwl/tests/matcher.py b/sdk/cwl/tests/matcher.py
index 50c0c60b1d0a337527ddd2342ae1ed46e22b1708..674e8fc60de10afe1cf9c36f878ce3c52a0e03f5 100644 (file)
--- a/sdk/cwl/tests/matcher.py
+++ b/sdk/cwl/tests/matcher.py
@@ -1,3 +1,4 @@
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0

diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index a34d550f62ad2b24a1ff156d7b740377c811cab5..9ad7300b3a386809f45f4c6e20c7307f767c5bec 100644 (file)
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0

diff --git a/sdk/cwl/tests/test_http.py b/sdk/cwl/tests/test_http.py
index 0c66c39c0b4607e492725a280d8dfca3aa52ec19..34f2bbb9d71ba455ee2b40caddbe645638a4860d 100644 (file)
--- a/sdk/cwl/tests/test_http.py
+++ b/sdk/cwl/tests/test_http.py
@@ -1,9 +1,11 @@
+from future import standard_library
+standard_library.install_aliases()
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
 import copy
-import cStringIO
+import io
 import functools
 import hashlib
 import json

diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 2aaac0ae50699f5c012f36ba2f28eee1ccd281c4..387d12f388dc4fce6af87eb4de7a2e7dd267ce07 100644 (file)
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -1,3 +1,6 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -9,7 +12,7 @@ import mock
 import os
 import unittest
 import copy
-import StringIO
+import io
 
 import arvados
 import arvados_cwl
@@ -213,7 +216,7 @@ class TestJob(unittest.TestCase):
         runner.num_retries = 0
         runner.ignore_docker_for_reuse = False
 
-        reader().open.return_value = StringIO.StringIO(
+        reader().open.return_value = io.StringIO(
             """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
@@ -286,7 +289,7 @@ class TestJob(unittest.TestCase):
         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
         runner.num_retries = 0
 
-        reader().open.return_value = StringIO.StringIO(
+        reader().open.return_value = io.StringIO(
             """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep

diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index baeb4145ee6dbc5ba4db326f88acd54ce04352f4..73f9fc90cbe0bcc19888ee2f0227c0e5da565346 100644 (file)
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -1,3 +1,5 @@
+from future import standard_library
+standard_library.install_aliases()
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -7,7 +9,7 @@ import json
 import logging
 import mock
 import os
-import StringIO
+import io
 import unittest
 
 import arvados
@@ -35,7 +37,7 @@ class TestMakeOutput(unittest.TestCase):
         final_uuid = final.manifest_locator()
         num_retries = runner.num_retries
 
-        cwlout = StringIO.StringIO()
+        cwlout = io.StringIO()
         openmock = mock.MagicMock()
         final.open.return_value = openmock
         openmock.__enter__.return_value = cwlout

diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 90dab01471ef61ab380955e6301a73306648edef..5d2d0d01114b06659f548619121660e0c4b542ea 100644 (file)
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1,9 +1,12 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
 import copy
-import cStringIO
+import io
 import functools
 import hashlib
 import json
@@ -33,7 +36,7 @@ def stubs(func):
     @mock.patch("arvados.keep.KeepClient")
     @mock.patch("arvados.events.subscribe")
     def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
-        class Stubs:
+        class Stubs(object):
             pass
         stubs = Stubs()
         stubs.events = events
@@ -89,7 +92,7 @@ def stubs(func):
             return CollectionExecute(created_collections[uuid])
 
         def collection_getstub(created_collections, uuid):
-            for v in created_collections.itervalues():
+            for v in created_collections.values():
                 if uuid in (v["uuid"], v["portable_data_hash"]):
                     return CollectionExecute(v)
 
@@ -318,7 +321,7 @@ class TestSubmit(unittest.TestCase):
                 return '999999999999999999999999999999d4+99'
         arvdock.side_effect = get_image
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -361,7 +364,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_no_reuse(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -389,7 +392,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_on_error(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -408,7 +411,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_runner_ram(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -427,7 +430,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_invalid_runner_ram(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -439,7 +442,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_output_name(self, stubs, tm):
         output_name = "test_output_name"
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--output-name", output_name,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -458,7 +461,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_pipeline_name(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--name=hello job 123",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -478,7 +481,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_output_tags(self, stubs, tm):
         output_tags = "tag0,tag1,tag2"
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -512,7 +515,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -544,7 +547,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_no_reuse(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
@@ -572,7 +575,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_reuse_disabled_by_workflow(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -608,7 +611,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_on_error(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
@@ -635,7 +638,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_container_output_name(self, stubs):
         output_name = "test_output_name"
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
@@ -661,7 +664,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_storage_classes(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
@@ -728,7 +731,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_output_ttl(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
@@ -754,7 +757,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_trash_intermediate(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
@@ -782,7 +785,7 @@ class TestSubmit(unittest.TestCase):
     def test_submit_container_output_tags(self, stubs):
         output_tags = "tag0,tag1,tag2"
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
@@ -807,7 +810,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_runner_ram(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
@@ -829,7 +832,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_file_keepref(self, stubs, tm, collectionReader):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -842,7 +845,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_keepref(self, stubs, tm, reader):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         with open("tests/wf/expect_arvworkflow.cwl") as f:
             reader().open().__enter__().read.return_value = f.read()
@@ -904,7 +907,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_jobs_keepref(self, stubs, tm, reader):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         with open("tests/wf/expect_arvworkflow.cwl") as f:
             reader().open().__enter__().read.return_value = f.read()
@@ -927,7 +930,7 @@ class TestSubmit(unittest.TestCase):
     @mock.patch("time.sleep")
     @stubs
     def test_submit_arvworkflow(self, stubs, tm):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         with open("tests/wf/expect_arvworkflow.cwl") as f:
             stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
@@ -1026,7 +1029,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_name(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
@@ -1047,14 +1050,14 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_missing_input(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
             capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
         self.assertEqual(exited, 0)
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug",
              "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
@@ -1065,7 +1068,7 @@ class TestSubmit(unittest.TestCase):
     @stubs
     def test_submit_container_project(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
@@ -1093,7 +1096,7 @@ class TestSubmit(unittest.TestCase):
     @stubs
     def test_submit_container_eval_timeout(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
@@ -1119,7 +1122,7 @@ class TestSubmit(unittest.TestCase):
     @stubs
     def test_submit_container_collection_cache(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
@@ -1147,7 +1150,7 @@ class TestSubmit(unittest.TestCase):
     @stubs
     def test_submit_container_thread_count(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
@@ -1173,7 +1176,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_job_runner_image(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
@@ -1193,7 +1196,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_runner_image(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
@@ -1213,7 +1216,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_priority(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
@@ -1234,7 +1237,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_wf_runner_resources(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -1315,7 +1318,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_secrets(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug",
@@ -1492,7 +1495,7 @@ class TestSubmit(unittest.TestCase):
             "state": "Queued"
         }
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         try:
             exited = arvados_cwl.main(
                 ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
@@ -1509,7 +1512,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_container_cluster_id(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
         try:
             exited = arvados_cwl.main(
@@ -1530,7 +1533,7 @@ class TestSubmit(unittest.TestCase):
 
     @stubs
     def test_submit_validate_cluster_id(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
         stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
         exited = arvados_cwl.main(
             ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
@@ -1565,7 +1568,7 @@ class TestCreateTemplate(unittest.TestCase):
     def test_create(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
@@ -1598,7 +1601,7 @@ class TestCreateTemplate(unittest.TestCase):
     def test_create_name(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
@@ -1632,7 +1635,7 @@ class TestCreateTemplate(unittest.TestCase):
     def test_update_name(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_template_uuid,
@@ -1673,7 +1676,7 @@ class TestCreateWorkflow(unittest.TestCase):
     def test_create(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
@@ -1705,7 +1708,7 @@ class TestCreateWorkflow(unittest.TestCase):
     def test_create_name(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
@@ -1735,7 +1738,7 @@ class TestCreateWorkflow(unittest.TestCase):
 
     @stubs
     def test_incompatible_api(self, stubs):
-        capture_stderr = cStringIO.StringIO()
+        capture_stderr = io.StringIO()
         logging.getLogger('arvados.cwl-runner').addHandler(
             logging.StreamHandler(capture_stderr))
 
@@ -1752,7 +1755,7 @@ class TestCreateWorkflow(unittest.TestCase):
 
     @stubs
     def test_update(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
@@ -1777,7 +1780,7 @@ class TestCreateWorkflow(unittest.TestCase):
 
     @stubs
     def test_update_name(self, stubs):
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--update-workflow", self.existing_workflow_uuid,
@@ -1804,7 +1807,7 @@ class TestCreateWorkflow(unittest.TestCase):
     def test_create_collection_per_tool(self, stubs):
         project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
 
-        capture_stdout = cStringIO.StringIO()
+        capture_stdout = io.StringIO()
 
         exited = arvados_cwl.main(
             ["--create-workflow", "--debug",
@@ -1884,7 +1887,7 @@ class TestTemplateInputs(unittest.TestCase):
         exited = arvados_cwl.main(
             ["--create-template",
              "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
-            cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
+            io.StringIO(), sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
 
         stubs.api.pipeline_templates().create.assert_called_with(
@@ -1895,7 +1898,7 @@ class TestTemplateInputs(unittest.TestCase):
         exited = arvados_cwl.main(
             ["--create-template",
              "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
-            cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
+            io.StringIO(), sys.stderr, api_client=stubs.api)
         self.assertEqual(exited, 0)
 
         expect_template = copy.deepcopy(self.expect_template)

diff --git a/sdk/cwl/tests/wf/check_mem.py b/sdk/cwl/tests/wf/check_mem.py
index bad16ed06bcfd0c4661309834f3e26ec80cb3084..88eb2fd513c19a8359571d6aab3ac6777be95ff6 100644 (file)
--- a/sdk/cwl/tests/wf/check_mem.py
+++ b/sdk/cwl/tests/wf/check_mem.py
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
 import arvados
 import sys
 import os
@@ -10,7 +12,7 @@ import os
 if "JOB_UUID" in os.environ:
     requested = arvados.api().jobs().get(uuid=os.environ["JOB_UUID"]).execute()["runtime_constraints"]["min_ram_mb_per_node"]
 else:
-    requested = arvados.api().containers().current().execute()["runtime_constraints"]["ram"]/(1024*1024)
+    requested = old_div(arvados.api().containers().current().execute()["runtime_constraints"]["ram"],(1024*1024))
 
 print("Requested %d expected %d" % (requested, int(sys.argv[1])))