From d6cccb3ea4e5f076a436d9935e3835d4b620b859 Mon Sep 17 00:00:00 2001 From: Eric Biagiotti Date: Thu, 10 Jan 2019 09:06:42 -0500 Subject: [PATCH] 13306: Changes to arvados-cwl-runner code after running futurize --stage2 Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti --- .gitignore | 3 +- sdk/cwl/arvados_cwl/__init__.py | 2 +- sdk/cwl/arvados_cwl/arvcontainer.py | 7 +- sdk/cwl/arvados_cwl/arvjob.py | 10 ++- sdk/cwl/arvados_cwl/arvtool.py | 2 +- sdk/cwl/arvados_cwl/arvworkflow.py | 3 +- sdk/cwl/arvados_cwl/crunch_script.py | 3 +- sdk/cwl/arvados_cwl/done.py | 6 +- sdk/cwl/arvados_cwl/executor.py | 10 ++- sdk/cwl/arvados_cwl/fsaccess.py | 19 +++-- sdk/cwl/arvados_cwl/http.py | 22 +++--- sdk/cwl/arvados_cwl/pathmapper.py | 16 ++-- sdk/cwl/arvados_cwl/perf.py | 1 + sdk/cwl/arvados_cwl/runner.py | 14 ++-- sdk/cwl/arvados_cwl/task_queue.py | 14 ++-- sdk/cwl/arvados_version.py | 2 + sdk/cwl/gittaggers.py | 2 + sdk/cwl/tests/matcher.py | 1 + sdk/cwl/tests/test_container.py | 2 + sdk/cwl/tests/test_http.py | 4 +- sdk/cwl/tests/test_job.py | 9 ++- sdk/cwl/tests/test_make_output.py | 6 +- sdk/cwl/tests/test_submit.py | 105 ++++++++++++++------------- sdk/cwl/tests/wf/check_mem.py | 4 +- 24 files changed, 158 insertions(+), 109 deletions(-) diff --git a/.gitignore b/.gitignore index 517166c3fb..db3020ae82 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ services/api/config/arvados-clients.yml .DS_Store .vscode .Rproj.user -_version.py \ No newline at end of file +_version.py +*.bak diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index 7e14952830..eb2a0df592 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -271,7 +271,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None, add_arv_hints() - for key, val in cwltool.argparser.get_default_args().items(): + for key, val in list(cwltool.argparser.get_default_args().items()): if not hasattr(arvargs, key): setattr(arvargs, key, val) diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py index 6a91d6ff3c..d2f04b5dd0 100644 --- a/sdk/cwl/arvados_cwl/arvcontainer.py +++ b/sdk/cwl/arvados_cwl/arvcontainer.py @@ -1,3 +1,6 @@ +from future import standard_library +standard_library.install_aliases() +from builtins import str # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -5,7 +8,7 @@ import logging import json import os -import urllib +import urllib.request, urllib.parse, urllib.error import time import datetime import ciso8601 @@ -136,7 +139,7 @@ class ArvadosContainer(JobBase): generatemapper = NoFollowPathMapper(self.generatefiles["listing"], "", "", separateDirs=False) - sorteditems = sorted(generatemapper.items(), None, key=lambda n: n[1].target) + sorteditems = sorted(list(generatemapper.items()), None, key=lambda n: n[1].target) logger.debug("generatemapper is %s", sorteditems) diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py index 87d6d1049a..bd8ab1137e 100644 --- a/sdk/cwl/arvados_cwl/arvjob.py +++ b/sdk/cwl/arvados_cwl/arvjob.py @@ -1,3 +1,5 @@ +from past.builtins import basestring +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -71,7 +73,7 @@ class ArvadosJob(JobBase): separateDirs=False) with Perf(metrics, "createfiles %s" % self.name): - for f, p in generatemapper.items(): + for f, p in list(generatemapper.items()): if p.type == "CreateFile": with vwd.open(p.target, "w") as n: n.write(p.resolved.encode("utf-8")) @@ -85,7 +87,7 @@ class ArvadosJob(JobBase): trash_at=info["trash_at"], properties=info["properties"]) - for f, p in generatemapper.items(): + for f, p in list(generatemapper.items()): if p.type == "File": script_parameters["task.vwd"][p.target] = p.resolved if p.type == "CreateFile": @@ -243,7 +245,7 @@ class ArvadosJob(JobBase): api_client=self.arvrunner.api, keep_client=self.arvrunner.keep_client, num_retries=self.arvrunner.num_retries) - log = logc.open(logc.keys()[0]) + log = logc.open(list(logc.keys())[0]) dirs = { "tmpdir": "/tmpdir", "outdir": "/outdir", @@ -343,7 +345,7 @@ class RunnerJob(Runner): find_or_create=self.enable_reuse ).execute(num_retries=self.arvrunner.num_retries) - for k,v in job_spec["script_parameters"].items(): + for k,v in list(job_spec["script_parameters"].items()): if v is False or v is None or isinstance(v, dict): job_spec["script_parameters"][k] = {"value": v} diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py index c4e9f44abb..31e6be12b5 100644 --- a/sdk/cwl/arvados_cwl/arvtool.py +++ b/sdk/cwl/arvados_cwl/arvtool.py @@ -16,7 +16,7 @@ def validate_cluster_target(arvrunner, runtimeContext): runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]): raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster, - ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys()))) + ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys())))) def set_cluster_target(tool, arvrunner, builder, runtimeContext): cluster_target_req = None for field in ("hints", "requirements"): diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index ea167d4044..b878c4d9b6 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -1,3 +1,4 @@ +from past.builtins import basestring # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -135,7 +136,7 @@ class ArvadosWorkflowStep(WorkflowStep): runtimeContext = runtimeContext.copy() runtimeContext.toplevel = True # Preserve behavior for #13365 - builder = make_builder({shortname(k): v for k,v in joborder.items()}, self.hints, self.requirements, runtimeContext) + builder = make_builder({shortname(k): v for k,v in list(joborder.items())}, self.hints, self.requirements, runtimeContext) runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext) return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext) diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py index 61f9cbbe0d..fc2d0e8767 100644 --- a/sdk/cwl/arvados_cwl/crunch_script.py +++ b/sdk/cwl/arvados_cwl/crunch_script.py @@ -1,3 +1,4 @@ +from past.builtins import basestring # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -61,7 +62,7 @@ def run(): if "location" in v: v["location"] = keeppath(v["location"]) - for k,v in job_order_object.items(): + for k,v in list(job_order_object.items()): if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v): job_order_object[k] = { "class": "File", diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py index 6d46e79cb8..9c9195a6ab 100644 --- a/sdk/cwl/arvados_cwl/done.py +++ b/sdk/cwl/arvados_cwl/done.py @@ -63,7 +63,7 @@ def logtail(logcollection, logfunc, header, maxlen=25): containersapi = ("crunch-run.txt" in logcollection) mergelogs = {} - for log in logcollection.keys(): + for log in list(logcollection.keys()): if not containersapi or log in ("crunch-run.txt", "stdout.txt", "stderr.txt"): logname = log[:-4] logt = deque([], maxlen) @@ -77,7 +77,7 @@ def logtail(logcollection, logfunc, header, maxlen=25): logt.append(l) if containersapi: - keys = mergelogs.keys() + keys = list(mergelogs.keys()) loglines = [] while True: earliest = None @@ -91,7 +91,7 @@ def logtail(logcollection, logfunc, header, maxlen=25): loglines.append("%s %s %s" % (ts, earliest, msg)) loglines = loglines[-maxlen:] else: - loglines = mergelogs.values()[0] + loglines = list(mergelogs.values())[0] logtxt = "\n ".join(l.strip() for l in loglines) logfunc("%s\n\n %s", header, logtxt) diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py index 31ac4c2478..dbe4992cba 100644 --- a/sdk/cwl/arvados_cwl/executor.py +++ b/sdk/cwl/arvados_cwl/executor.py @@ -1,3 +1,7 @@ +from __future__ import division +from builtins import next +from builtins import object +from past.utils import old_div # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -401,7 +405,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods "Option 'dockerOutputDirectory' must be an absolute path.") if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers": raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs") - for v in obj.itervalues(): + for v in obj.values(): self.check_features(v) elif isinstance(obj, list): for i,v in enumerate(obj): @@ -424,7 +428,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods keep_client=self.keep_client, num_retries=self.num_retries) - for k,v in generatemapper.items(): + for k,v in list(generatemapper.items()): if k.startswith("_:"): if v.type == "Directory": continue @@ -608,7 +612,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods visited.add(m.group(1)) estimated_size[0] += int(m.group(2)) visit_class(job_order, ("File", "Directory"), estimate_collection_cache) - runtimeContext.collection_cache_size = max(((estimated_size[0]*192) / (1024*1024))+1, 256) + runtimeContext.collection_cache_size = max((old_div((estimated_size[0]*192), (1024*1024)))+1, 256) self.collection_cache.set_cap(runtimeContext.collection_cache_size*1024*1024) logger.info("Using collection cache size %s MiB", runtimeContext.collection_cache_size) diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py index 0816ee8fc0..708b81818c 100644 --- a/sdk/cwl/arvados_cwl/fsaccess.py +++ b/sdk/cwl/arvados_cwl/fsaccess.py @@ -1,3 +1,6 @@ +from future import standard_library +standard_library.install_aliases() +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -5,7 +8,7 @@ import fnmatch import os import errno -import urlparse +import urllib.parse import re import logging import threading @@ -48,7 +51,7 @@ class CollectionCache(object): def cap_cache(self, required): # ordered dict iterates from oldest to newest - for pdh, v in self.collections.items(): + for pdh, v in list(self.collections.items()): available = self.cap - self.total if available >= required or len(self.collections) < self.min_entries: return @@ -90,7 +93,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess): p = sp[0] if p.startswith("keep:") and arvados.util.keep_locator_pattern.match(p[5:]): pdh = p[5:] - return (self.collection_cache.get(pdh), urlparse.unquote(sp[1]) if len(sp) == 2 else None) + return (self.collection_cache.get(pdh), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None) else: return (None, path) @@ -188,7 +191,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess): raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash())) if not isinstance(dir, arvados.collection.RichCollectionBase): raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash())) - return [abspath(l, fn) for l in dir.keys()] + return [abspath(l, fn) for l in list(dir.keys())] else: return super(CollectionFsAccess, self).listdir(fn) @@ -243,11 +246,11 @@ class CollectionFetcher(DefaultFetcher): if not url: return base_url - urlsp = urlparse.urlsplit(url) + urlsp = urllib.parse.urlsplit(url) if urlsp.scheme or not base_url: return url - basesp = urlparse.urlsplit(base_url) + basesp = urllib.parse.urlsplit(base_url) if basesp.scheme in ("keep", "arvwf"): if not basesp.path: raise IOError(errno.EINVAL, "Invalid Keep locator", base_url) @@ -268,7 +271,7 @@ class CollectionFetcher(DefaultFetcher): baseparts.pop() path = "/".join([pdh] + baseparts + urlparts) - return urlparse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment)) + return urllib.parse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment)) return super(CollectionFetcher, self).urljoin(base_url, url) @@ -290,7 +293,7 @@ def collectionResolver(api_client, document_loader, uri, num_retries=4): if pipeline_template_uuid_pattern.match(uri): pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries) - return "keep:" + pt["components"].values()[0]["script_parameters"]["cwl:tool"] + return "keep:" + list(pt["components"].values())[0]["script_parameters"]["cwl:tool"] p = uri.split("/") if arvados.util.keep_locator_pattern.match(p[0]): diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py index 4516de021b..9b3649a61a 100644 --- a/sdk/cwl/arvados_cwl/http.py +++ b/sdk/cwl/arvados_cwl/http.py @@ -1,3 +1,7 @@ +from __future__ import division +from future import standard_library +standard_library.install_aliases() +from past.utils import old_div # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -9,7 +13,7 @@ import datetime import re import arvados import arvados.collection -import urlparse +import urllib.parse import logging import calendar @@ -91,13 +95,13 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow): if fresh_cache(url, properties, now): # Do nothing cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api) - return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0]) + return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0]) if not changed(url, properties, now): # ETag didn't change, same content, just update headers api.collections().update(uuid=item["uuid"], body={"collection":{"properties": properties}}).execute() cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api) - return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0]) + return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0]) properties = {} req = requests.get(url, stream=True, allow_redirects=True) @@ -123,7 +127,7 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow): else: name = grp.group(4) else: - name = urlparse.urlparse(url).path.split("/")[-1] + name = urllib.parse.urlparse(url).path.split("/")[-1] count = 0 start = time.time() @@ -134,14 +138,14 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow): f.write(chunk) loopnow = time.time() if (loopnow - checkpoint) > 20: - bps = (float(count)/float(loopnow - start)) + bps = (old_div(float(count),float(loopnow - start))) if cl is not None: logger.info("%2.1f%% complete, %3.2f MiB/s, %1.0f seconds left", - float(count * 100) / float(cl), - bps/(1024*1024), - (cl-count)/bps) + old_div(float(count * 100), float(cl)), + old_div(bps,(1024*1024)), + old_div((cl-count),bps)) else: - logger.info("%d downloaded, %3.2f MiB/s", count, bps/(1024*1024)) + logger.info("%d downloaded, %3.2f MiB/s", count, old_div(bps,(1024*1024))) checkpoint = loopnow c.save_new(name="Downloaded from %s" % url, owner_uuid=project_uuid, ensure_unique_name=True) diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py index 0b2a22788e..7353d7239c 100644 --- a/sdk/cwl/arvados_cwl/pathmapper.py +++ b/sdk/cwl/arvados_cwl/pathmapper.py @@ -1,3 +1,7 @@ +from future import standard_library +standard_library.install_aliases() +from builtins import str +from past.builtins import basestring # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -6,7 +10,7 @@ import re import logging import uuid import os -import urllib +import urllib.request, urllib.parse, urllib.error import arvados_cwl.util import arvados.commands.run @@ -60,7 +64,7 @@ class ArvPathMapper(PathMapper): src = src[:src.index("#")] if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src): - self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True) + self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True) debug = logger.isEnabledFor(logging.DEBUG) @@ -77,7 +81,7 @@ class ArvPathMapper(PathMapper): if isinstance(st, arvados.commands.run.UploadFile): uploadfiles.add((src, ab, st)) elif isinstance(st, arvados.commands.run.ArvFile): - self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True) + self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True) else: raise WorkflowException("Input file path '%s' is invalid" % st) elif src.startswith("_:"): @@ -176,7 +180,7 @@ class ArvPathMapper(PathMapper): packed=False) for src, ab, st in uploadfiles: - self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:], + self._pathmap[src] = MapperEnt(urllib.parse.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:], "Directory" if os.path.isdir(ab) else "File", True) for srcobj in referenced_files: @@ -228,7 +232,7 @@ class ArvPathMapper(PathMapper): ab, "File", True) if srcobj.get("secondaryFiles"): ab = self.collection_pattern % c.portable_data_hash() - self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True) + self._pathmap["_:" + str(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True) if remap: for loc, sub in remap: @@ -301,7 +305,7 @@ class VwdPathMapper(StagingPathMapper): # with any secondary files. self.visitlisting(referenced_files, self.stagedir, basedir) - for path, (ab, tgt, type, staged) in self._pathmap.items(): + for path, (ab, tgt, type, staged) in list(self._pathmap.items()): if type in ("File", "Directory") and ab.startswith("keep:"): self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type, staged) diff --git a/sdk/cwl/arvados_cwl/perf.py b/sdk/cwl/arvados_cwl/perf.py index 39f475fe8d..e736d9ef00 100644 --- a/sdk/cwl/arvados_cwl/perf.py +++ b/sdk/cwl/arvados_cwl/perf.py @@ -1,3 +1,4 @@ +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py index 4b3275fa36..9f6390e307 100644 --- a/sdk/cwl/arvados_cwl/runner.py +++ b/sdk/cwl/arvados_cwl/runner.py @@ -1,16 +1,18 @@ +from future import standard_library +standard_library.install_aliases() # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 import os -import urlparse +import urllib.parse from functools import partial import logging import json import subprocess32 as subprocess from collections import namedtuple -from StringIO import StringIO +from io import StringIO from schema_salad.sourceline import SourceLine, cmap @@ -61,7 +63,7 @@ def find_defaults(d, op): if "default" in d: op(d) else: - for i in d.itervalues(): + for i in d.values(): find_defaults(i, op) def setSecondary(t, fileobj, discovered): @@ -98,7 +100,7 @@ def upload_dependencies(arvrunner, name, document_loader, loaded = set() def loadref(b, u): joined = document_loader.fetcher.urljoin(b, u) - defrg, _ = urlparse.urldefrag(joined) + defrg, _ = urllib.parse.urldefrag(joined) if defrg not in loaded: loaded.add(defrg) # Use fetch_text to get raw file (before preprocessing). @@ -232,7 +234,7 @@ def packed_workflow(arvrunner, tool, merged_map): packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]), tool.tool["id"], tool.metadata, rewrite_out=rewrites) - rewrite_to_orig = {v: k for k,v in rewrites.items()} + rewrite_to_orig = {v: k for k,v in list(rewrites.items())} def visit(v, cur_id): if isinstance(v, dict): @@ -314,7 +316,7 @@ def upload_workflow_deps(arvrunner, tool): discovered_secondaryfiles=discovered_secondaryfiles) document_loader.idx[deptool["id"]] = deptool toolmap = {} - for k,v in pm.items(): + for k,v in list(pm.items()): toolmap[k] = v.resolved merged_map[deptool["id"]] = FileUpdates(toolmap, discovered_secondaryfiles) diff --git a/sdk/cwl/arvados_cwl/task_queue.py b/sdk/cwl/arvados_cwl/task_queue.py index 1c233fac0a..f7d7c071ce 100644 --- a/sdk/cwl/arvados_cwl/task_queue.py +++ b/sdk/cwl/arvados_cwl/task_queue.py @@ -1,8 +1,12 @@ +from future import standard_library +standard_library.install_aliases() +from builtins import range +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 -import Queue +import queue import threading import logging @@ -11,13 +15,13 @@ logger = logging.getLogger('arvados.cwl-runner') class TaskQueue(object): def __init__(self, lock, thread_count): self.thread_count = thread_count - self.task_queue = Queue.Queue(maxsize=self.thread_count) + self.task_queue = queue.Queue(maxsize=self.thread_count) self.task_queue_threads = [] self.lock = lock self.in_flight = 0 self.error = None - for r in xrange(0, self.thread_count): + for r in range(0, self.thread_count): t = threading.Thread(target=self.task_queue_func) self.task_queue_threads.append(t) t.start() @@ -51,7 +55,7 @@ class TaskQueue(object): return self.task_queue.put(task, block=True, timeout=3) return - except Queue.Full: + except queue.Full: pass finally: unlock.acquire() @@ -62,7 +66,7 @@ class TaskQueue(object): # Drain queue while not self.task_queue.empty(): self.task_queue.get(True, .1) - except Queue.Empty: + except queue.Empty: pass def join(self): diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py index d13dd5ec53..bd23cdd0af 100644 --- a/sdk/cwl/arvados_version.py +++ b/sdk/cwl/arvados_version.py @@ -1,3 +1,5 @@ +from builtins import str +from builtins import next # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py index 4dc8448476..f32ab8000a 100644 --- a/sdk/cwl/gittaggers.py +++ b/sdk/cwl/gittaggers.py @@ -1,3 +1,5 @@ +from builtins import str +from builtins import next # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/tests/matcher.py b/sdk/cwl/tests/matcher.py index 50c0c60b1d..674e8fc60d 100644 --- a/sdk/cwl/tests/matcher.py +++ b/sdk/cwl/tests/matcher.py @@ -1,3 +1,4 @@ +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index a34d550f62..9ad7300b3a 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -1,3 +1,5 @@ +from builtins import str +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/sdk/cwl/tests/test_http.py b/sdk/cwl/tests/test_http.py index 0c66c39c0b..34f2bbb9d7 100644 --- a/sdk/cwl/tests/test_http.py +++ b/sdk/cwl/tests/test_http.py @@ -1,9 +1,11 @@ +from future import standard_library +standard_library.install_aliases() # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 import copy -import cStringIO +import io import functools import hashlib import json diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py index 2aaac0ae50..387d12f388 100644 --- a/sdk/cwl/tests/test_job.py +++ b/sdk/cwl/tests/test_job.py @@ -1,3 +1,6 @@ +from future import standard_library +standard_library.install_aliases() +from builtins import str # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -9,7 +12,7 @@ import mock import os import unittest import copy -import StringIO +import io import arvados import arvados_cwl @@ -213,7 +216,7 @@ class TestJob(unittest.TestCase): runner.num_retries = 0 runner.ignore_docker_for_reuse = False - reader().open.return_value = StringIO.StringIO( + reader().open.return_value = io.StringIO( """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep @@ -286,7 +289,7 @@ class TestJob(unittest.TestCase): runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.num_retries = 0 - reader().open.return_value = StringIO.StringIO( + reader().open.return_value = io.StringIO( """2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir 2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py index baeb4145ee..73f9fc90cb 100644 --- a/sdk/cwl/tests/test_make_output.py +++ b/sdk/cwl/tests/test_make_output.py @@ -1,3 +1,5 @@ +from future import standard_library +standard_library.install_aliases() # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -7,7 +9,7 @@ import json import logging import mock import os -import StringIO +import io import unittest import arvados @@ -35,7 +37,7 @@ class TestMakeOutput(unittest.TestCase): final_uuid = final.manifest_locator() num_retries = runner.num_retries - cwlout = StringIO.StringIO() + cwlout = io.StringIO() openmock = mock.MagicMock() final.open.return_value = openmock openmock.__enter__.return_value = cwlout diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index 90dab01471..5d2d0d0111 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -1,9 +1,12 @@ +from future import standard_library +standard_library.install_aliases() +from builtins import object # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 import copy -import cStringIO +import io import functools import hashlib import json @@ -33,7 +36,7 @@ def stubs(func): @mock.patch("arvados.keep.KeepClient") @mock.patch("arvados.events.subscribe") def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs): - class Stubs: + class Stubs(object): pass stubs = Stubs() stubs.events = events @@ -89,7 +92,7 @@ def stubs(func): return CollectionExecute(created_collections[uuid]) def collection_getstub(created_collections, uuid): - for v in created_collections.itervalues(): + for v in created_collections.values(): if uuid in (v["uuid"], v["portable_data_hash"]): return CollectionExecute(v) @@ -318,7 +321,7 @@ class TestSubmit(unittest.TestCase): return '999999999999999999999999999999d4+99' arvdock.side_effect = get_image - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=jobs", "--debug", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -361,7 +364,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_no_reuse(self, stubs, tm): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -389,7 +392,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_on_error(self, stubs, tm): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -408,7 +411,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_runner_ram(self, stubs, tm): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -427,7 +430,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_invalid_runner_ram(self, stubs, tm): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -439,7 +442,7 @@ class TestSubmit(unittest.TestCase): def test_submit_output_name(self, stubs, tm): output_name = "test_output_name" - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--debug", "--output-name", output_name, "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -458,7 +461,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_pipeline_name(self, stubs, tm): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--debug", "--name=hello job 123", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -478,7 +481,7 @@ class TestSubmit(unittest.TestCase): def test_submit_output_tags(self, stubs, tm): output_tags = "tag0,tag1,tag2" - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--debug", "--output-tags", output_tags, "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], @@ -512,7 +515,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", @@ -544,7 +547,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_no_reuse(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse", @@ -572,7 +575,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_reuse_disabled_by_workflow(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", @@ -608,7 +611,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_on_error(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop", @@ -635,7 +638,7 @@ class TestSubmit(unittest.TestCase): def test_submit_container_output_name(self, stubs): output_name = "test_output_name" - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name, @@ -661,7 +664,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_storage_classes(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo", @@ -728,7 +731,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_output_ttl(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600", @@ -754,7 +757,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_trash_intermediate(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate", @@ -782,7 +785,7 @@ class TestSubmit(unittest.TestCase): def test_submit_container_output_tags(self, stubs): output_tags = "tag0,tag1,tag2" - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags, @@ -807,7 +810,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_runner_ram(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048", @@ -829,7 +832,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_file_keepref(self, stubs, tm, collectionReader): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt") exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", @@ -842,7 +845,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_keepref(self, stubs, tm, reader): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() with open("tests/wf/expect_arvworkflow.cwl") as f: reader().open().__enter__().read.return_value = f.read() @@ -904,7 +907,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_jobs_keepref(self, stubs, tm, reader): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() with open("tests/wf/expect_arvworkflow.cwl") as f: reader().open().__enter__().read.return_value = f.read() @@ -927,7 +930,7 @@ class TestSubmit(unittest.TestCase): @mock.patch("time.sleep") @stubs def test_submit_arvworkflow(self, stubs, tm): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() with open("tests/wf/expect_arvworkflow.cwl") as f: stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"} @@ -1026,7 +1029,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_name(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123", @@ -1047,14 +1050,14 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_missing_input(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"], capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) self.assertEqual(exited, 0) - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"], @@ -1065,7 +1068,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_project(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid, @@ -1093,7 +1096,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_eval_timeout(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60", @@ -1119,7 +1122,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_collection_cache(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500", @@ -1147,7 +1150,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_thread_count(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20", @@ -1173,7 +1176,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_job_runner_image(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123", @@ -1193,7 +1196,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_runner_image(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123", @@ -1213,7 +1216,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_priority(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669", @@ -1234,7 +1237,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_wf_runner_resources(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", @@ -1315,7 +1318,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_secrets(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", @@ -1492,7 +1495,7 @@ class TestSubmit(unittest.TestCase): "state": "Queued" } - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy", @@ -1509,7 +1512,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_container_cluster_id(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123" try: exited = arvados_cwl.main( @@ -1530,7 +1533,7 @@ class TestSubmit(unittest.TestCase): @stubs def test_submit_validate_cluster_id(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123" exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc", @@ -1565,7 +1568,7 @@ class TestCreateTemplate(unittest.TestCase): def test_create(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--create-workflow", "--debug", @@ -1598,7 +1601,7 @@ class TestCreateTemplate(unittest.TestCase): def test_create_name(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--create-workflow", "--debug", @@ -1632,7 +1635,7 @@ class TestCreateTemplate(unittest.TestCase): def test_update_name(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--update-workflow", self.existing_template_uuid, @@ -1673,7 +1676,7 @@ class TestCreateWorkflow(unittest.TestCase): def test_create(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--create-workflow", "--debug", @@ -1705,7 +1708,7 @@ class TestCreateWorkflow(unittest.TestCase): def test_create_name(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--create-workflow", "--debug", @@ -1735,7 +1738,7 @@ class TestCreateWorkflow(unittest.TestCase): @stubs def test_incompatible_api(self, stubs): - capture_stderr = cStringIO.StringIO() + capture_stderr = io.StringIO() logging.getLogger('arvados.cwl-runner').addHandler( logging.StreamHandler(capture_stderr)) @@ -1752,7 +1755,7 @@ class TestCreateWorkflow(unittest.TestCase): @stubs def test_update(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--update-workflow", self.existing_workflow_uuid, @@ -1777,7 +1780,7 @@ class TestCreateWorkflow(unittest.TestCase): @stubs def test_update_name(self, stubs): - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--update-workflow", self.existing_workflow_uuid, @@ -1804,7 +1807,7 @@ class TestCreateWorkflow(unittest.TestCase): def test_create_collection_per_tool(self, stubs): project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz' - capture_stdout = cStringIO.StringIO() + capture_stdout = io.StringIO() exited = arvados_cwl.main( ["--create-workflow", "--debug", @@ -1884,7 +1887,7 @@ class TestTemplateInputs(unittest.TestCase): exited = arvados_cwl.main( ["--create-template", "tests/wf/inputs_test.cwl", "tests/order/empty_order.json"], - cStringIO.StringIO(), sys.stderr, api_client=stubs.api) + io.StringIO(), sys.stderr, api_client=stubs.api) self.assertEqual(exited, 0) stubs.api.pipeline_templates().create.assert_called_with( @@ -1895,7 +1898,7 @@ class TestTemplateInputs(unittest.TestCase): exited = arvados_cwl.main( ["--create-template", "tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"], - cStringIO.StringIO(), sys.stderr, api_client=stubs.api) + io.StringIO(), sys.stderr, api_client=stubs.api) self.assertEqual(exited, 0) expect_template = copy.deepcopy(self.expect_template) diff --git a/sdk/cwl/tests/wf/check_mem.py b/sdk/cwl/tests/wf/check_mem.py index bad16ed06b..88eb2fd513 100644 --- a/sdk/cwl/tests/wf/check_mem.py +++ b/sdk/cwl/tests/wf/check_mem.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import print_function +from __future__ import division +from past.utils import old_div import arvados import sys import os @@ -10,7 +12,7 @@ import os if "JOB_UUID" in os.environ: requested = arvados.api().jobs().get(uuid=os.environ["JOB_UUID"]).execute()["runtime_constraints"]["min_ram_mb_per_node"] else: - requested = arvados.api().containers().current().execute()["runtime_constraints"]["ram"]/(1024*1024) + requested = old_div(arvados.api().containers().current().execute()["runtime_constraints"]["ram"],(1024*1024)) print("Requested %d expected %d" % (requested, int(sys.argv[1]))) -- 2.30.2