projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch '21765-short-dialog'
[arvados.git]
/
sdk
/
cwl
/
arvados_cwl
/
fsaccess.py
diff --git
a/sdk/cwl/arvados_cwl/fsaccess.py
b/sdk/cwl/arvados_cwl/fsaccess.py
index 252ca57d47bb30ad1834e7070b3cfdf1e0ffdbb1..a88380b468cfa3100725b79d248a370b8c136e38 100644
(file)
--- a/
sdk/cwl/arvados_cwl/fsaccess.py
+++ b/
sdk/cwl/arvados_cwl/fsaccess.py
@@
-2,12
+2,6
@@
#
# SPDX-License-Identifier: Apache-2.0
#
# SPDX-License-Identifier: Apache-2.0
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-from builtins import str
-from future.utils import viewvalues
-
import fnmatch
import os
import errno
import fnmatch
import os
import errno
@@
-16,8
+10,9
@@
import re
import logging
import threading
from collections import OrderedDict
import logging
import threading
from collections import OrderedDict
+from io import StringIO
-import ruamel.yaml
as yaml
+import ruamel.yaml
import cwltool.stdfsaccess
from cwltool.pathmapper import abspath
import cwltool.stdfsaccess
from cwltool.pathmapper import abspath
@@
-100,7
+95,8
@@
class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
if p.startswith("keep:") and (arvados.util.keep_locator_pattern.match(p[5:]) or
arvados.util.collection_uuid_pattern.match(p[5:])):
locator = p[5:]
if p.startswith("keep:") and (arvados.util.keep_locator_pattern.match(p[5:]) or
arvados.util.collection_uuid_pattern.match(p[5:])):
locator = p[5:]
- return (self.collection_cache.get(locator), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None)
+ rest = os.path.normpath(urllib.parse.unquote(sp[1])) if len(sp) == 2 else None
+ return (self.collection_cache.get(locator), rest)
else:
return (None, path)
else:
return (None, path)
@@
-128,15
+124,15
@@
class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
def glob(self, pattern):
collection, rest = self.get_collection(pattern)
def glob(self, pattern):
collection, rest = self.get_collection(pattern)
- if collection is not None and
not rest
:
+ if collection is not None and
rest in (None, "", ".")
:
return [pattern]
patternsegments = rest.split("/")
return sorted(self._match(collection, patternsegments, "keep:" + collection.manifest_locator()))
return [pattern]
patternsegments = rest.split("/")
return sorted(self._match(collection, patternsegments, "keep:" + collection.manifest_locator()))
- def open(self, fn, mode):
+ def open(self, fn, mode
, encoding=None
):
collection, rest = self.get_collection(fn)
if collection is not None:
collection, rest = self.get_collection(fn)
if collection is not None:
- return collection.open(rest, mode)
+ return collection.open(rest, mode
, encoding=encoding
)
else:
return super(CollectionFsAccess, self).open(self._abs(fn), mode)
else:
return super(CollectionFsAccess, self).open(self._abs(fn), mode)
@@
-148,6
+144,11
@@
class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
return False
else:
raise
return False
else:
raise
+ except IOError as err:
+ if err.errno == errno.ENOENT:
+ return False
+ else:
+ raise
if collection is not None:
if rest:
return collection.exists(rest)
if collection is not None:
if rest:
return collection.exists(rest)
@@
-223,24
+224,29
@@
class CollectionFetcher(DefaultFetcher):
self.fsaccess = fs_access
self.num_retries = num_retries
self.fsaccess = fs_access
self.num_retries = num_retries
- def fetch_text(self, url):
+ def fetch_text(self, url
, content_types=None
):
if url.startswith("keep:"):
if url.startswith("keep:"):
- with self.fsaccess.open(url, "r") as f:
+ with self.fsaccess.open(url, "r"
, encoding="utf-8"
) as f:
return f.read()
if url.startswith("arvwf:"):
record = self.api_client.workflows().get(uuid=url[6:]).execute(num_retries=self.num_retries)
return f.read()
if url.startswith("arvwf:"):
record = self.api_client.workflows().get(uuid=url[6:]).execute(num_retries=self.num_retries)
- definition = record["definition"] + ('\nlabel: "%s"\n' % record["name"].replace('"', '\\"'))
- return definition
+ yaml = ruamel.yaml.YAML(typ='rt', pure=True)
+ definition = yaml.load(record["definition"])
+ definition["label"] = record["name"]
+ stream = StringIO()
+ yaml.dump(definition, stream)
+ return stream.getvalue()
return super(CollectionFetcher, self).fetch_text(url)
def check_exists(self, url):
try:
if url.startswith("http://arvados.org/cwl"):
return True
return super(CollectionFetcher, self).fetch_text(url)
def check_exists(self, url):
try:
if url.startswith("http://arvados.org/cwl"):
return True
- if url.startswith("keep:"):
- return self.fsaccess.exists(url)
- if url.startswith("arvwf:"):
- if self.fetch_text(url):
+ urld, _ = urllib.parse.urldefrag(url)
+ if urld.startswith("keep:"):
+ return self.fsaccess.exists(urld)
+ if urld.startswith("arvwf:"):
+ if self.fetch_text(urld):
return True
except arvados.errors.NotFoundError:
return False
return True
except arvados.errors.NotFoundError:
return False
@@
-302,7
+308,7
@@
def collectionResolver(api_client, document_loader, uri, num_retries=4):
if pipeline_template_uuid_pattern.match(uri):
pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
if pipeline_template_uuid_pattern.match(uri):
pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
- return u"keep:" +
viewvalues(pt["components"])[0]
["script_parameters"]["cwl:tool"]
+ return u"keep:" +
next(pt["components"].values())
["script_parameters"]["cwl:tool"]
p = uri.split("/")
if arvados.util.keep_locator_pattern.match(p[0]):
p = uri.split("/")
if arvados.util.keep_locator_pattern.match(p[0]):