Merge branch '18723-cwl-upload' refs #18723

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index c3848b2629188c9cf687e8b69f054237b9e5b77e..734945c0f7ccb707258ab60be52107195afff0e0 100644 (file)
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -258,7 +258,11 @@ def exit_signal_handler(sigcode, frame):
     logger.error(str(u"Caught signal {}, exiting.").format(sigcode))
     sys.exit(-sigcode)
 
-def main(args, stdout, stderr, api_client=None, keep_client=None,
+def main(args=sys.argv[1:],
+         stdout=sys.stdout,
+         stderr=sys.stderr,
+         api_client=None,
+         keep_client=None,
          install_sig_handlers=True):
     parser = arg_parser()
 

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 145f1ad7f9d7572f1f4074da514cd0ea0f771d3e..7d6d287a207455d04e2a1d5469e053ea57f7cf6e 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -283,9 +283,10 @@ def upload_dependencies(arvrunner, name, document_loader,
     metadata = scanobj
 
     sc_result = scandeps(uri, scanobj,
-                  loadref_fields,
-                  set(("$include", "$schemas", "location")),
-                  loadref, urljoin=document_loader.fetcher.urljoin)
+                         loadref_fields,
+                         set(("$include", "$schemas", "location")),
+                         loadref, urljoin=document_loader.fetcher.urljoin,
+                         nestdirs=False)
 
     sc = []
     uuids = {}

diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index f034ca5ab2f1ed82a385431fb1e32acea7c17fc7..e739235953cf799e07e9f13440ef7e05963241c9 100644 (file)
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -31,15 +31,12 @@ setup(name='arvados-cwl-runner',
       license='Apache 2.0',
       packages=find_packages(),
       package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml', 'arv-cwl-schema-v1.2.yml']},
-      scripts=[
-          'bin/cwl-runner',
-          'bin/arvados-cwl-runner',
-      ],
+      entry_points={"console_scripts": ["cwl-runner=arvados_cwl:main", "arvados-cwl-runner=arvados_cwl:main"]},
       # Note that arvados/build/run-build-packages.sh looks at this
       # file to determine what version of cwltool and schema-salad to
       # build.
       install_requires=[
-          'cwltool==3.1.20211107152837',
+          'cwltool==3.1.20220210171524',
           'schema-salad==8.2.20211116214159',
           'arvados-python-client{}'.format(pysdk_dep),
           'setuptools',

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index ae22d65f4db22f3a2b285320d1501a79d0a2e139..5282e93929fd19c932c521ac3e004950f66c533f 100644 (file)
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -439,3 +439,8 @@
     "outstr": "foo woble bar"
   tool: 17879-ignore-sbg-fields.cwl
   doc: "Test issue 17879 - ignores sbg fields"
+
+- job: chipseq/chip-seq-single.json
+  output: {}
+  tool: chipseq/cwl-packed.json
+  doc: "Test issue 18723 - correctly upload two directories with the same basename"

diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 1e0068ffd48250375c60b8dc1331668a930fb809..b55b056b2d38339fe4bb42ddd15ba267099975ea 100644 (file)
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -23,9 +23,7 @@ ARG pipcmd=pip3
 
 RUN apt-get update -q && apt-get install -qy --no-install-recommends \
     git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
-    libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
-
-RUN $pipcmd install -U setuptools six requests
+    libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential ${pythoncmd}-setuptools
 
 ARG sdk
 ARG runner
@@ -39,7 +37,7 @@ ADD cwl/dist/$runner /tmp/
 
 RUN cd /tmp/arvados-python-client-* && $pipcmd install .
 RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install networkx==2.2 && $pipcmd install . ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
 RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
 
 # Install dependencies and set up system.