Merge branch 'master' into 10081-cwl-run-same-job

Conflicts:
	sdk/cwl/arvados_cwl/runner.py

diff --combined sdk/cwl/arvados_cwl/pathmapper.py
index ffdade0f5ecbc1562e67dfacdf090c77e7fa24c2,83715548a45a423bdc27ba768f00d592b313563a..0fd9a0ed332ececaf4474ca3939fe9b4785db82d
--- 1/sdk/cwl/arvados_cwl/pathmapper.py
--- 2/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@@ -6,7 -6,7 +6,7 @@@ import o
  import arvados.commands.run
  import arvados.collection
  
- from cwltool.pathmapper import PathMapper, MapperEnt, abspath
+ from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
  from cwltool.workflow import WorkflowException
  
  logger = logging.getLogger('arvados.cwl-runner')
@@@ -42,8 -42,11 +42,11 @@@ class ArvPathMapper(PathMapper)
                      uploadfiles.add((src, ab, st))
                  elif isinstance(st, arvados.commands.run.ArvFile):
                      self._pathmap[src] = MapperEnt(ab, st.fn, "File")
-                 elif src.startswith("_:") and "contents" in srcobj:
-                     pass
+                 elif src.startswith("_:"):
+                     if "contents" in srcobj:
+                         pass
+                     else:
+                         raise WorkflowException("File literal '%s' is missing contents" % src)
                  else:
                      raise WorkflowException("Input file path '%s' is invalid" % st)
              if "secondaryFiles" in srcobj:
@@@ -52,7 -55,7 +55,7 @@@
          elif srcobj["class"] == "Directory":
              if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
                  self._pathmap[src] = MapperEnt(src, self.collection_pattern % src[5:], "Directory")
 -            for l in srcobj["listing"]:
 +            for l in srcobj.get("listing", []):
                  self.visit(l, uploadfiles)
  
      def addentry(self, obj, c, path, subdirs):

diff --combined sdk/cwl/arvados_cwl/runner.py
index 2e52b63ed695d36b9e9d28efc49547c9875e6c53,bdd80771ff224392516fc216a6e25d0248f3c211..cf2f1db997b9db9688b52258994bc6179863783d
--- 1/sdk/cwl/arvados_cwl/runner.py
--- 2/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@@ -69,26 -69,19 +69,19 @@@ def upload_dependencies(arvrunner, name
                    set(("$include", "$schemas", "location")),
                    loadref)
  
-     files = []
-     def visitFiles(path):
-         files.append(path)
- 
-     adjustFileObjs(sc, visitFiles)
-     adjustDirObjs(sc, visitFiles)
- 
-     normalizeFilesDirs(files)
+     normalizeFilesDirs(sc)
  
      if "id" in workflowobj:
-         files.append({"class": "File", "location": workflowobj["id"]})
+         sc.append({"class": "File", "location": workflowobj["id"]})
  
-     mapper = ArvPathMapper(arvrunner, files, "",
+     mapper = ArvPathMapper(arvrunner, sc, "",
                             "keep:%s",
                             "keep:%s/%s",
                             name=name)
  
      def setloc(p):
-         if not p["location"].startswith("keep:"):
-             p["location"] = mapper.mapper(p["location"]).resolved
 -        if not p["location"].startswith("_:"):
++        if not p["location"].startswith("_:") and not p["location"].startswith("keep:"):
+             p["location"] = mapper.mapper(p["location"]).target
      adjustFileObjs(workflowobj, setloc)
      adjustDirObjs(workflowobj, setloc)
  

diff --combined sdk/cwl/setup.py
index 25b6c8366e89afce215cb627b6f56e1a0b723636,31841df42060d70235cac1ced4c38a38342d8364..fc5a52c588cba05395f0fe4064027c6a6be3aebd
--- 1/sdk/cwl/setup.py
--- 2/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@@ -25,7 -25,6 +25,7 @@@ setup(name='arvados-cwl-runner'
        download_url="https://github.com/curoverse/arvados.git",
        license='Apache 2.0',
        packages=find_packages(),
 +      package_data={'arvados_cwl': ['arv-cwl-schema.yml']},
        scripts=[
            'bin/cwl-runner',
            'bin/arvados-cwl-runner'
@@@ -33,7 -32,7 +33,7 @@@
        # Make sure to update arvados/build/run-build-packages.sh as well
        # when updating the cwltool version pin.
        install_requires=[
-           'cwltool==1.0.20160922135240',
+           'cwltool==1.0.20160923180109',
            'arvados-python-client>=0.1.20160826210445'
        ],
        data_files=[