Merge branch 'master' of git.curoverse.com:arvados into 13330-cwl-intermediate-collec...

Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic@capeannenterprises.com>

diff --cc sdk/cwl/arvados_cwl/arvcontainer.py
Simple merge

diff --cc sdk/cwl/arvados_cwl/arvjob.py
index 332c6a7b9fb3383c13ce23e606a4243fe286d7a5,70c2173db9fa2f7ff5054ff4be7252bd64156b67..2d112c87a7c5affa2b6d490e526ae1e2d2aea28b
--- 1/sdk/cwl/arvados_cwl/arvjob.py
--- 2/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@@ -6,10 -6,9 +6,10 @@@ import loggin
  import re
  import copy
  import json
 +import datetime
  import time
  
- from cwltool.process import get_feature, shortname, UnsupportedRequirement
+ from cwltool.process import shortname, UnsupportedRequirement
  from cwltool.errors import WorkflowException
  from cwltool.command_line_tool import revmap_file, CommandLineTool
  from cwltool.load_tool import fetch_document

diff --cc sdk/cwl/arvados_cwl/pathmapper.py
Simple merge

diff --cc sdk/cwl/tests/test_container.py
index a8a91efd3ad01432a281a9b14a107ebc2e7a88d2,dd484690c14ebb11caeae501da978f81b0e24abd..bd5b1a1b613850995ecdc36cb741d81c8b13766e
--- 1/sdk/cwl/tests/test_container.py
--- 2/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@@ -641,15 -649,45 +656,59 @@@ class TestContainer(unittest.TestCase)
                      }
                  }))
  
+     # The test passes no builder.resources
+     # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+     @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+     def test_timelimit(self, keepdocker):
+         arv_docker_clear_cache()
+ 
+         runner = mock.MagicMock()
+         runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+         runner.ignore_docker_for_reuse = False
+         runner.intermediate_output_ttl = 0
+         runner.secret_store = cwltool.secrets.SecretStore()
+ 
+         keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+         runner.api.collections().get().execute.return_value = {
+             "portable_data_hash": "99999999999999999999999999999993+99"}
+ 
+         tool = cmap({
+             "inputs": [],
+             "outputs": [],
+             "baseCommand": "ls",
+             "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+             "id": "#",
+             "class": "CommandLineTool",
+             "hints": [
+                 {
+                     "class": "http://commonwl.org/cwltool#TimeLimit",
+                     "timelimit": 42
+                 }
+             ]
+         })
+ 
+         loadingContext, runtimeContext = self.helper(runner)
+         runtimeContext.name = "test_timelimit"
+ 
+         arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
+         arvtool.formatgraph = None
+ 
+         for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+             j.run(runtimeContext)
+ 
+         _, kwargs = runner.api.container_requests().create.call_args
+         self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
++
++
 +    def test_get_intermediate_collection_info(self):
 +        arvrunner = mock.MagicMock()
 +        arvrunner.intermediate_output_ttl = 60
 +        arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
 +
 +        container = arvados_cwl.ArvadosContainer(arvrunner)
 +
 +        info = container._get_intermediate_collection_info()
 +
 +        self.assertEqual(info["name"], "Intermediate collection")
 +        self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
 +        self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})

diff --cc sdk/cwl/tests/test_job.py
index 1841fd3f89ea7a89aa1ae246f4f0dd5d305c9532,c110bc5d53cd4634656d93fab2937954be973d07..b9f1c396172b6d32e0b22423878ed2b0c78696cf
--- 1/sdk/cwl/tests/test_job.py
--- 2/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@@ -25,15 -24,30 +25,37 @@@ if not os.getenv('ARVADOS_DEBUG')
      logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
      logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
  
 +class MockDateTime(datetime.datetime):
 +    @classmethod
 +    def now(cls):
 +        return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
 +
 +datetime.datetime = MockDateTime
 +
  class TestJob(unittest.TestCase):
  
+     def helper(self, runner, enable_reuse=True):
+         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ 
+         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+         loadingContext = arvados_cwl.context.ArvLoadingContext(
+             {"avsc_names": avsc_names,
+              "basedir": "",
+              "make_fs_access": make_fs_access,
+              "loader": Loader({}),
+              "metadata": {"cwlVersion": "v1.0"},
+              "makeTool": runner.arv_make_tool})
+         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+             {"work_api": "jobs",
+              "basedir": "",
+              "name": "test_run_job_"+str(enable_reuse),
+              "make_fs_access": make_fs_access,
+              "enable_reuse": enable_reuse,
+              "priority": 500})
+ 
+         return loadingContext, runtimeContext
+ 
      # The test passes no builder.resources
      # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
      @mock.patch('arvados.commands.keepdocker.list_images_in_arv')
@@@ -315,22 -331,36 +339,50 @@@
  
          arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
  
 +    def test_get_intermediate_collection_info(self):
 +        arvrunner = mock.MagicMock()
 +        arvrunner.intermediate_output_ttl = 60
 +        arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
 +
 +        job = arvados_cwl.ArvadosJob(arvrunner)
 +
 +        info = job._get_intermediate_collection_info()
 +
 +        self.assertEqual(info["name"], "Intermediate collection")
 +        self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
 +        self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
 +
 +
  
  class TestWorkflow(unittest.TestCase):
+     def helper(self, runner, enable_reuse=True):
+         document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+ 
+         make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+                                          collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ 
+         document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
+         document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
+         document_loader.fetch_text = document_loader.fetcher.fetch_text
+         document_loader.check_exists = document_loader.fetcher.check_exists
+ 
+         loadingContext = arvados_cwl.context.ArvLoadingContext(
+             {"avsc_names": avsc_names,
+              "basedir": "",
+              "make_fs_access": make_fs_access,
+              "loader": document_loader,
+              "metadata": {"cwlVersion": "v1.0"},
+              "construct_tool_object": runner.arv_make_tool})
+         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+             {"work_api": "jobs",
+              "basedir": "",
+              "name": "test_run_wf_"+str(enable_reuse),
+              "make_fs_access": make_fs_access,
+              "enable_reuse": enable_reuse,
+              "priority": 500})
+ 
+         return loadingContext, runtimeContext
+ 
      # The test passes no builder.resources
      # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
      @mock.patch("arvados.collection.CollectionReader")