From: Peter Amstutz Date: Thu, 31 Mar 2016 14:21:17 +0000 (-0400) Subject: 8857: Add --ignore-docker-for-reuse option to assist workflow development. X-Git-Tag: 1.1.0~1016^2~1 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/92c876d90253a0db23d40aa125cba335f7bf7e27?ds=sidebyside 8857: Add --ignore-docker-for-reuse option to assist workflow development. --- diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index e3fd1fccd3..339b91ce59 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -157,16 +157,26 @@ class ArvadosJob(object): runtime_constraints["min_ram_mb_per_node"] = resources.get("ram") runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0) + filters = [["repository", "=", "arvados"], + ["script", "=", "crunchrunner"], + ["script_version", "in git", "9e5b98e8f5f4727856b53447191f9c06e3da2ba6"]] + if not self.arvrunner.ignore_docker_for_reuse: + filters.append(["docker_image_locator", "in docker", runtime_constraints["docker_image"]]) + try: - response = self.arvrunner.api.jobs().create(body={ - "owner_uuid": self.arvrunner.project_uuid, - "script": "crunchrunner", - "repository": "arvados", - "script_version": "master", - "minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6", - "script_parameters": {"tasks": [script_parameters]}, - "runtime_constraints": runtime_constraints - }, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries) + response = self.arvrunner.api.jobs().create( + body={ + "owner_uuid": self.arvrunner.project_uuid, + "script": "crunchrunner", + "repository": "arvados", + "script_version": "master", + "minimum_script_version": "9e5b98e8f5f4727856b53447191f9c06e3da2ba6", + "script_parameters": {"tasks": [script_parameters]}, + "runtime_constraints": runtime_constraints + }, + filters=filters, + find_or_create=kwargs.get("enable_reuse", True) + ).execute(num_retries=self.arvrunner.num_retries) self.arvrunner.jobs[response["uuid"]] = self @@ -396,6 +406,7 @@ class ArvCwlRunner(object): events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message) self.debug = args.debug + self.ignore_docker_for_reuse = args.ignore_docker_for_reuse self.fs_access = CollectionFsAccess(input_basedir) kwargs["fs_access"] = self.fs_access @@ -474,6 +485,9 @@ def main(args, stdout, stderr, api_client=None): default=True, dest="enable_reuse", help="") parser.add_argument("--project-uuid", type=str, help="Project that will own the workflow jobs") + parser.add_argument("--ignore-docker-for-reuse", action="store_true", + help="Ignore Docker image version when deciding whether to reuse past jobs.", + default=False) try: runner = ArvCwlRunner(api_client=arvados.api('v1', model=OrderedJsonModel())) diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py index cb306ed69e..bb15e60f1f 100644 --- a/sdk/cwl/tests/test_job.py +++ b/sdk/cwl/tests/test_job.py @@ -9,6 +9,8 @@ class TestJob(unittest.TestCase): def test_run(self): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + tool = { "inputs": [], "outputs": [], @@ -18,32 +20,41 @@ class TestJob(unittest.TestCase): arvtool.formatgraph = None for j in arvtool.job({}, "", mock.MagicMock()): j.run() - runner.api.jobs().create.assert_called_with(body={ - 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', - 'runtime_constraints': {}, - 'script_parameters': { - 'tasks': [{ - 'task.env': {'TMPDIR': '$(task.tmpdir)'}, - 'command': ['ls'] - }] + runner.api.jobs().create.assert_called_with( + body={ + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'runtime_constraints': {}, + 'script_parameters': { + 'tasks': [{ + 'task.env': {'TMPDIR': '$(task.tmpdir)'}, + 'command': ['ls'] + }] }, 'script_version': 'master', - 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', - 'repository': 'arvados', - 'script': 'crunchrunner', - 'runtime_constraints': { - 'docker_image': 'arvados/jobs', - 'min_cores_per_node': 1, - 'min_ram_mb_per_node': 1024, - 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize - } - }, find_or_create=True) + 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', + 'repository': 'arvados', + 'script': 'crunchrunner', + 'runtime_constraints': { + 'docker_image': 'arvados/jobs', + 'min_cores_per_node': 1, + 'min_ram_mb_per_node': 1024, + 'min_scratch_mb_per_node': 2048 # tmpdirSize + outdirSize + } + }, + find_or_create=True, + filters=[['repository', '=', 'arvados'], + ['script', '=', 'crunchrunner'], + ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'], + ['docker_image_locator', 'in docker', 'arvados/jobs']] + ) # The test passes some fields in builder.resources # For the remaining fields, the defaults will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} def test_resource_requirements(self): runner = mock.MagicMock() runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + tool = { "inputs": [], "outputs": [], @@ -59,26 +70,32 @@ class TestJob(unittest.TestCase): arvtool.formatgraph = None for j in arvtool.job({}, "", mock.MagicMock()): j.run() - runner.api.jobs().create.assert_called_with(body={ - 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', - 'runtime_constraints': {}, - 'script_parameters': { - 'tasks': [{ - 'task.env': {'TMPDIR': '$(task.tmpdir)'}, - 'command': ['ls'] - }] + runner.api.jobs().create.assert_called_with( + body={ + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'runtime_constraints': {}, + 'script_parameters': { + 'tasks': [{ + 'task.env': {'TMPDIR': '$(task.tmpdir)'}, + 'command': ['ls'] + }] }, 'script_version': 'master', - 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', - 'repository': 'arvados', - 'script': 'crunchrunner', - 'runtime_constraints': { - 'docker_image': 'arvados/jobs', - 'min_cores_per_node': 3, - 'min_ram_mb_per_node': 3000, - 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize - } - }, find_or_create=True) + 'minimum_script_version': '9e5b98e8f5f4727856b53447191f9c06e3da2ba6', + 'repository': 'arvados', + 'script': 'crunchrunner', + 'runtime_constraints': { + 'docker_image': 'arvados/jobs', + 'min_cores_per_node': 3, + 'min_ram_mb_per_node': 3000, + 'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize + } + }, + find_or_create=True, + filters=[['repository', '=', 'arvados'], + ['script', '=', 'crunchrunner'], + ['script_version', 'in git', '9e5b98e8f5f4727856b53447191f9c06e3da2ba6'], + ['docker_image_locator', 'in docker', 'arvados/jobs']]) @mock.patch("arvados.collection.Collection") def test_done(self, col): @@ -88,6 +105,7 @@ class TestJob(unittest.TestCase): runner.api = api runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.num_retries = 0 + runner.ignore_docker_for_reuse = False col().open.return_value = [] api.collections().list().execute.side_effect = ({"items": []},