From cbc620e6517c169fb265153eb6c995eebe959721 Mon Sep 17 00:00:00 2001
From: Peter Amstutz <peter.amstutz@curoverse.com>
Date: Thu, 22 Sep 2016 12:29:33 -0400
Subject: [PATCH] 9988: Add arvados-specfic hints RuntimeConstraints and
 APIRequirement.

---
 sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 33 ++++++++++++++++++++++++++
 sdk/cwl/arvados_cwl/arvcontainer.py    |  8 +++++++
 sdk/cwl/arvados_cwl/arvjob.py          |  4 ++++
 sdk/cwl/tests/test_container.py        |  8 ++++++-
 sdk/cwl/tests/test_job.py              |  8 ++++++-
 5 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 2e5044dc0a..2a81ed4ed7 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -12,3 +12,36 @@ $graph:
       jsonldPredicate:
         _id: "@type"
         _type: "@vocab"
+
+- name: RuntimeConstraints
+  type: record
+  doc: |
+    Set Arvados-specific runtime hints.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'arv:RuntimeConstraints'"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    - name: keep_cache
+      type: int?
+      doc: |
+        Size of file data buffer for Keep mount in MiB. Default is 256
+        MiB. Increase this to reduce cache thrashing in situtations such as
+        accessing multiple large (64+ MiB) files at the same time, or
+        performing random access on a large file.
+
+- name: APIRequirement
+  type: record
+  doc: |
+    Indicates that process wants to access to the Arvados API.  Will be granted
+    limited network access and have ARVADOS_API_HOST and ARVADOS_API_TOKEN set
+    in the environment.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'arv:APIRequirement'"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 413435db23..aaae7d9f66 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -90,6 +90,14 @@ class ArvadosContainer(object):
             runtime_constraints["vcpus"] = resources.get("cores", 1)
             runtime_constraints["ram"] = resources.get("ram") * 2**20
 
+        api_req, _ = get_feature(self, "http://arvados.org/cwl#APIRequirement")
+        if api_req:
+            runtime_constraints["API"] = True
+
+        runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints")
+        if runtime_req:
+            logger.warn("RuntimeConstraints not yet supported by container API")
+
         container_request["mounts"] = mounts
         container_request["runtime_constraints"] = runtime_constraints
 
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 78bca6e0e1..1098002481 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -85,6 +85,10 @@ class ArvadosJob(object):
             runtime_constraints["min_ram_mb_per_node"] = resources.get("ram")
             runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
 
+        runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints")
+        if runtime_req:
+            runtime_constraints["keep_cache_mb_per_task"] = runtime_req["keep_cache"]
+
         filters = [["repository", "=", "arvados"],
                    ["script", "=", "crunchrunner"],
                    ["script_version", "in git", "9e5b98e8f5f4727856b53447191f9c06e3da2ba6"]]
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 7d941cf8fc..822a213fe7 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -86,6 +86,11 @@ class TestContainer(unittest.TestCase):
                 "coresMin": 3,
                 "ramMin": 3000,
                 "tmpdirMin": 4000
+            }, {
+                "class": "http://arvados.org/cwl#RuntimeConstraints",
+                "keep_cache": 512
+            }, {
+                "class": "http://arvados.org/cwl#APIRequirement",
             }],
             "baseCommand": "ls"
         }
@@ -107,7 +112,8 @@ class TestContainer(unittest.TestCase):
                 'name': 'test_resource_requirements',
                 'runtime_constraints': {
                     'vcpus': 3,
-                    'ram': 3145728000
+                    'ram': 3145728000,
+                    'API': True
                 }, 'priority': 1,
                 'mounts': {
                     '/var/spool/cwl': {'kind': 'tmp'}
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index fdb4520018..e36a6076ab 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -80,6 +80,11 @@ class TestJob(unittest.TestCase):
                 "coresMin": 3,
                 "ramMin": 3000,
                 "tmpdirMin": 4000
+            }, {
+                "class": "http://arvados.org/cwl#RuntimeConstraints",
+                "keep_cache": 512
+            }, {
+                "class": "http://arvados.org/cwl#APIRequirement",
             }],
             "baseCommand": "ls"
         }
@@ -107,7 +112,8 @@ class TestJob(unittest.TestCase):
                     'docker_image': 'arvados/jobs',
                     'min_cores_per_node': 3,
                     'min_ram_mb_per_node': 3000,
-                    'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
+                    'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
+                    'keep_cache_mb_per_task': 512
                 }
             },
             find_or_create=True,
-- 
2.30.2