sdk/cwl/tests/test_copy_deps.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import arvados.collection
   7 import subprocess
   8
   9 api = arvados.api()
  10
  11 workflow_content = """# Copyright (C) The Arvados Authors. All rights reserved.
  12 #
  13 # SPDX-License-Identifier: Apache-2.0
  14
  15 cwlVersion: v1.2
  16 class: CommandLineTool
  17 baseCommand: echo
  18 inputs:
  19   message:
  20     type: File
  21     inputBinding:
  22       position: 1
  23     default:
  24       class: File
  25       location: keep:d7514270f356df848477718d58308cc4+94/b
  26
  27 outputs: []
  28 """
  29
  30 expect_file = "19070-copy-deps.cwl"
  31
  32 def check_workflow_content(uuid):
  33     c = arvados.collection.Collection(uuid)
  34     try:
  35         with c.open(expect_file) as f:
  36             content = f.read()
  37         match = (content == workflow_content)
  38         return match
  39     except:
  40         return False
  41
  42 def check_contents(group, wf_uuid):
  43     contents = api.groups().contents(uuid=group["uuid"]).execute()
  44     if len(contents["items"]) != 4:
  45         raise Exception("Expected 4 items in "+group["uuid"]+" was "+str(len(contents["items"])))
  46
  47     found = False
  48     for c in contents["items"]:
  49         if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  50             found = True
  51     if not found:
  52         raise Exception("Couldn't find workflow in "+group["uuid"])
  53
  54     found = False
  55     for c in contents["items"]:
  56         if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
  57             found = True
  58     if not found:
  59         raise Exception("Couldn't find collection dependency")
  60
  61     found = False
  62     for c in contents["items"]:
  63         if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
  64             found = True
  65     if not found:
  66         raise Exception("Couldn't find jobs image dependency")
  67
  68     found = False
  69     for c in contents["items"]:
  70         if c["kind"] == "arvados#collection" and check_workflow_content(c["portable_data_hash"]):
  71             found = True
  72     if not found:
  73         raise Exception("Couldn't find collection containing expected "+expect_file)
  74
  75
  76 def check_create():
  77     group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
  78     try:
  79         contents = api.groups().contents(uuid=group["uuid"]).execute()
  80         if len(contents["items"]) != 0:
  81             raise Exception("Expected 0 items")
  82
  83         # Create workflow, by default should also copy dependencies
  84         cmd = ["arvados-cwl-runner", "--disable-git", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
  85         print(" ".join(cmd))
  86         wf_uuid = subprocess.check_output(cmd)
  87         wf_uuid = wf_uuid.decode("utf-8").strip()
  88         check_contents(group, wf_uuid)
  89     finally:
  90         api.groups().delete(uuid=group["uuid"]).execute()
  91
  92
  93 def check_update():
  94     group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
  95     try:
  96         contents = api.groups().contents(uuid=group["uuid"]).execute()
  97         if len(contents["items"]) != 0:
  98             raise Exception("Expected 0 items")
  99
 100         # Create workflow, but with --no-copy-deps it shouldn't copy anything
 101         cmd = ["arvados-cwl-runner", "--disable-git", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
 102         print(" ".join(cmd))
 103         wf_uuid = subprocess.check_output(cmd)
 104         wf_uuid = wf_uuid.decode("utf-8").strip()
 105
 106         contents = api.groups().contents(uuid=group["uuid"]).execute()
 107         if len(contents["items"]) != 2:
 108             raise Exception("Expected 2 items")
 109
 110         found = False
 111         for c in contents["items"]:
 112             if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
 113                 found = True
 114         if not found:
 115             raise Exception("Couldn't find workflow")
 116
 117         found = False
 118         for c in contents["items"]:
 119             if c["kind"] == "arvados#collection" and check_workflow_content(c["portable_data_hash"]):
 120                 found = True
 121         if not found:
 122             raise Exception("Couldn't find collection containing expected "+expect_file)
 123
 124         # Updating by default will copy missing items
 125         cmd = ["arvados-cwl-runner", "--disable-git", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]
 126         print(" ".join(cmd))
 127         wf_uuid = subprocess.check_output(cmd)
 128         wf_uuid = wf_uuid.decode("utf-8").strip()
 129         check_contents(group, wf_uuid)
 130
 131     finally:
 132         api.groups().delete(uuid=group["uuid"]).execute()
 133
 134
 135 def check_execute():
 136     group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
 137     try:
 138         contents = api.groups().contents(uuid=group["uuid"]).execute()
 139         if len(contents["items"]) != 0:
 140             raise Exception("Expected 0 items")
 141
 142         # Execute workflow, shouldn't copy anything.
 143         cmd = ["arvados-cwl-runner", "--disable-git", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
 144         print(" ".join(cmd))
 145         wf_uuid = subprocess.check_output(cmd)
 146         wf_uuid = wf_uuid.decode("utf-8").strip()
 147
 148         contents = api.groups().contents(uuid=group["uuid"]).execute()
 149         # container request
 150         # final output collection
 151         # container log
 152         # step output collection
 153         # container request log
 154         if len(contents["items"]) != 5:
 155             raise Exception("Expected 5 items")
 156
 157         found = False
 158         for c in contents["items"]:
 159             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 160                 found = True
 161         if found:
 162             raise Exception("Didn't expect to find collection dependency")
 163
 164         found = False
 165         for c in contents["items"]:
 166             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 167                 found = True
 168         if found:
 169             raise Exception("Didn't expect to find jobs image dependency")
 170
 171         # Execute workflow with --copy-deps
 172         cmd = ["arvados-cwl-runner", "--disable-git", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]
 173         print(" ".join(cmd))
 174         wf_uuid = subprocess.check_output(cmd)
 175         wf_uuid = wf_uuid.decode("utf-8").strip()
 176
 177         contents = api.groups().contents(uuid=group["uuid"]).execute()
 178         found = False
 179         for c in contents["items"]:
 180             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 181                 found = True
 182         if not found:
 183             raise Exception("Couldn't find collection dependency")
 184
 185         found = False
 186         for c in contents["items"]:
 187             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 188                 found = True
 189         if not found:
 190             raise Exception("Couldn't find jobs image dependency")
 191
 192     finally:
 193         api.groups().delete(uuid=group["uuid"]).execute()
 194
 195 if __name__ == '__main__':
 196     check_create()
 197     check_update()
 198     check_execute()