sdk/cwl/tests/test_copy_deps.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import subprocess
   7
   8 api = arvados.api()
   9
  10 def check_contents(group, wf_uuid):
  11     contents = api.groups().contents(uuid=group["uuid"]).execute()
  12     if len(contents["items"]) != 4:
  13         raise Exception("Expected 4 items in "+group["uuid"]+" was "+str(len(contents["items"])))
  14
  15     found = False
  16     for c in contents["items"]:
  17         if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  18             found = True
  19     if not found:
  20         raise Exception("Couldn't find workflow in "+group["uuid"])
  21
  22     found = False
  23     for c in contents["items"]:
  24         if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
  25             found = True
  26     if not found:
  27         raise Exception("Couldn't find collection dependency")
  28
  29     found = False
  30     for c in contents["items"]:
  31         if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
  32             found = True
  33     if not found:
  34         raise Exception("Couldn't find jobs image dependency")
  35
  36     found = False
  37     for c in contents["items"]:
  38         if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "73b7a68c78205e793a5eb9520326d3ae+61":
  39             found = True
  40     if not found:
  41         raise Exception("Couldn't find collection containing workflow")
  42
  43
  44 def test_create():
  45     group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
  46     try:
  47         contents = api.groups().contents(uuid=group["uuid"]).execute()
  48         if len(contents["items"]) != 0:
  49             raise Exception("Expected 0 items")
  50
  51         # Create workflow, by default should also copy dependencies
  52         cmd = ["arvados-cwl-runner", "--disable-git", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
  53         print(" ".join(cmd))
  54         wf_uuid = subprocess.check_output(cmd)
  55         wf_uuid = wf_uuid.decode("utf-8").strip()
  56         check_contents(group, wf_uuid)
  57     finally:
  58         api.groups().delete(uuid=group["uuid"]).execute()
  59
  60
  61 def test_update():
  62     group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
  63     try:
  64         contents = api.groups().contents(uuid=group["uuid"]).execute()
  65         if len(contents["items"]) != 0:
  66             raise Exception("Expected 0 items")
  67
  68         # Create workflow, but with --no-copy-deps it shouldn't copy anything
  69         cmd = ["arvados-cwl-runner", "--disable-git", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
  70         print(" ".join(cmd))
  71         wf_uuid = subprocess.check_output(cmd)
  72         wf_uuid = wf_uuid.decode("utf-8").strip()
  73
  74         contents = api.groups().contents(uuid=group["uuid"]).execute()
  75         if len(contents["items"]) != 2:
  76             raise Exception("Expected 2 items")
  77
  78         found = False
  79         for c in contents["items"]:
  80             if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  81                 found = True
  82         if not found:
  83             raise Exception("Couldn't find workflow")
  84
  85         found = False
  86         for c in contents["items"]:
  87             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "73b7a68c78205e793a5eb9520326d3ae+61":
  88                 found = True
  89         if not found:
  90             raise Exception("Couldn't find collection containing workflow")
  91
  92         # Updating by default will copy missing items
  93         cmd = ["arvados-cwl-runner", "--disable-git", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]
  94         print(" ".join(cmd))
  95         wf_uuid = subprocess.check_output(cmd)
  96         wf_uuid = wf_uuid.decode("utf-8").strip()
  97         check_contents(group, wf_uuid)
  98
  99     finally:
 100         api.groups().delete(uuid=group["uuid"]).execute()
 101
 102
 103 def test_execute():
 104     group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
 105     try:
 106         contents = api.groups().contents(uuid=group["uuid"]).execute()
 107         if len(contents["items"]) != 0:
 108             raise Exception("Expected 0 items")
 109
 110         # Execute workflow, shouldn't copy anything.
 111         cmd = ["arvados-cwl-runner", "--disable-git", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
 112         print(" ".join(cmd))
 113         wf_uuid = subprocess.check_output(cmd)
 114         wf_uuid = wf_uuid.decode("utf-8").strip()
 115
 116         contents = api.groups().contents(uuid=group["uuid"]).execute()
 117         # container request
 118         # final output collection
 119         # container log
 120         # step output collection
 121         # container request log
 122         if len(contents["items"]) != 5:
 123             raise Exception("Expected 5 items")
 124
 125         found = False
 126         for c in contents["items"]:
 127             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 128                 found = True
 129         if found:
 130             raise Exception("Didn't expect to find collection dependency")
 131
 132         found = False
 133         for c in contents["items"]:
 134             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 135                 found = True
 136         if found:
 137             raise Exception("Didn't expect to find jobs image dependency")
 138
 139         # Execute workflow with --copy-deps
 140         cmd = ["arvados-cwl-runner", "--disable-git", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]
 141         print(" ".join(cmd))
 142         wf_uuid = subprocess.check_output(cmd)
 143         wf_uuid = wf_uuid.decode("utf-8").strip()
 144
 145         contents = api.groups().contents(uuid=group["uuid"]).execute()
 146         found = False
 147         for c in contents["items"]:
 148             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 149                 found = True
 150         if not found:
 151             raise Exception("Couldn't find collection dependency")
 152
 153         found = False
 154         for c in contents["items"]:
 155             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 156                 found = True
 157         if not found:
 158             raise Exception("Couldn't find jobs image dependency")
 159
 160     finally:
 161         api.groups().delete(uuid=group["uuid"]).execute()
 162
 163 if __name__ == '__main__':
 164     test_create()
 165     test_update()
 166     test_execute()