sdk/cwl/tests/federation/main.cwl

   1 #!/usr/bin/env cwl-runner
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: Apache-2.0
   5
   6 cwlVersion: v1.0
   7 class: Workflow
   8 $namespaces:
   9   arv: "http://arvados.org/cwl#"
  10   cwltool: "http://commonwl.org/cwltool#"
  11 hints:
  12   cwltool:Secrets:
  13     secrets: [arvados_api_token]
  14 requirements:
  15   StepInputExpressionRequirement: {}
  16   InlineJavascriptRequirement: {}
  17   SubworkflowFeatureRequirement: {}
  18 inputs:
  19   arvados_api_token: string
  20   arvado_api_host_insecure:
  21     type: boolean
  22     default: false
  23   arvados_api_hosts: string[]
  24   arvados_cluster_ids: string[]
  25   acr: string?
  26   testcases:
  27     type: string[]
  28     default:
  29       - base-case
  30       - runner-home-step-remote
  31       - runner-remote-step-home
  32 outputs:
  33   base-case-success:
  34     type: Any
  35     outputSource: base-case/success
  36   runner-home-step-remote-success:
  37     type: Any
  38     outputSource: runner-home-step-remote/success
  39   runner-remote-step-home-success:
  40     type: Any
  41     outputSource: runner-remote-step-home/success
  42   remote-case-success:
  43     type: Any
  44     outputSource: remote-case/success
  45   twostep-home-to-remote-success:
  46     type: Any
  47     outputSource: twostep-home-to-remote/success
  48   twostep-remote-to-home-success:
  49     type: Any
  50     outputSource: twostep-remote-to-home/success
  51   twostep-both-remote-success:
  52     type: Any
  53     outputSource: twostep-both-remote/success
  54
  55 steps:
  56   base-case:
  57     doc: |
  58       Base case (no federation), single step workflow with both the
  59       runner and step on the same cluster.
  60     in:
  61       arvados_api_token: arvados_api_token
  62       arvado_api_host_insecure: arvado_api_host_insecure
  63       arvados_api_hosts: arvados_api_hosts
  64       arvados_cluster_ids: arvados_cluster_ids
  65       acr: acr
  66       wf:
  67         default:
  68           class: File
  69           location: cases/base-case.cwl
  70           secondaryFiles:
  71             - class: File
  72               location: cases/md5sum.cwl
  73       obj:
  74         default:
  75           inp:
  76             class: File
  77             location: data/base-case-input.txt
  78         valueFrom: |-
  79           ${
  80           self["runOnCluster"] = inputs.arvados_cluster_ids[0];
  81           return self;
  82           }
  83       scrub_image: {default: "arvados/fed-test:base-case"}
  84       scrub_collections:
  85         default:
  86           - 031a4ced0aa99de90fb630568afc6e9b+67   # input collection
  87           - eb93a6718eb1a1a8ee9f66ee7d683472+51   # md5sum output collection
  88           - f654d4048612135f4a5e7707ec0fcf3e+112  # final output json
  89     out: [out, success]
  90     run: framework/testcase.cwl
  91
  92   runner-home-step-remote:
  93     doc: |
  94       Single step workflow with the runner on the home cluster and the
  95       step on the remote cluster.
  96     in:
  97       arvados_api_token: arvados_api_token
  98       arvado_api_host_insecure: arvado_api_host_insecure
  99       arvados_api_hosts: arvados_api_hosts
 100       arvados_cluster_ids: arvados_cluster_ids
 101       acr: acr
 102       wf:
 103         default:
 104           class: File
 105           location: cases/runner-home-step-remote.cwl
 106           secondaryFiles:
 107             - class: File
 108               location: cases/md5sum.cwl
 109       obj:
 110         default:
 111           inp:
 112             class: File
 113             location: data/runner-home-step-remote-input.txt
 114         valueFrom: |-
 115           ${
 116           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
 117           return self;
 118           }
 119       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 120       scrub_image: {default: "arvados/fed-test:runner-home-step-remote"}
 121       scrub_collections:
 122         default:
 123           - 3bc373e38751fe13dcbd62778d583242+81   # input collection
 124           - 428e6d91e41a3af3ae287b453949e7fd+51   # md5sum output collection
 125           - a4b0ddd866525655e8480f83a1ca83c6+112  # runner output json
 126     out: [out, success]
 127     run: framework/testcase.cwl
 128
 129   runner-remote-step-home:
 130     doc: |
 131       Single step workflow with the runner on the remote cluster and the
 132       step on the home cluster.
 133     in:
 134       arvados_api_token: arvados_api_token
 135       arvado_api_host_insecure: arvado_api_host_insecure
 136       arvados_api_hosts: arvados_api_hosts
 137       arvados_cluster_ids: arvados_cluster_ids
 138       acr: acr
 139       wf:
 140         default:
 141           class: File
 142           location: cases/runner-remote-step-home.cwl
 143           secondaryFiles:
 144             - class: File
 145               location: cases/md5sum.cwl
 146       obj:
 147         default:
 148           inp:
 149             class: File
 150             location: data/runner-remote-step-home-input.txt
 151         valueFrom: |-
 152           ${
 153           self["runOnCluster"] = inputs.arvados_cluster_ids[0];
 154           return self;
 155           }
 156       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
 157       scrub_image: {default: "arvados/fed-test:runner-remote-step-home"}
 158       scrub_collections:
 159         default:
 160           - 25fe10d8e8530329a738de69d9bc8ab5+81   # input collection
 161           - 7f052d1a04b851b6f73fba77c7802e1d+51   # md5sum output collection
 162           - ecb639201f454b6493757f5117f540df+112  # runner output json
 163     out: [out, success]
 164     run: framework/testcase.cwl
 165
 166   remote-case:
 167     doc: |
 168       Single step workflow with both the runner and the step on the
 169       remote cluster.
 170     in:
 171       arvados_api_token: arvados_api_token
 172       arvado_api_host_insecure: arvado_api_host_insecure
 173       arvados_api_hosts: arvados_api_hosts
 174       arvados_cluster_ids: arvados_cluster_ids
 175       acr: acr
 176       wf:
 177         default:
 178           class: File
 179           location: cases/remote-case.cwl
 180           secondaryFiles:
 181             - class: File
 182               location: cases/md5sum.cwl
 183       obj:
 184         default:
 185           inp:
 186             class: File
 187             location: data/remote-case-input.txt
 188         valueFrom: |-
 189           ${
 190           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
 191           return self;
 192           }
 193       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
 194       scrub_image: {default: "arvados/fed-test:remote-case"}
 195       scrub_collections:
 196         default:
 197           - fccd49fdef8e452295f718208abafd88+69   # input collection
 198           - 58c0e8ea6b148134ef8577ee11307eec+51   # md5sum output collection
 199           - 1fd679c5ab64c123b9764024dbf560f0+112  # final output json
 200     out: [out, success]
 201     run: framework/testcase.cwl
 202
 203   twostep-home-to-remote:
 204     doc: |
 205       Two step workflow.  The runner is on the home cluster, the first
 206       step is on the home cluster, the second step is on the remote
 207       cluster.
 208     in:
 209       arvados_api_token: arvados_api_token
 210       arvado_api_host_insecure: arvado_api_host_insecure
 211       arvados_api_hosts: arvados_api_hosts
 212       arvados_cluster_ids: arvados_cluster_ids
 213       acr: acr
 214       wf:
 215         default:
 216           class: File
 217           location: cases/twostep-home-to-remote.cwl
 218           secondaryFiles:
 219             - class: File
 220               location: cases/md5sum.cwl
 221             - class: File
 222               location: cases/rev.cwl
 223       obj:
 224         default:
 225           inp:
 226             class: File
 227             location: data/twostep-home-to-remote.txt
 228         valueFrom: |-
 229           ${
 230           self["md5sumCluster"] = inputs.arvados_cluster_ids[0];
 231           self["revCluster"] = inputs.arvados_cluster_ids[1];
 232           return self;
 233           }
 234       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 235       scrub_image: {default: "arvados/fed-test:twostep-home-to-remote"}
 236       scrub_collections:
 237         default:
 238           - 268a54947fb75115cfe05bb54cc62c30+74   # input collection
 239           - 400f03b8c5d2dc3dcb513a21b626ef88+51   # md5sum output collection
 240           - 3738166916ca5f6f6ad12bf7e06b4a21+51   # rev output collection
 241           - bc37c17a37aa25229e5de1339b27fbcc+112  # runner output json
 242     out: [out, success]
 243     run: framework/testcase.cwl
 244
 245   twostep-remote-to-home:
 246     doc: |
 247       Two step workflow.  The runner is on the home cluster, the first
 248       step is on the remote cluster, the second step is on the home
 249       cluster.
 250     in:
 251       arvados_api_token: arvados_api_token
 252       arvado_api_host_insecure: arvado_api_host_insecure
 253       arvados_api_hosts: arvados_api_hosts
 254       arvados_cluster_ids: arvados_cluster_ids
 255       acr: acr
 256       wf:
 257         default:
 258           class: File
 259           location: cases/twostep-remote-to-home.cwl
 260           secondaryFiles:
 261             - class: File
 262               location: cases/md5sum.cwl
 263             - class: File
 264               location: cases/rev.cwl
 265       obj:
 266         default:
 267           inp:
 268             class: File
 269             location: data/twostep-remote-to-home.txt
 270         valueFrom: |-
 271           ${
 272           self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
 273           self["revCluster"] = inputs.arvados_cluster_ids[0];
 274           return self;
 275           }
 276       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 277       scrub_image: {default: "arvados/fed-test:twostep-remote-to-home"}
 278       scrub_collections:
 279         default:
 280           - cce89b9f7b6e163978144051ce5f071a+74   # input collection
 281           - 0c358c3af63644c6343766feff1b7238+51   # md5sum output collection
 282           - 33fb7d512bf21f04847eca58cea46e74+51   # rev output collection
 283           - 912e04aa3db04aba008cf5cd46c277b2+112  # runner output json
 284     out: [out, success]
 285     run: framework/testcase.cwl
 286
 287   twostep-both-remote:
 288     doc: |
 289       Two step workflow.  The runner is on the home cluster, both steps are
 290       on the remote cluster.
 291     in:
 292       arvados_api_token: arvados_api_token
 293       arvado_api_host_insecure: arvado_api_host_insecure
 294       arvados_api_hosts: arvados_api_hosts
 295       arvados_cluster_ids: arvados_cluster_ids
 296       acr: acr
 297       wf:
 298         default:
 299           class: File
 300           location: cases/twostep-both-remote.cwl
 301           secondaryFiles:
 302             - class: File
 303               location: cases/md5sum.cwl
 304             - class: File
 305               location: cases/rev.cwl
 306       obj:
 307         default:
 308           inp:
 309             class: File
 310             location: data/twostep-both-remote.txt
 311         valueFrom: |-
 312           ${
 313           self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
 314           self["revCluster"] = inputs.arvados_cluster_ids[1];
 315           return self;
 316           }
 317       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 318       scrub_image: {default: "arvados/fed-test:twostep-both-remote"}
 319       scrub_collections:
 320         default:
 321           - 3c5e39939cf197d304ac1eac20841238+71   # input collection
 322           - 3edb99aa607731593969cdab663d65b4+51   # md5sum output collection
 323           - a91625b7139e60fe61a88cae42fbee13+51   # rev output collection
 324           - ddfa58a81953dad08436d571615dd584+112  # runner output json
 325     out: [out, success]
 326     run: framework/testcase.cwl
 327
 328   twostep-remote-copy-to-home:
 329     doc: |
 330       Two step workflow.  The runner is on the home cluster, the first
 331       step is on the remote cluster, the second step is on the home
 332       cluster, and propagates its input file directly from input to
 333       output by symlinking the input file in the output directory.
 334       Tests that crunch-run will copy blocks from remote to local
 335       when preparing output collection.
 336     in:
 337       arvados_api_token: arvados_api_token
 338       arvado_api_host_insecure: arvado_api_host_insecure
 339       arvados_api_hosts: arvados_api_hosts
 340       arvados_cluster_ids: arvados_cluster_ids
 341       acr: acr
 342       wf:
 343         default:
 344           class: File
 345           location: cases/twostep-remote-copy-to-home.cwl
 346           secondaryFiles:
 347             - class: File
 348               location: cases/md5sum.cwl
 349             - class: File
 350               location: cases/rev-input-to-output.cwl
 351       obj:
 352         default:
 353           inp:
 354             class: File
 355             location: data/twostep-remote-copy-to-home.txt
 356         valueFrom: |-
 357           ${
 358           self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
 359           self["revCluster"] = inputs.arvados_cluster_ids[0];
 360           return self;
 361           }
 362       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 363       scrub_image: {default: "arvados/fed-test:twostep-remote-copy-to-home"}
 364       scrub_collections:
 365         default:
 366           - 538887bc29a3098bf79abdb8536d17bd+79   # input collection
 367           - 14da0e0d52d7ab2945427074b275e9ee+51   # md5sum output collection
 368           - 2d3a4a840077390a0d7788f169eaba89+112  # rev output collection
 369           - 2d3a4a840077390a0d7788f169eaba89+112  # runner output json
 370     out: [out, success]
 371     run: framework/testcase.cwl