sdk/python/tests/fed-migrate/fed-migrate.cwl

   1 #!/usr/bin/env cwl-runner
   2 $graph:
   3   - class: Workflow
   4     cwlVersion: v1.0
   5     id: '#run_test'
   6     inputs:
   7       - id: arvados_api_hosts
   8         type:
   9           items: string
  10           type: array
  11       - id: superuser_tokens
  12         type:
  13           items: string
  14           type: array
  15       - default: arv-federation-migrate
  16         id: fed_migrate
  17         type: string
  18     outputs:
  19       - id: report
  20         outputSource: main_2/report
  21         type: File
  22     requirements:
  23       InlineJavascriptRequirement: {}
  24       MultipleInputFeatureRequirement: {}
  25       ScatterFeatureRequirement: {}
  26       StepInputExpressionRequirement: {}
  27       SubworkflowFeatureRequirement: {}
  28     steps:
  29       - id: main_1
  30         in:
  31           arvados_api_hosts:
  32             source: arvados_api_hosts
  33           create_users:
  34             default:
  35               class: File
  36               location: create_users.py
  37           superuser_tokens:
  38             source: superuser_tokens
  39         out:
  40           - ar
  41         run:
  42           arguments:
  43             - python
  44             - $(inputs.create_users)
  45             - _script
  46           class: CommandLineTool
  47           id: main_1_embed
  48           inputs:
  49             - id: arvados_api_hosts
  50               type:
  51                 items: string
  52                 type: array
  53             - id: superuser_tokens
  54               type:
  55                 items: string
  56                 type: array
  57             - id: create_users
  58               type: File
  59           outputs:
  60             - id: ar
  61               outputBinding:
  62                 outputEval: $(inputs.arvados_api_hosts)
  63               type:
  64                 items: string
  65                 type: array
  66           requirements:
  67             InitialWorkDirRequirement:
  68               listing:
  69                 - entry: |
  70                     {
  71                       "arvados_api_hosts": $(inputs.arvados_api_hosts),
  72                       "superuser_tokens": $(inputs.superuser_tokens)
  73                     }
  74                   entryname: _script
  75             InlineJavascriptRequirement: {}
  76       - id: main_2
  77         in:
  78           ar:
  79             source: main_1/ar
  80           arvados_api_hosts:
  81             source: arvados_api_hosts
  82           fed_migrate:
  83             source: fed_migrate
  84           host:
  85             valueFrom: '$(inputs.arvados_api_hosts[0])'
  86           superuser_tokens:
  87             source: superuser_tokens
  88           token:
  89             valueFrom: '$(inputs.superuser_tokens[0])'
  90         out:
  91           - report
  92         run:
  93           arguments:
  94             - $(inputs.fed_migrate)
  95             - '--report'
  96             - report.csv
  97           class: CommandLineTool
  98           id: main_2_embed
  99           inputs:
 100             - id: arvados_api_hosts
 101               type:
 102                 items: string
 103                 type: array
 104             - id: superuser_tokens
 105               type:
 106                 items: string
 107                 type: array
 108             - id: ar
 109               type:
 110                 items: string
 111                 type: array
 112             - id: fed_migrate
 113               type: string
 114             - id: host
 115               type: Any
 116             - id: token
 117               type: Any
 118           outputs:
 119             - id: report
 120               outputBinding:
 121                 glob: report.csv
 122               type: File
 123           requirements:
 124             - class: EnvVarRequirement
 125               envDef:
 126                 ARVADOS_API_HOST: $(inputs.host)
 127                 ARVADOS_API_TOKEN: $(inputs.token)
 128   - arguments:
 129       - arvbox
 130       - cat
 131       - /var/lib/arvados/superuser_token
 132     class: CommandLineTool
 133     cwlVersion: v1.0
 134     id: '#superuser_tok'
 135     inputs:
 136       - id: container
 137         type: string
 138     outputs:
 139       - id: superuser_token
 140         outputBinding:
 141           glob: superuser_token.txt
 142           loadContents: true
 143           outputEval: '$(self[0].contents.trim())'
 144         type: string
 145     requirements:
 146       EnvVarRequirement:
 147         envDef:
 148           ARVBOX_CONTAINER: $(inputs.container)
 149       InlineJavascriptRequirement: {}
 150     stdout: superuser_token.txt
 151   - class: Workflow
 152     id: '#main'
 153     inputs:
 154       - id: arvados_api_hosts
 155         type:
 156           items: string
 157           type: array
 158       - id: arvados_cluster_ids
 159         type:
 160           items: string
 161           type: array
 162       - id: superuser_tokens
 163         type:
 164           items: string
 165           type: array
 166       - id: arvbox_containers
 167         type:
 168           items: string
 169           type: array
 170       - default: arv-federation-migrate
 171         id: fed_migrate
 172         type: string
 173     outputs:
 174       - id: supertok
 175         outputSource: main_2/supertok
 176         type:
 177           items: string
 178           type: array
 179       - id: report
 180         outputSource: run_test_3/report
 181         type: File
 182     requirements:
 183       InlineJavascriptRequirement: {}
 184       MultipleInputFeatureRequirement: {}
 185       ScatterFeatureRequirement: {}
 186       StepInputExpressionRequirement: {}
 187       SubworkflowFeatureRequirement: {}
 188     steps:
 189       - id: main_1
 190         in:
 191           arvados_cluster_ids:
 192             source: arvados_cluster_ids
 193         out:
 194           - logincluster
 195         run:
 196           class: ExpressionTool
 197           expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
 198           inputs:
 199             - id: arvados_cluster_ids
 200               type:
 201                 items: string
 202                 type: array
 203           outputs:
 204             - id: logincluster
 205               type: string
 206       - id: main_2
 207         in:
 208           cluster_id:
 209             source: arvados_cluster_ids
 210           container:
 211             source: arvbox_containers
 212           host:
 213             source: arvados_api_hosts
 214           logincluster:
 215             source: main_1/logincluster
 216         out:
 217           - supertok
 218         run:
 219           class: Workflow
 220           id: main_2_embed
 221           inputs:
 222             - id: container
 223               type: string
 224             - id: cluster_id
 225               type: string
 226             - id: host
 227               type: string
 228             - id: logincluster
 229               type: string
 230           outputs:
 231             - id: supertok
 232               outputSource: superuser_tok_3/superuser_token
 233               type: string
 234           requirements:
 235             - class: EnvVarRequirement
 236               envDef:
 237                 ARVBOX_CONTAINER: $(inputs.container)
 238           steps:
 239             - id: main_2_embed_1
 240               in:
 241                 cluster_id:
 242                   source: cluster_id
 243                 container:
 244                   source: container
 245                 logincluster:
 246                   source: logincluster
 247                 set_login:
 248                   default:
 249                     class: File
 250                     location: set_login.py
 251               out:
 252                 - c
 253               run:
 254                 arguments:
 255                   - sh
 256                   - _script
 257                 class: CommandLineTool
 258                 id: main_2_embed_1_embed
 259                 inputs:
 260                   - id: container
 261                     type: string
 262                   - id: cluster_id
 263                     type: string
 264                   - id: logincluster
 265                     type: string
 266                   - id: set_login
 267                     type: File
 268                 outputs:
 269                   - id: c
 270                     outputBinding:
 271                       outputEval: $(inputs.container)
 272                     type: string
 273                 requirements:
 274                   InitialWorkDirRequirement:
 275                     listing:
 276                       - entry: >
 277                           set -x
 278
 279                           docker cp
 280                           $(inputs.container):/var/lib/arvados/cluster_config.yml.override
 281                           .
 282
 283                           chmod +w cluster_config.yml.override
 284
 285                           python $(inputs.set_login.path)
 286                           cluster_config.yml.override $(inputs.cluster_id)
 287                           $(inputs.logincluster)
 288
 289                           docker cp cluster_config.yml.override
 290                           $(inputs.container):/var/lib/arvados
 291                         entryname: _script
 292                   InlineJavascriptRequirement: {}
 293             - id: main_2_embed_2
 294               in:
 295                 c:
 296                   source: main_2_embed_1/c
 297                 container:
 298                   source: container
 299                 host:
 300                   source: host
 301               out:
 302                 - d
 303               run:
 304                 arguments:
 305                   - sh
 306                   - _script
 307                 class: CommandLineTool
 308                 id: main_2_embed_2_embed
 309                 inputs:
 310                   - id: container
 311                     type: string
 312                   - id: host
 313                     type: string
 314                   - id: c
 315                     type: string
 316                 outputs:
 317                   - id: d
 318                     outputBinding:
 319                       outputEval: $(inputs.c)
 320                     type: string
 321                 requirements:
 322                   InitialWorkDirRequirement:
 323                     listing:
 324                       - entry: >
 325                           set -x
 326
 327                           arvbox hotreset
 328
 329                           while ! curl --fail --insecure --silent
 330                           https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
 331                           >/dev/null ; do sleep 3 ; done
 332
 333                           export ARVADOS_API_HOST=$(inputs.host)
 334
 335                           export ARVADOS_API_TOKEN=\$(arvbox cat
 336                           /var/lib/arvados/superuser_token)
 337
 338                           export ARVADOS_API_HOST_INSECURE=1
 339
 340                           ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat
 341                           /var/lib/arvados/vm-uuid)
 342
 343                           while ! python -c "import arvados ;
 344                           arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
 345                           2>/dev/null ; do sleep 3; done
 346                         entryname: _script
 347                   InlineJavascriptRequirement: {}
 348             - id: superuser_tok_3
 349               in:
 350                 container:
 351                   source: container
 352                 d:
 353                   source: main_2_embed_2/d
 354               out:
 355                 - superuser_token
 356               run: '#superuser_tok'
 357         scatter:
 358           - container
 359           - cluster_id
 360           - host
 361         scatterMethod: dotproduct
 362       - id: run_test_3
 363         in:
 364           arvados_api_hosts:
 365             source: arvados_api_hosts
 366           fed_migrate:
 367             source: fed_migrate
 368           superuser_tokens:
 369             source: main_2/supertok
 370         out:
 371           - report
 372         run: '#run_test'
 373 cwlVersion: v1.0
 374