sdk/python/tests/fed-migrate/fed-migrate.cwl

   1 #!/usr/bin/env cwl-runner
   2 $graph:
   3   - class: Workflow
   4     cwlVersion: v1.0
   5     id: '#run_test'
   6     inputs:
   7       - id: arvados_api_hosts
   8         type:
   9           items: string
  10           type: array
  11       - id: superuser_tokens
  12         type:
  13           items: string
  14           type: array
  15       - default: arv-federation-migrate
  16         id: fed_migrate
  17         type: string
  18     outputs:
  19       - id: report3
  20         outputSource: main_2/report3
  21         type: File
  22     requirements:
  23       InlineJavascriptRequirement: {}
  24       MultipleInputFeatureRequirement: {}
  25       ScatterFeatureRequirement: {}
  26       StepInputExpressionRequirement: {}
  27       SubworkflowFeatureRequirement: {}
  28     steps:
  29       - id: main_1
  30         in:
  31           arvados_api_hosts:
  32             source: arvados_api_hosts
  33           create_users:
  34             default:
  35               class: File
  36               location: create_users.py
  37           superuser_tokens:
  38             source: superuser_tokens
  39         out:
  40           - ar
  41         run:
  42           arguments:
  43             - python
  44             - $(inputs.create_users)
  45             - _script
  46           class: CommandLineTool
  47           id: main_1_embed
  48           inputs:
  49             - id: arvados_api_hosts
  50               type:
  51                 items: string
  52                 type: array
  53             - id: superuser_tokens
  54               type:
  55                 items: string
  56                 type: array
  57             - id: create_users
  58               type: File
  59           outputs:
  60             - id: ar
  61               outputBinding:
  62                 outputEval: $(inputs.arvados_api_hosts)
  63               type:
  64                 items: string
  65                 type: array
  66           requirements:
  67             InitialWorkDirRequirement:
  68               listing:
  69                 - entry: |
  70                     {
  71                       "arvados_api_hosts": $(inputs.arvados_api_hosts),
  72                       "superuser_tokens": $(inputs.superuser_tokens)
  73                     }
  74                   entryname: _script
  75             InlineJavascriptRequirement: {}
  76       - id: main_2
  77         in:
  78           ar:
  79             source: main_1/ar
  80           arvados_api_hosts:
  81             source: arvados_api_hosts
  82           fed_migrate:
  83             source: fed_migrate
  84           host:
  85             valueFrom: '$(inputs.arvados_api_hosts[0])'
  86           superuser_tokens:
  87             source: superuser_tokens
  88           token:
  89             valueFrom: '$(inputs.superuser_tokens[0])'
  90         out:
  91           - report
  92           - report2
  93           - report3
  94           - r
  95         run:
  96           class: Workflow
  97           id: main_2_embed
  98           inputs:
  99             - id: ar
 100               type:
 101                 items: string
 102                 type: array
 103             - id: arvados_api_hosts
 104               type:
 105                 items: string
 106                 type: array
 107             - id: superuser_tokens
 108               type:
 109                 items: string
 110                 type: array
 111             - id: fed_migrate
 112               type: string
 113             - id: host
 114               type: Any
 115             - id: token
 116               type: Any
 117           outputs:
 118             - id: report
 119               outputSource: main_2_embed_1/report
 120               type: File
 121             - id: report2
 122               outputSource: main_2_embed_2/report2
 123               type: File
 124             - id: report3
 125               outputSource: main_2_embed_3/report3
 126               type: File
 127             - id: r
 128               outputSource: main_2_embed_4/r
 129               type: File
 130           requirements:
 131             - class: EnvVarRequirement
 132               envDef:
 133                 ARVADOS_API_HOST: $(inputs.host)
 134                 ARVADOS_API_HOST_INSECURE: '1'
 135                 ARVADOS_API_TOKEN: $(inputs.token)
 136           steps:
 137             - id: main_2_embed_1
 138               in:
 139                 fed_migrate:
 140                   source: fed_migrate
 141                 host:
 142                   source: host
 143                 token:
 144                   source: token
 145               out:
 146                 - report
 147               run:
 148                 arguments:
 149                   - $(inputs.fed_migrate)
 150                   - '--report'
 151                   - report.csv
 152                 class: CommandLineTool
 153                 id: main_2_embed_1_embed
 154                 inputs:
 155                   - id: fed_migrate
 156                     type: string
 157                   - id: host
 158                     type: Any
 159                   - id: token
 160                     type: Any
 161                 outputs:
 162                   - id: report
 163                     outputBinding:
 164                       glob: report.csv
 165                     type: File
 166                 requirements:
 167                   InlineJavascriptRequirement: {}
 168             - id: main_2_embed_2
 169               in:
 170                 host:
 171                   source: host
 172                 report:
 173                   source: main_2_embed_1/report
 174                 token:
 175                   source: token
 176               out:
 177                 - report2
 178               run:
 179                 arguments:
 180                   - sed
 181                   - '-E'
 182                   - 's/,(case[1-8])2?,/,\1,/g'
 183                 class: CommandLineTool
 184                 id: main_2_embed_2_embed
 185                 inputs:
 186                   - id: report
 187                     type: File
 188                   - id: host
 189                     type: Any
 190                   - id: token
 191                     type: Any
 192                 outputs:
 193                   - id: report2
 194                     outputBinding:
 195                       glob: report.csv
 196                     type: File
 197                 requirements:
 198                   InlineJavascriptRequirement: {}
 199                 stdin: $(inputs.report.path)
 200                 stdout: report.csv
 201             - id: main_2_embed_3
 202               in:
 203                 fed_migrate:
 204                   source: fed_migrate
 205                 host:
 206                   source: host
 207                 report2:
 208                   source: main_2_embed_2/report2
 209                 token:
 210                   source: token
 211               out:
 212                 - report3
 213               run:
 214                 arguments:
 215                   - $(inputs.fed_migrate)
 216                   - '--migrate'
 217                   - $(inputs.report2)
 218                 class: CommandLineTool
 219                 id: main_2_embed_3_embed
 220                 inputs:
 221                   - id: report2
 222                     type: File
 223                   - id: fed_migrate
 224                     type: string
 225                   - id: host
 226                     type: Any
 227                   - id: token
 228                     type: Any
 229                 outputs:
 230                   - id: report3
 231                     outputBinding:
 232                       outputEval: $(inputs.report2)
 233                     type: File
 234                 requirements:
 235                   InlineJavascriptRequirement: {}
 236             - id: main_2_embed_4
 237               in:
 238                 arvados_api_hosts:
 239                   source: arvados_api_hosts
 240                 check:
 241                   default:
 242                     class: File
 243                     location: check.py
 244                 host:
 245                   source: host
 246                 report3:
 247                   source: main_2_embed_3/report3
 248                 superuser_tokens:
 249                   source: superuser_tokens
 250                 token:
 251                   source: token
 252               out:
 253                 - r
 254               run:
 255                 arguments:
 256                   - python
 257                   - $(inputs.check)
 258                   - _script
 259                 class: CommandLineTool
 260                 id: main_2_embed_4_embed
 261                 inputs:
 262                   - id: report3
 263                     type: File
 264                   - id: host
 265                     type: Any
 266                   - id: token
 267                     type: Any
 268                   - id: arvados_api_hosts
 269                     type:
 270                       items: string
 271                       type: array
 272                   - id: superuser_tokens
 273                     type:
 274                       items: string
 275                       type: array
 276                   - id: check
 277                     type: File
 278                 outputs:
 279                   - id: r
 280                     outputBinding:
 281                       outputEval: $(inputs.report3)
 282                     type: File
 283                 requirements:
 284                   InitialWorkDirRequirement:
 285                     listing:
 286                       - entry: |
 287                           {
 288                             "arvados_api_hosts": $(inputs.arvados_api_hosts),
 289                             "superuser_tokens": $(inputs.superuser_tokens)
 290                           }
 291                         entryname: _script
 292                   InlineJavascriptRequirement: {}
 293   - arguments:
 294       - arvbox
 295       - cat
 296       - /var/lib/arvados/superuser_token
 297     class: CommandLineTool
 298     cwlVersion: v1.0
 299     id: '#superuser_tok'
 300     inputs:
 301       - id: container
 302         type: string
 303     outputs:
 304       - id: superuser_token
 305         outputBinding:
 306           glob: superuser_token.txt
 307           loadContents: true
 308           outputEval: '$(self[0].contents.trim())'
 309         type: string
 310     requirements:
 311       EnvVarRequirement:
 312         envDef:
 313           ARVBOX_CONTAINER: $(inputs.container)
 314       InlineJavascriptRequirement: {}
 315     stdout: superuser_token.txt
 316   - class: Workflow
 317     id: '#main'
 318     inputs:
 319       - id: arvados_api_hosts
 320         type:
 321           items: string
 322           type: array
 323       - id: arvados_cluster_ids
 324         type:
 325           items: string
 326           type: array
 327       - id: superuser_tokens
 328         type:
 329           items: string
 330           type: array
 331       - id: arvbox_containers
 332         type:
 333           items: string
 334           type: array
 335       - default: arv-federation-migrate
 336         id: fed_migrate
 337         type: string
 338       - id: arvbox_bin
 339         type: File
 340       - default: master
 341         id: refspec
 342         type: string
 343     outputs:
 344       - id: supertok
 345         outputSource: main_2/supertok
 346         type:
 347           items: string
 348           type: array
 349       - id: report
 350         outputSource: run_test_3/report3
 351         type: File
 352     requirements:
 353       InlineJavascriptRequirement: {}
 354       MultipleInputFeatureRequirement: {}
 355       ScatterFeatureRequirement: {}
 356       StepInputExpressionRequirement: {}
 357       SubworkflowFeatureRequirement: {}
 358     steps:
 359       - id: main_1
 360         in:
 361           arvados_cluster_ids:
 362             source: arvados_cluster_ids
 363         out:
 364           - logincluster
 365         run:
 366           class: ExpressionTool
 367           expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
 368           inputs:
 369             - id: arvados_cluster_ids
 370               type:
 371                 items: string
 372                 type: array
 373           outputs:
 374             - id: logincluster
 375               type: string
 376       - id: main_2
 377         in:
 378           arvbox_bin:
 379             source: arvbox_bin
 380           cluster_id:
 381             source: arvados_cluster_ids
 382           container:
 383             source: arvbox_containers
 384           host:
 385             source: arvados_api_hosts
 386           logincluster:
 387             source: main_1/logincluster
 388           refspec:
 389             source: refspec
 390         out:
 391           - supertok
 392         run:
 393           class: Workflow
 394           id: main_2_embed
 395           inputs:
 396             - id: container
 397               type: string
 398             - id: cluster_id
 399               type: string
 400             - id: host
 401               type: string
 402             - id: logincluster
 403               type: string
 404             - id: arvbox_bin
 405               type: File
 406             - id: refspec
 407               type: string
 408           outputs:
 409             - id: supertok
 410               outputSource: superuser_tok_2/superuser_token
 411               type: string
 412           requirements:
 413             InlineJavascriptRequirement: {}
 414           steps:
 415             - id: main_2_embed_1
 416               in:
 417                 arvbox_bin:
 418                   source: arvbox_bin
 419                 container:
 420                   source: container
 421                 host:
 422                   source: host
 423                 refspec:
 424                   source: refspec
 425               out:
 426                 - d
 427               run:
 428                 arguments:
 429                   - sh
 430                   - _script
 431                 class: CommandLineTool
 432                 id: main_2_embed_1_embed
 433                 inputs:
 434                   - id: container
 435                     type: string
 436                   - id: host
 437                     type: string
 438                   - id: arvbox_bin
 439                     type: File
 440                   - id: refspec
 441                     type: string
 442                 outputs:
 443                   - id: d
 444                     outputBinding:
 445                       outputEval: $(inputs.container)
 446                     type: string
 447                 requirements:
 448                   InitialWorkDirRequirement:
 449                     listing:
 450                       - entry: >+
 451                           set -xe
 452
 453                           export ARVBOX_CONTAINER="$(inputs.container)"
 454
 455                           $(inputs.arvbox_bin.path) pipe <<EOF
 456
 457                           cd /usr/src/arvados
 458
 459                           git fetch
 460
 461                           git checkout -f $(inputs.refspec)
 462
 463                           git pull
 464
 465                           chown arvbox:arvbox -R .
 466
 467                           EOF
 468
 469
 470                           $(inputs.arvbox_bin.path) hotreset
 471
 472
 473                           while ! curl --fail --insecure --silent
 474                           https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
 475                           >/dev/null ; do sleep 3 ; done
 476
 477
 478                           ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path)
 479                           cat /var/lib/arvados/vm-uuid)
 480
 481                           ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat
 482                           /var/lib/arvados/superuser_token)
 483
 484                           while ! curl --fail --insecure --silent -H
 485                           "Authorization: Bearer $ARVADOS_API_TOKEN"
 486                           https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID
 487                           >/dev/null ; do sleep 3 ; done
 488
 489                         entryname: _script
 490                   InlineJavascriptRequirement: {}
 491             - id: superuser_tok_2
 492               in:
 493                 container:
 494                   source: container
 495                 d:
 496                   source: main_2_embed_1/d
 497               out:
 498                 - superuser_token
 499               run: '#superuser_tok'
 500         scatter:
 501           - container
 502           - cluster_id
 503           - host
 504         scatterMethod: dotproduct
 505       - id: run_test_3
 506         in:
 507           arvados_api_hosts:
 508             source: arvados_api_hosts
 509           fed_migrate:
 510             source: fed_migrate
 511           superuser_tokens:
 512             source: main_2/supertok
 513         out:
 514           - report3
 515         run: '#run_test'
 516 cwlVersion: v1.0
 517