services/api/test/unit/container_test.rb

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 require 'test_helper'
   6
   7 class ContainerTest < ActiveSupport::TestCase
   8   include DbCurrentTime
   9
  10   DEFAULT_ATTRS = {
  11     command: ['echo', 'foo'],
  12     container_image: 'fa3c1a9cb6783f85f2ecda037e07b8c3+167',
  13     output_path: '/tmp',
  14     priority: 1,
  15     runtime_constraints: {"vcpus" => 1, "ram" => 1},
  16   }
  17
  18   REUSABLE_COMMON_ATTRS = {
  19     container_image: "9ae44d5792468c58bcf85ce7353c7027+124",
  20     cwd: "test",
  21     command: ["echo", "hello"],
  22     output_path: "test",
  23     runtime_constraints: {
  24       "ram" => 12000000000,
  25       "vcpus" => 4,
  26     },
  27     mounts: {
  28       "test" => {"kind" => "json"},
  29     },
  30     environment: {
  31       "var" => "val",
  32     },
  33   }
  34
  35   def minimal_new attrs={}
  36     cr = ContainerRequest.new DEFAULT_ATTRS.merge(attrs)
  37     cr.state = ContainerRequest::Committed
  38     act_as_user users(:active) do
  39       cr.save!
  40     end
  41     c = Container.find_by_uuid cr.container_uuid
  42     assert_not_nil c
  43     return c, cr
  44   end
  45
  46   def check_illegal_updates c, bad_updates
  47     bad_updates.each do |u|
  48       refute c.update_attributes(u), u.inspect
  49       refute c.valid?, u.inspect
  50       c.reload
  51     end
  52   end
  53
  54   def check_illegal_modify c
  55     check_illegal_updates c, [{command: ["echo", "bar"]},
  56                               {container_image: "arvados/apitestfixture:june10"},
  57                               {cwd: "/tmp2"},
  58                               {environment: {"FOO" => "BAR"}},
  59                               {mounts: {"FOO" => "BAR"}},
  60                               {output_path: "/tmp3"},
  61                               {locked_by_uuid: "zzzzz-gj3su-027z32aux8dg2s1"},
  62                               {auth_uuid: "zzzzz-gj3su-017z32aux8dg2s1"},
  63                               {runtime_constraints: {"FOO" => "BAR"}}]
  64   end
  65
  66   def check_bogus_states c
  67     check_illegal_updates c, [{state: nil},
  68                               {state: "Flubber"}]
  69   end
  70
  71   def check_no_change_from_cancelled c
  72     check_illegal_modify c
  73     check_bogus_states c
  74     check_illegal_updates c, [{ priority: 3 },
  75                               { state: Container::Queued },
  76                               { state: Container::Locked },
  77                               { state: Container::Running },
  78                               { state: Container::Complete }]
  79   end
  80
  81   test "Container create" do
  82     act_as_system_user do
  83       c, _ = minimal_new(environment: {},
  84                       mounts: {"BAR" => "FOO"},
  85                       output_path: "/tmp",
  86                       priority: 1,
  87                       runtime_constraints: {"vcpus" => 1, "ram" => 1})
  88
  89       check_illegal_modify c
  90       check_bogus_states c
  91
  92       c.reload
  93       c.priority = 2
  94       c.save!
  95     end
  96   end
  97
  98   test "Container serialized hash attributes sorted before save" do
  99     env = {"C" => 3, "B" => 2, "A" => 1}
 100     m = {"F" => {"kind" => 3}, "E" => {"kind" => 2}, "D" => {"kind" => 1}}
 101     rc = {"vcpus" => 1, "ram" => 1, "keep_cache_ram" => 1}
 102     c, _ = minimal_new(environment: env, mounts: m, runtime_constraints: rc)
 103     assert_equal c.environment.to_json, Container.deep_sort_hash(env).to_json
 104     assert_equal c.mounts.to_json, Container.deep_sort_hash(m).to_json
 105     assert_equal c.runtime_constraints.to_json, Container.deep_sort_hash(rc).to_json
 106   end
 107
 108   test 'deep_sort_hash on array of hashes' do
 109     a = {'z' => [[{'a' => 'a', 'b' => 'b'}]]}
 110     b = {'z' => [[{'b' => 'b', 'a' => 'a'}]]}
 111     assert_equal Container.deep_sort_hash(a).to_json, Container.deep_sort_hash(b).to_json
 112   end
 113
 114   test "find_reusable method should select higher priority queued container" do
 115     set_user_from_auth :active
 116     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment:{"var" => "queued"}})
 117     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:1}))
 118     c_high_priority, _ = minimal_new(common_attrs.merge({use_existing:false, priority:2}))
 119     assert_not_equal c_low_priority.uuid, c_high_priority.uuid
 120     assert_equal Container::Queued, c_low_priority.state
 121     assert_equal Container::Queued, c_high_priority.state
 122     reused = Container.find_reusable(common_attrs)
 123     assert_not_nil reused
 124     assert_equal reused.uuid, c_high_priority.uuid
 125   end
 126
 127   test "find_reusable method should select latest completed container" do
 128     set_user_from_auth :active
 129     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
 130     completed_attrs = {
 131       state: Container::Complete,
 132       exit_code: 0,
 133       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 134       output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 135     }
 136
 137     c_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 138     c_recent, _ = minimal_new(common_attrs.merge({use_existing: false}))
 139     assert_not_equal c_older.uuid, c_recent.uuid
 140
 141     set_user_from_auth :dispatch1
 142     c_older.update_attributes!({state: Container::Locked})
 143     c_older.update_attributes!({state: Container::Running})
 144     c_older.update_attributes!(completed_attrs)
 145
 146     c_recent.update_attributes!({state: Container::Locked})
 147     c_recent.update_attributes!({state: Container::Running})
 148     c_recent.update_attributes!(completed_attrs)
 149
 150     reused = Container.find_reusable(common_attrs)
 151     assert_not_nil reused
 152     assert_equal reused.uuid, c_older.uuid
 153   end
 154
 155   test "find_reusable method should select oldest completed container when inconsistent outputs exist" do
 156     set_user_from_auth :active
 157     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}, priority: 1})
 158     completed_attrs = {
 159       state: Container::Complete,
 160       exit_code: 0,
 161       log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 162     }
 163
 164     cr = ContainerRequest.new common_attrs
 165     cr.use_existing = false
 166     cr.state = ContainerRequest::Committed
 167     cr.save!
 168     c_output1 = Container.where(uuid: cr.container_uuid).first
 169
 170     cr = ContainerRequest.new common_attrs
 171     cr.use_existing = false
 172     cr.state = ContainerRequest::Committed
 173     cr.save!
 174     c_output2 = Container.where(uuid: cr.container_uuid).first
 175
 176     assert_not_equal c_output1.uuid, c_output2.uuid
 177
 178     set_user_from_auth :dispatch1
 179
 180     out1 = '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'
 181     log1 = collections(:real_log_collection).portable_data_hash
 182     c_output1.update_attributes!({state: Container::Locked})
 183     c_output1.update_attributes!({state: Container::Running})
 184     c_output1.update_attributes!(completed_attrs.merge({log: log1, output: out1}))
 185
 186     out2 = 'fa7aeb5140e2848d39b416daeef4ffc5+45'
 187     c_output2.update_attributes!({state: Container::Locked})
 188     c_output2.update_attributes!({state: Container::Running})
 189     c_output2.update_attributes!(completed_attrs.merge({log: log1, output: out2}))
 190
 191     reused = Container.resolve(ContainerRequest.new(common_attrs))
 192     assert_equal c_output1.uuid, reused.uuid
 193   end
 194
 195   test "find_reusable method should select running container by start date" do
 196     set_user_from_auth :active
 197     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
 198     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 199     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 200     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 201     # Confirm the 3 container UUIDs are different.
 202     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 203     set_user_from_auth :dispatch1
 204     c_slower.update_attributes!({state: Container::Locked})
 205     c_slower.update_attributes!({state: Container::Running,
 206                                  progress: 0.1})
 207     c_faster_started_first.update_attributes!({state: Container::Locked})
 208     c_faster_started_first.update_attributes!({state: Container::Running,
 209                                                progress: 0.15})
 210     c_faster_started_second.update_attributes!({state: Container::Locked})
 211     c_faster_started_second.update_attributes!({state: Container::Running,
 212                                                 progress: 0.15})
 213     reused = Container.find_reusable(common_attrs)
 214     assert_not_nil reused
 215     # Selected container is the one that started first
 216     assert_equal reused.uuid, c_faster_started_first.uuid
 217   end
 218
 219   test "find_reusable method should select running container by progress" do
 220     set_user_from_auth :active
 221     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
 222     c_slower, _ = minimal_new(common_attrs.merge({use_existing: false}))
 223     c_faster_started_first, _ = minimal_new(common_attrs.merge({use_existing: false}))
 224     c_faster_started_second, _ = minimal_new(common_attrs.merge({use_existing: false}))
 225     # Confirm the 3 container UUIDs are different.
 226     assert_equal 3, [c_slower.uuid, c_faster_started_first.uuid, c_faster_started_second.uuid].uniq.length
 227     set_user_from_auth :dispatch1
 228     c_slower.update_attributes!({state: Container::Locked})
 229     c_slower.update_attributes!({state: Container::Running,
 230                                  progress: 0.1})
 231     c_faster_started_first.update_attributes!({state: Container::Locked})
 232     c_faster_started_first.update_attributes!({state: Container::Running,
 233                                                progress: 0.15})
 234     c_faster_started_second.update_attributes!({state: Container::Locked})
 235     c_faster_started_second.update_attributes!({state: Container::Running,
 236                                                 progress: 0.2})
 237     reused = Container.find_reusable(common_attrs)
 238     assert_not_nil reused
 239     # Selected container is the one with most progress done
 240     assert_equal reused.uuid, c_faster_started_second.uuid
 241   end
 242
 243   test "find_reusable method should select locked container most likely to start sooner" do
 244     set_user_from_auth :active
 245     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
 246     c_low_priority, _ = minimal_new(common_attrs.merge({use_existing: false}))
 247     c_high_priority_older, _ = minimal_new(common_attrs.merge({use_existing: false}))
 248     c_high_priority_newer, _ = minimal_new(common_attrs.merge({use_existing: false}))
 249     # Confirm the 3 container UUIDs are different.
 250     assert_equal 3, [c_low_priority.uuid, c_high_priority_older.uuid, c_high_priority_newer.uuid].uniq.length
 251     set_user_from_auth :dispatch1
 252     c_low_priority.update_attributes!({state: Container::Locked,
 253                                        priority: 1})
 254     c_high_priority_older.update_attributes!({state: Container::Locked,
 255                                               priority: 2})
 256     c_high_priority_newer.update_attributes!({state: Container::Locked,
 257                                               priority: 2})
 258     reused = Container.find_reusable(common_attrs)
 259     assert_not_nil reused
 260     assert_equal reused.uuid, c_high_priority_older.uuid
 261   end
 262
 263   test "find_reusable method should select running over failed container" do
 264     set_user_from_auth :active
 265     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
 266     c_failed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 267     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 268     assert_not_equal c_failed.uuid, c_running.uuid
 269     set_user_from_auth :dispatch1
 270     c_failed.update_attributes!({state: Container::Locked})
 271     c_failed.update_attributes!({state: Container::Running})
 272     c_failed.update_attributes!({state: Container::Complete,
 273                                  exit_code: 42,
 274                                  log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 275                                  output: 'ea10d51bcf88862dbcc36eb292017dfd+45'})
 276     c_running.update_attributes!({state: Container::Locked})
 277     c_running.update_attributes!({state: Container::Running,
 278                                   progress: 0.15})
 279     reused = Container.find_reusable(common_attrs)
 280     assert_not_nil reused
 281     assert_equal reused.uuid, c_running.uuid
 282   end
 283
 284   test "find_reusable method should select complete over running container" do
 285     set_user_from_auth :active
 286     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
 287     c_completed, _ = minimal_new(common_attrs.merge({use_existing: false}))
 288     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 289     assert_not_equal c_completed.uuid, c_running.uuid
 290     set_user_from_auth :dispatch1
 291     c_completed.update_attributes!({state: Container::Locked})
 292     c_completed.update_attributes!({state: Container::Running})
 293     c_completed.update_attributes!({state: Container::Complete,
 294                                     exit_code: 0,
 295                                     log: 'ea10d51bcf88862dbcc36eb292017dfd+45',
 296                                     output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45'})
 297     c_running.update_attributes!({state: Container::Locked})
 298     c_running.update_attributes!({state: Container::Running,
 299                                   progress: 0.15})
 300     reused = Container.find_reusable(common_attrs)
 301     assert_not_nil reused
 302     assert_equal c_completed.uuid, reused.uuid
 303   end
 304
 305   test "find_reusable method should select running over locked container" do
 306     set_user_from_auth :active
 307     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 308     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 309     c_running, _ = minimal_new(common_attrs.merge({use_existing: false}))
 310     assert_not_equal c_running.uuid, c_locked.uuid
 311     set_user_from_auth :dispatch1
 312     c_locked.update_attributes!({state: Container::Locked})
 313     c_running.update_attributes!({state: Container::Locked})
 314     c_running.update_attributes!({state: Container::Running,
 315                                   progress: 0.15})
 316     reused = Container.find_reusable(common_attrs)
 317     assert_not_nil reused
 318     assert_equal reused.uuid, c_running.uuid
 319   end
 320
 321   test "find_reusable method should select locked over queued container" do
 322     set_user_from_auth :active
 323     common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running_vs_locked"}})
 324     c_locked, _ = minimal_new(common_attrs.merge({use_existing: false}))
 325     c_queued, _ = minimal_new(common_attrs.merge({use_existing: false}))
 326     assert_not_equal c_queued.uuid, c_locked.uuid
 327     set_user_from_auth :dispatch1
 328     c_locked.update_attributes!({state: Container::Locked})
 329     reused = Container.find_reusable(common_attrs)
 330     assert_not_nil reused
 331     assert_equal reused.uuid, c_locked.uuid
 332   end
 333
 334   test "find_reusable method should not select failed container" do
 335     set_user_from_auth :active
 336     attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed"}})
 337     c, _ = minimal_new(attrs)
 338     set_user_from_auth :dispatch1
 339     c.update_attributes!({state: Container::Locked})
 340     c.update_attributes!({state: Container::Running})
 341     c.update_attributes!({state: Container::Complete,
 342                           exit_code: 33})
 343     reused = Container.find_reusable(attrs)
 344     assert_nil reused
 345   end
 346
 347   test "find_reusable with logging disabled" do
 348     set_user_from_auth :active
 349     Rails.logger.expects(:info).never
 350     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 351   end
 352
 353   test "find_reusable with logging enabled" do
 354     set_user_from_auth :active
 355     Rails.configuration.log_reuse_decisions = true
 356     Rails.logger.expects(:info).at_least(3)
 357     Container.find_reusable(REUSABLE_COMMON_ATTRS)
 358   end
 359
 360   test "Container running" do
 361     c, _ = minimal_new priority: 1
 362
 363     set_user_from_auth :dispatch1
 364     check_illegal_updates c, [{state: Container::Running},
 365                               {state: Container::Complete}]
 366
 367     c.lock
 368     c.update_attributes! state: Container::Running
 369
 370     check_illegal_modify c
 371     check_bogus_states c
 372
 373     check_illegal_updates c, [{state: Container::Queued}]
 374     c.reload
 375
 376     c.update_attributes! priority: 3
 377   end
 378
 379   test "Lock and unlock" do
 380     c, cr = minimal_new priority: 0
 381
 382     set_user_from_auth :dispatch1
 383     assert_equal Container::Queued, c.state
 384
 385     assert_raise(ArvadosModel::LockFailedError) do
 386       # "no priority"
 387       c.lock
 388     end
 389     c.reload
 390     assert cr.update_attributes priority: 1
 391
 392     refute c.update_attributes(state: Container::Running), "not locked"
 393     c.reload
 394     refute c.update_attributes(state: Container::Complete), "not locked"
 395     c.reload
 396
 397     assert c.lock, show_errors(c)
 398     assert c.locked_by_uuid
 399     assert c.auth_uuid
 400
 401     assert_raise(ArvadosModel::LockFailedError) {c.lock}
 402     c.reload
 403
 404     assert c.unlock, show_errors(c)
 405     refute c.locked_by_uuid
 406     refute c.auth_uuid
 407
 408     refute c.update_attributes(state: Container::Running), "not locked"
 409     c.reload
 410     refute c.locked_by_uuid
 411     refute c.auth_uuid
 412
 413     assert c.lock, show_errors(c)
 414     assert c.update_attributes(state: Container::Running), show_errors(c)
 415     assert c.locked_by_uuid
 416     assert c.auth_uuid
 417
 418     auth_uuid_was = c.auth_uuid
 419
 420     assert_raise(ArvadosModel::LockFailedError) do
 421       # Running to Locked is not allowed
 422       c.lock
 423     end
 424     c.reload
 425     assert_raise(ArvadosModel::InvalidStateTransitionError) do
 426       # Running to Queued is not allowed
 427       c.unlock
 428     end
 429     c.reload
 430
 431     assert c.update_attributes(state: Container::Complete), show_errors(c)
 432     refute c.locked_by_uuid
 433     refute c.auth_uuid
 434
 435     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
 436     assert_operator auth_exp, :<, db_current_time
 437   end
 438
 439   test "Container queued cancel" do
 440     c, _ = minimal_new
 441     set_user_from_auth :dispatch1
 442     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 443     check_no_change_from_cancelled c
 444   end
 445
 446   test "Container locked cancel" do
 447     c, _ = minimal_new
 448     set_user_from_auth :dispatch1
 449     assert c.lock, show_errors(c)
 450     assert c.update_attributes(state: Container::Cancelled), show_errors(c)
 451     check_no_change_from_cancelled c
 452   end
 453
 454   test "Container running cancel" do
 455     c, _ = minimal_new
 456     set_user_from_auth :dispatch1
 457     c.lock
 458     c.update_attributes! state: Container::Running
 459     c.update_attributes! state: Container::Cancelled
 460     check_no_change_from_cancelled c
 461   end
 462
 463   test "Container create forbidden for non-admin" do
 464     set_user_from_auth :active_trustedclient
 465     c = Container.new DEFAULT_ATTRS
 466     c.environment = {}
 467     c.mounts = {"BAR" => "FOO"}
 468     c.output_path = "/tmp"
 469     c.priority = 1
 470     c.runtime_constraints = {}
 471     assert_raises(ArvadosModel::PermissionDeniedError) do
 472       c.save!
 473     end
 474   end
 475
 476   test "Container only set exit code on complete" do
 477     c, _ = minimal_new
 478     set_user_from_auth :dispatch1
 479     c.lock
 480     c.update_attributes! state: Container::Running
 481
 482     check_illegal_updates c, [{exit_code: 1},
 483                               {exit_code: 1, state: Container::Cancelled}]
 484
 485     assert c.update_attributes(exit_code: 1, state: Container::Complete)
 486   end
 487
 488   test "locked_by_uuid can set output on running container" do
 489     c, _ = minimal_new
 490     set_user_from_auth :dispatch1
 491     c.lock
 492     c.update_attributes! state: Container::Running
 493
 494     assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
 495
 496     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 497     assert c.update_attributes! state: Container::Complete
 498   end
 499
 500   test "auth_uuid can set output on running container, but not change container state" do
 501     c, _ = minimal_new
 502     set_user_from_auth :dispatch1
 503     c.lock
 504     c.update_attributes! state: Container::Running
 505
 506     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 507     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 508     assert c.update_attributes output: collections(:collection_owned_by_active).portable_data_hash
 509
 510     assert_raises ArvadosModel::PermissionDeniedError do
 511       # auth_uuid cannot set container state
 512       c.update_attributes state: Container::Complete
 513     end
 514   end
 515
 516   test "not allowed to set output that is not readable by current user" do
 517     c, _ = minimal_new
 518     set_user_from_auth :dispatch1
 519     c.lock
 520     c.update_attributes! state: Container::Running
 521
 522     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 523     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 524
 525     assert_raises ActiveRecord::RecordInvalid do
 526       c.update_attributes! output: collections(:collection_not_readable_by_active).portable_data_hash
 527     end
 528   end
 529
 530   test "other token cannot set output on running container" do
 531     c, _ = minimal_new
 532     set_user_from_auth :dispatch1
 533     c.lock
 534     c.update_attributes! state: Container::Running
 535
 536     set_user_from_auth :not_running_container_auth
 537     assert_raises ArvadosModel::PermissionDeniedError do
 538       c.update_attributes! output: collections(:foo_file).portable_data_hash
 539     end
 540   end
 541
 542   test "can set trashed output on running container" do
 543     c, _ = minimal_new
 544     set_user_from_auth :dispatch1
 545     c.lock
 546     c.update_attributes! state: Container::Running
 547
 548     output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jk')
 549
 550     assert output.is_trashed
 551     assert c.update_attributes output: output.portable_data_hash
 552     assert c.update_attributes! state: Container::Complete
 553   end
 554
 555   test "not allowed to set trashed output that is not readable by current user" do
 556     c, _ = minimal_new
 557     set_user_from_auth :dispatch1
 558     c.lock
 559     c.update_attributes! state: Container::Running
 560
 561     output = Collection.unscoped.find_by_uuid('zzzzz-4zz18-mto52zx1s7sn3jr')
 562
 563     Thread.current[:api_client_authorization] = ApiClientAuthorization.find_by_uuid(c.auth_uuid)
 564     Thread.current[:user] = User.find_by_id(Thread.current[:api_client_authorization].user_id)
 565
 566     assert_raises ActiveRecord::RecordInvalid do
 567       c.update_attributes! output: output.portable_data_hash
 568     end
 569   end
 570
 571 end